diff --git a/.ai/analytics-output-port-design.md b/.ai/analytics-output-port-design.md index 41ce63cca..e64af86ae 100644 --- a/.ai/analytics-output-port-design.md +++ b/.ai/analytics-output-port-design.md @@ -1,6 +1,7 @@ # Analytics Output Port Design ## Status: Approved + ## Date: 2025-01-21 ## Problem Statement @@ -12,6 +13,7 @@ When connecting a component's `rawOutput` (which contains complex nested JSON) t 3. **Varying schemas**: Different scanner outputs accumulate unique field paths over time Example error: + ``` illegal_argument_exception: Limit of total fields [1000] has been exceeded ``` @@ -51,6 +53,7 @@ illegal_argument_exception: Limit of total fields [1000] has been exceeded ### Document Structure **Before (PRD design):** + ```json { "workflow_id": "...", @@ -69,6 +72,7 @@ illegal_argument_exception: Limit of total fields [1000] has been exceeded ``` **After (new design):** + ```json { "check_id": "DB_RLS_DISABLED", @@ -97,13 +101,14 @@ illegal_argument_exception: Limit of total fields [1000] has been exceeded Components should use their existing structured list outputs: -| Component | Port | Type | Notes | -|-----------|------|------|-------| -| Nuclei | `results` | `z.array(z.record(z.string(), z.unknown()))` | Scanner + asset_key added | -| TruffleHog | `results` | `z.array(z.record(z.string(), z.unknown()))` | Scanner + asset_key added | +| Component | Port | Type | Notes | +| ---------------- | --------- | -------------------------------------------- | ------------------------- | +| Nuclei | `results` | `z.array(z.record(z.string(), z.unknown()))` | Scanner + asset_key added | +| TruffleHog | `results` | `z.array(z.record(z.string(), z.unknown()))` | Scanner + asset_key added | | Supabase Scanner | `results` | `z.array(z.record(z.string(), z.unknown()))` | Scanner + asset_key added | All `results` ports include: + - `scanner`: Scanner identifier (e.g., `'nuclei'`, `'trufflehog'`, `'supabase-scanner'`) - `asset_key`: Primary asset identifier from the finding - `finding_hash`: Stable hash for deduplication (16-char hex from SHA-256) @@ -113,6 +118,7 @@ All `results` ports include: The `finding_hash` enables tracking findings across workflow runs: **Generation:** + ```typescript import { createHash } from 'crypto'; @@ -130,6 +136,7 @@ function generateFindingHash(...fields: (string | undefined | null)[]): string { | Supabase Scanner | `check_id + projectRef + resource` | **Use cases:** + - **New vs recurring**: Is this finding appearing for the first time? - **First-seen / last-seen**: When did we first detect this? Is it still present? - **Resolution tracking**: Findings that stop appearing may be resolved @@ -139,19 +146,20 @@ function generateFindingHash(...fields: (string | undefined | null)[]): string { The indexer automatically adds these fields under `shipsec`: -| Field | Description | -|-------|-------------| -| `organization_id` | Organization that owns the workflow | -| `run_id` | Unique identifier for this workflow execution | -| `workflow_id` | ID of the workflow definition | -| `workflow_name` | Human-readable workflow name | -| `component_id` | Component type (e.g., `core.analytics.sink`) | -| `node_ref` | Node reference in the workflow graph | -| `asset_key` | Auto-detected or specified asset identifier | +| Field | Description | +| ----------------- | --------------------------------------------- | +| `organization_id` | Organization that owns the workflow | +| `run_id` | Unique identifier for this workflow execution | +| `workflow_id` | ID of the workflow definition | +| `workflow_name` | Human-readable workflow name | +| `component_id` | Component type (e.g., `core.analytics.sink`) | +| `node_ref` | Node reference in the workflow graph | +| `asset_key` | Auto-detected or specified asset identifier | ### Querying in OpenSearch With this structure, users can: + - Filter by organization: `shipsec.organization_id: "org_123"` - Filter by workflow: `shipsec.workflow_id: "xxx"` - Filter by run: `shipsec.run_id: "xxx"` @@ -164,11 +172,11 @@ With this structure, users can: ### Trade-offs -| Decision | Pro | Con | -|----------|-----|-----| -| Serialize nested objects | Prevents field explosion | Can't query inside serialized fields | -| `shipsec` namespace | No field collision | Slightly more verbose queries | -| No generic schema | Better fit per component | Less consistency across components | +| Decision | Pro | Con | +| ------------------------ | ------------------------- | ------------------------------------------ | +| Serialize nested objects | Prevents field explosion | Can't query inside serialized fields | +| `shipsec` namespace | No field collision | Slightly more verbose queries | +| No generic schema | Better fit per component | Less consistency across components | | Same timestamp per batch | Accurate (same scan time) | Can't distinguish individual finding times | ### Implementation Files diff --git a/.github/workflows/upstream-sync.yml b/.github/workflows/upstream-sync.yml new file mode 100644 index 000000000..3f1488c19 --- /dev/null +++ b/.github/workflows/upstream-sync.yml @@ -0,0 +1,83 @@ +name: Sync upstream main + +on: + schedule: + - cron: "0 9 * * *" # Once daily at 9am UTC + workflow_dispatch: # Manual trigger + +permissions: + contents: write + pull-requests: write + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Add upstream remote + run: | + git remote add upstream https://github.com/ShipSecAI/studio.git || true + git fetch upstream main + + - name: Check for divergence + id: check + run: | + UPSTREAM_SHA=$(git rev-parse upstream/main) + # Check if upstream-sync branch exists on origin + if git ls-remote --exit-code origin upstream-sync &>/dev/null; then + CURRENT_SHA=$(git rev-parse origin/upstream-sync) + else + CURRENT_SHA="" + fi + + if [ "$UPSTREAM_SHA" = "$CURRENT_SHA" ]; then + echo "skip=true" >> "$GITHUB_OUTPUT" + echo "No new upstream commits" + else + echo "skip=false" >> "$GITHUB_OUTPUT" + AHEAD=$(git rev-list --count origin/main..upstream/main) + echo "ahead=$AHEAD" >> "$GITHUB_OUTPUT" + echo "Upstream is $AHEAD commits ahead" + fi + + - name: Push upstream-sync branch + if: steps.check.outputs.skip == 'false' + run: | + git checkout -B upstream-sync upstream/main + git push origin upstream-sync --force + + - name: Create or update PR + if: steps.check.outputs.skip == 'false' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + EXISTING_PR=$(gh pr list --head upstream-sync --base main --state open --json number --jq '.[0].number' 2>/dev/null || echo "") + + if [ -n "$EXISTING_PR" ]; then + echo "PR #$EXISTING_PR already exists, updated sync branch" + gh pr comment "$EXISTING_PR" --body "Sync branch updated. Upstream is now ${{ steps.check.outputs.ahead }} commits ahead of main." + else + gh pr create \ + --head upstream-sync \ + --base main \ + --title "sync: merge upstream main" \ + --body "$(cat <<'EOF' +Automated sync from [ShipSecAI/studio](https://github.com/ShipSecAI/studio) main. + +**${{ steps.check.outputs.ahead }} new upstream commits.** + +Review the changes and merge when ready. If there are conflicts, resolve them locally: +```bash +git fetch origin upstream-sync main +git checkout main +git merge origin/upstream-sync +# resolve conflicts +git push origin main +``` +EOF +)" + fi diff --git a/.gitignore b/.gitignore index 0b0e85af3..aef1bc115 100644 --- a/.gitignore +++ b/.gitignore @@ -75,3 +75,17 @@ vite.config.ts.timestamp-* .playground/ .omc/ MCP_FLOW_TRACE.md + +# Terraform / OpenTofu +.terraform/ +*.tfstate +*.tfstate.* +*.tfvars +*.tfvars.json +crash.log +crash.*.log +override.tf +override.tf.json +*_override.tf +*_override.tf.json +.terraform.lock.hcl.bak diff --git a/.prettierignore b/.prettierignore index 373eb18fc..e1eb6576f 100644 --- a/.prettierignore +++ b/.prettierignore @@ -11,3 +11,9 @@ node_modules/ # Generated files *.generated.ts + +# Helm templates (Go template syntax is not valid YAML) +deploy/helm/*/templates/ + +# GitHub Actions (uses ${{ }} template syntax) +.github/workflows/ diff --git a/Dockerfile b/Dockerfile index c2f28972e..aa8313a62 100644 --- a/Dockerfile +++ b/Dockerfile @@ -83,8 +83,8 @@ FROM base AS frontend # Frontend build-time configuration ARG VITE_AUTH_PROVIDER=local ARG VITE_CLERK_PUBLISHABLE_KEY="" -ARG VITE_API_URL=http://localhost:3211 -ARG VITE_BACKEND_URL=http://localhost:3211 +ARG VITE_API_URL="" +ARG VITE_BACKEND_URL="" ARG VITE_DEFAULT_ORG_ID=local-dev ARG VITE_GIT_SHA=unknown ARG VITE_PUBLIC_POSTHOG_KEY="" @@ -125,8 +125,8 @@ FROM base AS frontend-debug # Frontend build-time configuration ARG VITE_AUTH_PROVIDER=local ARG VITE_CLERK_PUBLISHABLE_KEY="" -ARG VITE_API_URL=http://localhost:3211 -ARG VITE_BACKEND_URL=http://localhost:3211 +ARG VITE_API_URL="" +ARG VITE_BACKEND_URL="" ARG VITE_DEFAULT_ORG_ID=local-dev ARG VITE_GIT_SHA=unknown ARG VITE_PUBLIC_POSTHOG_KEY="" diff --git a/bun.lock b/bun.lock index b440ecb48..98bfb4367 100644 --- a/bun.lock +++ b/bun.lock @@ -260,8 +260,10 @@ "@ai-sdk/mcp": "^1.0.13", "@ai-sdk/openai": "^3.0.18", "@aws-sdk/client-s3": "^3.975.0", + "@google-cloud/storage": "^7.14.0", "@googleapis/admin": "^29.0.0", "@grpc/grpc-js": "^1.14.3", + "@kubernetes/client-node": "^1.4.0", "@modelcontextprotocol/sdk": "^1.25.1", "@okta/okta-sdk-nodejs": "^7.3.0", "@opensearch-project/opensearch": "^3.5.1", @@ -580,6 +582,14 @@ "@floating-ui/utils": ["@floating-ui/utils@0.2.10", "", {}, "sha512-aGTxbpbg8/b5JfU1HXSrbH3wXZuLPJcNEcZQFMxLs3oSzgtVu6nFPkbbGGUvBcUjKV2YyB9Wxxabo+HEH9tcRQ=="], + "@google-cloud/paginator": ["@google-cloud/paginator@5.0.2", "", { "dependencies": { "arrify": "^2.0.0", "extend": "^3.0.2" } }, "sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg=="], + + "@google-cloud/projectify": ["@google-cloud/projectify@4.0.0", "", {}, "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA=="], + + "@google-cloud/promisify": ["@google-cloud/promisify@4.0.0", "", {}, "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g=="], + + "@google-cloud/storage": ["@google-cloud/storage@7.19.0", "", { "dependencies": { "@google-cloud/paginator": "^5.0.0", "@google-cloud/projectify": "^4.0.0", "@google-cloud/promisify": "<4.1.0", "abort-controller": "^3.0.0", "async-retry": "^1.3.3", "duplexify": "^4.1.3", "fast-xml-parser": "^5.3.4", "gaxios": "^6.0.2", "google-auth-library": "^9.6.3", "html-entities": "^2.5.2", "mime": "^3.0.0", "p-limit": "^3.0.1", "retry-request": "^7.0.0", "teeny-request": "^9.0.0", "uuid": "^8.0.0" } }, "sha512-n2FjE7NAOYyshogdc7KQOl/VZb4sneqPjWouSyia9CMDdMhRX5+RIbqalNmC7LOLzuLAN89VlF2HvG8na9G+zQ=="], + "@googleapis/admin": ["@googleapis/admin@30.3.0", "", { "dependencies": { "googleapis-common": "^8.0.0" } }, "sha512-9vBP163vUDGb7BZuGat0Hzajf010t4HuXrR13MWDF/2pCNcg65gAAOzu3PSTIcqiuxL7nsjhkzj+oxg/t7s3vA=="], "@grpc/grpc-js": ["@grpc/grpc-js@1.14.3", "", { "dependencies": { "@grpc/proto-loader": "^0.8.0", "@js-sdsl/ordered-map": "^4.4.2" } }, "sha512-Iq8QQQ/7X3Sac15oB6p0FmUg/klxQvXLeileoqrTRGJYLV+/9tubbr9ipz0GKHjmXVsgFPo/+W+2cA8eNcR+XA=="], @@ -614,6 +624,10 @@ "@js-sdsl/ordered-map": ["@js-sdsl/ordered-map@4.4.2", "", {}, "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw=="], + "@jsep-plugin/assignment": ["@jsep-plugin/assignment@1.3.0", "", { "peerDependencies": { "jsep": "^0.4.0||^1.0.0" } }, "sha512-VVgV+CXrhbMI3aSusQyclHkenWSAm95WaiKrMxRFam3JSUiIaQjoMIw2sEs/OX4XifnqeQUN4DYbJjlA8EfktQ=="], + + "@jsep-plugin/regex": ["@jsep-plugin/regex@1.0.4", "", { "peerDependencies": { "jsep": "^0.4.0||^1.0.0" } }, "sha512-q7qL4Mgjs1vByCaTnDFcBnV9HS7GVPJX5vyVoCgZHNSC9rjwIlmbXG5sUuorR5ndfHAIlJ8pVStxvjXHbNvtUg=="], + "@jsonjoy.com/base64": ["@jsonjoy.com/base64@1.1.2", "", { "peerDependencies": { "tslib": "2" } }, "sha512-q6XAnWQDIMA3+FTiOYajoYqySkO+JSat0ytXGSuRdq9uXE7o92gzuQwQM14xaCRlBLGq3v5miDGC4vkVTn54xA=="], "@jsonjoy.com/buffers": ["@jsonjoy.com/buffers@17.65.0", "", { "peerDependencies": { "tslib": "2" } }, "sha512-eBrIXd0/Ld3p9lpDDlMaMn6IEfWqtHMD+z61u0JrIiPzsV1r7m6xDZFRxJyvIFTEO+SWdYF9EiQbXZGd8BzPfA=="], @@ -642,6 +656,8 @@ "@jsonjoy.com/util": ["@jsonjoy.com/util@1.9.0", "", { "dependencies": { "@jsonjoy.com/buffers": "^1.0.0", "@jsonjoy.com/codegen": "^1.0.0" }, "peerDependencies": { "tslib": "2" } }, "sha512-pLuQo+VPRnN8hfPqUTLTHk126wuYdXVxE6aDmjSeV4NCAgyxWbiOIeNJVtID3h1Vzpoi9m4jXezf73I6LgabgQ=="], + "@kubernetes/client-node": ["@kubernetes/client-node@1.4.0", "", { "dependencies": { "@types/js-yaml": "^4.0.1", "@types/node": "^24.0.0", "@types/node-fetch": "^2.6.13", "@types/stream-buffers": "^3.0.3", "form-data": "^4.0.0", "hpagent": "^1.2.0", "isomorphic-ws": "^5.0.0", "js-yaml": "^4.1.0", "jsonpath-plus": "^10.3.0", "node-fetch": "^2.7.0", "openid-client": "^6.1.3", "rfc4648": "^1.3.0", "socks-proxy-agent": "^8.0.4", "stream-buffers": "^3.0.2", "tar-fs": "^3.0.9", "ws": "^8.18.2" } }, "sha512-Zge3YvF7DJi264dU1b3wb/GmzR99JhUpqTvp+VGHfwZT+g7EOOYNScDJNZwXy9cszyIGPIs0VHr+kk8e95qqrA=="], + "@lukeed/csprng": ["@lukeed/csprng@1.1.0", "", {}, "sha512-Z7C/xXCiGWsg0KuKsHTKJxbWhpI3Vs5GwLfOean7MGyVFGqdRgBbAjOCh6u4bbjPc/8MJ2pZmK/0DLdCbivLDA=="], "@microsoft/tsdoc": ["@microsoft/tsdoc@0.16.0", "", {}, "sha512-xgAyonlVVS+q7Vc7qLW0UrJU7rSFcETRWsqdXZtjzRU8dF+6CkozTK4V4y1LwOX7j8r/vHphjDeMeGI4tNGeGA=="], @@ -1080,6 +1096,8 @@ "@tokenizer/token": ["@tokenizer/token@0.3.0", "", {}, "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A=="], + "@tootallnate/once": ["@tootallnate/once@2.0.0", "", {}, "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A=="], + "@tootallnate/quickjs-emscripten": ["@tootallnate/quickjs-emscripten@0.23.0", "", {}, "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA=="], "@types/adm-zip": ["@types/adm-zip@0.5.7", "", { "dependencies": { "@types/node": "*" } }, "sha512-DNEs/QvmyRLurdQPChqq0Md4zGvPwHerAJYWk9l2jCbD1VPpnzRJorOdiq4zsw09NFbYnhfsoEhWtxIzXpn2yw=="], @@ -1100,6 +1118,8 @@ "@types/bun": ["@types/bun@1.3.8", "", { "dependencies": { "bun-types": "1.3.8" } }, "sha512-3LvWJ2q5GerAXYxO2mffLTqOzEu5qnhEAlh48Vnu8WQfnmSwbgagjGZV6BoHKJztENYEDn6QmVd949W4uESRJA=="], + "@types/caseless": ["@types/caseless@0.12.5", "", {}, "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg=="], + "@types/connect": ["@types/connect@3.4.38", "", { "dependencies": { "@types/node": "*" } }, "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug=="], "@types/cookie-parser": ["@types/cookie-parser@1.4.10", "", { "peerDependencies": { "@types/express": "*" } }, "sha512-B4xqkqfZ8Wek+rCOeRxsjMS9OgvzebEzzLYw7NHYuvzb7IdxOkI0ZHGgeEBX4PUM7QGVvNSK60T3OvWj3YfBRg=="], @@ -1218,6 +1238,8 @@ "@types/node": ["@types/node@24.10.9", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-ne4A0IpG3+2ETuREInjPNhUGis1SFjv1d5asp8MzEAGtOZeTeHVDOYqOgqfhvseqg/iXty2hjBf1zAOb7RNiNw=="], + "@types/node-fetch": ["@types/node-fetch@2.6.13", "", { "dependencies": { "@types/node": "*", "form-data": "^4.0.4" } }, "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw=="], + "@types/node-forge": ["@types/node-forge@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-mhVF2BnD4BO+jtOp7z1CdzaK4mbuK0LLQYAvdOLqHTavxFNq4zA1EmYkpnFjP8HOUzedfQkRnp0E2ulSAYSzAw=="], "@types/normalize-package-data": ["@types/normalize-package-data@2.4.4", "", {}, "sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA=="], @@ -1236,16 +1258,22 @@ "@types/readable-stream": ["@types/readable-stream@4.0.23", "", { "dependencies": { "@types/node": "*" } }, "sha512-wwXrtQvbMHxCbBgjHaMGEmImFTQxxpfMOR/ZoQnXxB1woqkUbdLGFDgauo00Py9IudiaqSeiBiulSV9i6XIPig=="], + "@types/request": ["@types/request@2.48.13", "", { "dependencies": { "@types/caseless": "*", "@types/node": "*", "@types/tough-cookie": "*", "form-data": "^2.5.5" } }, "sha512-FGJ6udDNUCjd19pp0Q3iTiDkwhYup7J8hpMW9c4k53NrccQFFWKRho6hvtPPEhnXWKvukfwAlB6DbDz4yhH5Gg=="], + "@types/send": ["@types/send@1.2.1", "", { "dependencies": { "@types/node": "*" } }, "sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ=="], "@types/serve-static": ["@types/serve-static@2.2.0", "", { "dependencies": { "@types/http-errors": "*", "@types/node": "*" } }, "sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ=="], + "@types/stream-buffers": ["@types/stream-buffers@3.0.8", "", { "dependencies": { "@types/node": "*" } }, "sha512-J+7VaHKNvlNPJPEJXX/fKa9DZtR/xPMwuIbe+yNOwp1YB+ApUOBv2aUpEoBJEi8nJgbgs1x8e73ttg0r1rSUdw=="], + "@types/superagent": ["@types/superagent@8.1.9", "", { "dependencies": { "@types/cookiejar": "^2.1.5", "@types/methods": "^1.1.4", "@types/node": "*", "form-data": "^4.0.0" } }, "sha512-pTVjI73witn+9ILmoJdajHGW2jkSaOzhiFYF1Rd3EQ94kymLqB9PjD9ISg7WaALC7+dCHT0FGe9T2LktLq/3GQ=="], "@types/supertest": ["@types/supertest@2.0.16", "", { "dependencies": { "@types/superagent": "*" } }, "sha512-6c2ogktZ06tr2ENoZivgm7YnprnhYE4ZoXGMY+oA7IuAf17M8FWvujXZGmxLv8y0PTyts4x5A+erSwVUFA8XSg=="], "@types/tinycolor2": ["@types/tinycolor2@1.4.6", "", {}, "sha512-iEN8J0BoMnsWBqjVbWH/c0G0Hh7O21lpR2/+PrvAVgWdzL7eexIFm4JN/Wn10PTcmNdtS6U67r499mlWMXOxNw=="], + "@types/tough-cookie": ["@types/tough-cookie@4.0.5", "", {}, "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA=="], + "@types/trusted-types": ["@types/trusted-types@2.0.7", "", {}, "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw=="], "@types/unist": ["@types/unist@3.0.3", "", {}, "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q=="], @@ -1392,7 +1420,7 @@ "arraybuffer.prototype.slice": ["arraybuffer.prototype.slice@1.0.4", "", { "dependencies": { "array-buffer-byte-length": "^1.0.1", "call-bind": "^1.0.8", "define-properties": "^1.2.1", "es-abstract": "^1.23.5", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.6", "is-array-buffer": "^3.0.4" } }, "sha512-BNoCY6SXXPQ7gF2opIP4GBE+Xw7U+pHMYKuzjgCN3GwiaIR09UUeKfheyIry77QtrCBlC0KK0q5/TER/tYh3PQ=="], - "arrify": ["arrify@1.0.1", "", {}, "sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA=="], + "arrify": ["arrify@2.0.1", "", {}, "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug=="], "asap": ["asap@2.0.6", "", {}, "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA=="], @@ -1402,6 +1430,8 @@ "async-function": ["async-function@1.0.0", "", {}, "sha512-hsU18Ae8CDTR6Kgu9DYf0EbCr/a5iGL0rytQDobUcdpYOKokk8LEjVphnXkDkgpi0wYVsqrXuP0bZxJaTqdgoA=="], + "async-retry": ["async-retry@1.3.3", "", { "dependencies": { "retry": "0.13.1" } }, "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw=="], + "asynckit": ["asynckit@0.4.0", "", {}, "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="], "autoprefixer": ["autoprefixer@10.4.24", "", { "dependencies": { "browserslist": "^4.28.1", "caniuse-lite": "^1.0.30001766", "fraction.js": "^5.3.4", "picocolors": "^1.1.1", "postcss-value-parser": "^4.2.0" }, "peerDependencies": { "postcss": "^8.1.0" }, "bin": { "autoprefixer": "bin/autoprefixer" } }, "sha512-uHZg7N9ULTVbutaIsDRoUkoS8/h3bdsmVJYZ5l3wv8Cp/6UIIoRDm90hZ+BwxUj/hGBEzLxdHNSKuFpn8WOyZw=="], @@ -1410,10 +1440,24 @@ "aws4": ["aws4@1.13.2", "", {}, "sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw=="], + "b4a": ["b4a@1.7.5", "", { "peerDependencies": { "react-native-b4a": "*" }, "optionalPeers": ["react-native-b4a"] }, "sha512-iEsKNwDh1wiWTps1/hdkNdmBgDlDVZP5U57ZVOlt+dNFqpc/lpPouCIxZw+DYBgc4P9NDfIZMPNR4CHNhzwLIA=="], + "bail": ["bail@2.0.2", "", {}, "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw=="], "balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], + "bare-events": ["bare-events@2.8.2", "", { "peerDependencies": { "bare-abort-controller": "*" }, "optionalPeers": ["bare-abort-controller"] }, "sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ=="], + + "bare-fs": ["bare-fs@4.5.4", "", { "dependencies": { "bare-events": "^2.5.4", "bare-path": "^3.0.0", "bare-stream": "^2.6.4", "bare-url": "^2.2.2", "fast-fifo": "^1.3.2" }, "peerDependencies": { "bare-buffer": "*" }, "optionalPeers": ["bare-buffer"] }, "sha512-POK4oplfA7P7gqvetNmCs4CNtm9fNsx+IAh7jH7GgU0OJdge2rso0R20TNWVq6VoWcCvsTdlNDaleLHGaKx8CA=="], + + "bare-os": ["bare-os@3.6.2", "", {}, "sha512-T+V1+1srU2qYNBmJCXZkUY5vQ0B4FSlL3QDROnKQYOqeiQR8UbjNHlPa+TIbM4cuidiN9GaTaOZgSEgsvPbh5A=="], + + "bare-path": ["bare-path@3.0.0", "", { "dependencies": { "bare-os": "^3.0.1" } }, "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw=="], + + "bare-stream": ["bare-stream@2.7.0", "", { "dependencies": { "streamx": "^2.21.0" }, "peerDependencies": { "bare-buffer": "*", "bare-events": "*" }, "optionalPeers": ["bare-buffer", "bare-events"] }, "sha512-oyXQNicV1y8nc2aKffH+BUHFRXmx6VrPzlnaEvMhram0nPBrKcEdcyBg5r08D0i8VxngHFAiVyn1QKXpSG0B8A=="], + + "bare-url": ["bare-url@2.3.2", "", { "dependencies": { "bare-path": "^3.0.0" } }, "sha512-ZMq4gd9ngV5aTMa5p9+UfY0b3skwhHELaDkhEHetMdX0LRkW9kzaym4oo/Eh+Ghm0CCDuMTsRIGM/ytUc1ZYmw=="], + "base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="], "base64url": ["base64url@3.0.1", "", {}, "sha512-ir1UPr3dkwexU7FdV8qBBbNDRUhMmIekYMFZfi+C/sLNnRESKPl23nB9b2pltqfOQNnGzsDdId90AEtG5tCx4A=="], @@ -1680,6 +1724,8 @@ "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], + "duplexify": ["duplexify@4.1.3", "", { "dependencies": { "end-of-stream": "^1.4.1", "inherits": "^2.0.3", "readable-stream": "^3.1.1", "stream-shift": "^1.0.2" } }, "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA=="], + "eastasianwidth": ["eastasianwidth@0.2.0", "", {}, "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="], "ecdsa-sig-formatter": ["ecdsa-sig-formatter@1.0.11", "", { "dependencies": { "safe-buffer": "^5.0.1" } }, "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ=="], @@ -1694,6 +1740,8 @@ "encodeurl": ["encodeurl@2.0.0", "", {}, "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="], + "end-of-stream": ["end-of-stream@1.4.5", "", { "dependencies": { "once": "^1.4.0" } }, "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg=="], + "enhanced-resolve": ["enhanced-resolve@5.18.4", "", { "dependencies": { "graceful-fs": "^4.2.4", "tapable": "^2.2.0" } }, "sha512-LgQMM4WXU3QI+SYgEc2liRgznaD5ojbmY3sb8LxyguVkIg5FxdpTkvk72te2R38/TGKxH634oLxXRGY6d7AP+Q=="], "enquirer": ["enquirer@2.3.6", "", { "dependencies": { "ansi-colors": "^4.1.1" } }, "sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg=="], @@ -1776,6 +1824,8 @@ "events": ["events@3.3.0", "", {}, "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q=="], + "events-universal": ["events-universal@1.0.1", "", { "dependencies": { "bare-events": "^2.7.0" } }, "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw=="], + "eventsource": ["eventsource@3.0.7", "", { "dependencies": { "eventsource-parser": "^3.0.1" } }, "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA=="], "eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="], @@ -1792,6 +1842,8 @@ "fast-diff": ["fast-diff@1.3.0", "", {}, "sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw=="], + "fast-fifo": ["fast-fifo@1.3.2", "", {}, "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ=="], + "fast-glob": ["fast-glob@3.3.3", "", { "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", "glob-parent": "^5.1.2", "merge2": "^1.3.0", "micromatch": "^4.0.8" } }, "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg=="], "fast-json-patch": ["fast-json-patch@3.1.1", "", {}, "sha512-vf6IHUX2SBcA+5/+4883dsIjpBTqmfBjmYiWK1savxQmFk4JfBMLa7ynTYOs1Rolp/T1betJxHiGD3g1Mn8lUQ=="], @@ -1864,7 +1916,7 @@ "functions-have-names": ["functions-have-names@1.2.3", "", {}, "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ=="], - "gaxios": ["gaxios@7.1.3", "", { "dependencies": { "extend": "^3.0.2", "https-proxy-agent": "^7.0.1", "node-fetch": "^3.3.2", "rimraf": "^5.0.1" } }, "sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ=="], + "gaxios": ["gaxios@6.7.1", "", { "dependencies": { "extend": "^3.0.2", "https-proxy-agent": "^7.0.1", "is-stream": "^2.0.0", "node-fetch": "^2.6.9", "uuid": "^9.0.1" } }, "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ=="], "gcp-metadata": ["gcp-metadata@8.1.2", "", { "dependencies": { "gaxios": "^7.0.0", "google-logging-utils": "^1.0.0", "json-bigint": "^1.0.0" } }, "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg=="], @@ -1976,6 +2028,8 @@ "html-encoding-sniffer": ["html-encoding-sniffer@6.0.0", "", { "dependencies": { "@exodus/bytes": "^1.6.0" } }, "sha512-CV9TW3Y3f8/wT0BRFc1/KAVQ3TUHiXmaAb6VW9vtiMFf7SLoMd1PdAc4W3KFOFETBJUb90KatHqlsZMWV+R9Gg=="], + "html-entities": ["html-entities@2.6.0", "", {}, "sha512-kig+rMn/QOVRvr7c86gQ8lWXq+Hkv6CbAH1hLu+RG338StTpE8Z0b44SDVaqVu7HGKf27frdmUYEs9hTUX/cLQ=="], + "html-to-image": ["html-to-image@1.11.11", "", {}, "sha512-9gux8QhvjRO/erSnDPv28noDZcPZmYE7e1vFsBLKLlRlKDSqNJYebj6Qz1TGd5lsRV+X+xYyjCKjuZdABinWjA=="], "html-url-attributes": ["html-url-attributes@3.0.1", "", {}, "sha512-ol6UPyBWqsrO6EJySPz2O7ZSr856WDrEzM5zMqp+FJJLGMW35cLYmmZnl0vztAZxRUoNZJFTCohfjuIJ8I4QBQ=="], @@ -2080,6 +2134,8 @@ "is-shared-array-buffer": ["is-shared-array-buffer@1.0.4", "", { "dependencies": { "call-bound": "^1.0.3" } }, "sha512-ISWac8drv4ZGfwKl5slpHG9OwPNty4jOWPRIhBpxOoD+hqITiwuipOQ2bNthAzwA3B4fIjO4Nln74N0S9byq8A=="], + "is-stream": ["is-stream@2.0.1", "", {}, "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg=="], + "is-string": ["is-string@1.1.1", "", { "dependencies": { "call-bound": "^1.0.3", "has-tostringtag": "^1.0.2" } }, "sha512-BtEeSsoaQjlSPBemMQIrY1MY0uM6vnS1g5fmufYOtnxLGUZM2178PKbhsk7Ffv58IX+ZtcvoGwccYsh0PglkAA=="], "is-symbol": ["is-symbol@1.1.1", "", { "dependencies": { "call-bound": "^1.0.2", "has-symbols": "^1.1.0", "safe-regex-test": "^1.1.0" } }, "sha512-9gGx6GTtCQM73BgmHQXfDmLtfjjTUDSyoxTCbp5WtoixAhfgsDirWIcVQ/IHpvI5Vgd5i/J5F7B9cN/WlVbC/w=="], @@ -2096,6 +2152,8 @@ "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], + "isomorphic-ws": ["isomorphic-ws@5.0.0", "", { "peerDependencies": { "ws": "*" } }, "sha512-muId7Zzn9ywDsyXgTIafTry2sV3nySZeUDe6YedVd1Hvuuep5AsIlqK+XefWpYTyJG5e503F2xIuT2lcU6rCSw=="], + "iterare": ["iterare@1.2.1", "", {}, "sha512-RKYVTCjAnRthyJes037NX/IiqeidgN1xc3j1RjFfECFp28A1GVwK9nA+i0rJPaHqSZwygLzRnFlzUuHFoWWy+Q=="], "iterator.prototype": ["iterator.prototype@1.1.5", "", { "dependencies": { "define-data-property": "^1.1.4", "es-object-atoms": "^1.0.0", "get-intrinsic": "^1.2.6", "get-proto": "^1.0.0", "has-symbols": "^1.1.0", "set-function-name": "^2.0.2" } }, "sha512-H0dkQoCa3b2VEeKQBOxFph+JAbcrQdE7KC0UkqwpLmv2EC4P41QXP+rqo9wYodACiG5/WM5s9oDApTU8utwj9g=="], @@ -2122,6 +2180,8 @@ "jsdom": ["jsdom@27.4.0", "", { "dependencies": { "@acemir/cssom": "^0.9.28", "@asamuzakjp/dom-selector": "^6.7.6", "@exodus/bytes": "^1.6.0", "cssstyle": "^5.3.4", "data-urls": "^6.0.0", "decimal.js": "^10.6.0", "html-encoding-sniffer": "^6.0.0", "http-proxy-agent": "^7.0.2", "https-proxy-agent": "^7.0.6", "is-potential-custom-element-name": "^1.0.1", "parse5": "^8.0.0", "saxes": "^6.0.0", "symbol-tree": "^3.2.4", "tough-cookie": "^6.0.0", "w3c-xmlserializer": "^5.0.0", "webidl-conversions": "^8.0.0", "whatwg-mimetype": "^4.0.0", "whatwg-url": "^15.1.0", "ws": "^8.18.3", "xml-name-validator": "^5.0.0" }, "peerDependencies": { "canvas": "^3.0.0" }, "optionalPeers": ["canvas"] }, "sha512-mjzqwWRD9Y1J1KUi7W97Gja1bwOOM5Ug0EZ6UDK3xS7j7mndrkwozHtSblfomlzyB4NepioNt+B2sOSzczVgtQ=="], + "jsep": ["jsep@1.4.0", "", {}, "sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw=="], + "jsesc": ["jsesc@3.1.0", "", { "bin": { "jsesc": "bin/jsesc" } }, "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA=="], "json-bigint": ["json-bigint@1.0.0", "", { "dependencies": { "bignumber.js": "^9.0.0" } }, "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ=="], @@ -2144,6 +2204,8 @@ "json5": ["json5@2.2.3", "", { "bin": { "json5": "lib/cli.js" } }, "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg=="], + "jsonpath-plus": ["jsonpath-plus@10.4.0", "", { "dependencies": { "@jsep-plugin/assignment": "^1.3.0", "@jsep-plugin/regex": "^1.0.4", "jsep": "^1.4.0" }, "bin": { "jsonpath": "bin/jsonpath-cli.js", "jsonpath-plus": "bin/jsonpath-cli.js" } }, "sha512-T92WWatJXmhBbKsgH/0hl+jxjdXrifi5IKeMY02DWggRxX0UElcbVzPlmgLTbvsPeW1PasQ6xE2Q75stkhGbsA=="], + "jsx-ast-utils": ["jsx-ast-utils@3.3.5", "", { "dependencies": { "array-includes": "^3.1.6", "array.prototype.flat": "^1.3.1", "object.assign": "^4.1.4", "object.values": "^1.1.6" } }, "sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ=="], "jwa": ["jwa@2.0.1", "", { "dependencies": { "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg=="], @@ -2320,7 +2382,7 @@ "micromatch": ["micromatch@4.0.8", "", { "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" } }, "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA=="], - "mime": ["mime@2.6.0", "", { "bin": { "mime": "cli.js" } }, "sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg=="], + "mime": ["mime@3.0.0", "", { "bin": { "mime": "cli.js" } }, "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A=="], "mime-db": ["mime-db@1.54.0", "", {}, "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ=="], @@ -2404,6 +2466,8 @@ "number-allocator": ["number-allocator@1.0.14", "", { "dependencies": { "debug": "^4.3.1", "js-sdsl": "4.3.0" } }, "sha512-OrL44UTVAvkKdOdRQZIJpLkAdjXGTRda052sN4sO77bKEzYYqWKMBjQvrJFzqygI99gL6Z4u2xctPW1tB8ErvA=="], + "oauth4webapi": ["oauth4webapi@3.8.5", "", {}, "sha512-A8jmyUckVhRJj5lspguklcl90Ydqk61H3dcU0oLhH3Yv13KpAliKTt5hknpGGPZSSfOwGyraNEFmofDYH+1kSg=="], + "object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="], "object-hash": ["object-hash@3.0.0", "", {}, "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw=="], @@ -2432,6 +2496,8 @@ "openapi-typescript-helpers": ["openapi-typescript-helpers@0.0.15", "", {}, "sha512-opyTPaunsklCBpTK8JGef6mfPhLSnyy5a0IN9vKtx3+4aExf+KxEqYwIy3hqkedXIB97u357uLMJsOnm3GVjsw=="], + "openid-client": ["openid-client@6.8.2", "", { "dependencies": { "jose": "^6.1.3", "oauth4webapi": "^3.8.4" } }, "sha512-uOvTCndr4udZsKihJ68H9bUICrriHdUVJ6Az+4Ns6cW55rwM5h0bjVIzDz2SxgOI84LKjFyjOFvERLzdTUROGA=="], + "optionator": ["optionator@0.9.4", "", { "dependencies": { "deep-is": "^0.1.3", "fast-levenshtein": "^2.0.6", "levn": "^0.4.1", "prelude-ls": "^1.2.1", "type-check": "^0.4.0", "word-wrap": "^1.2.5" } }, "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g=="], "own-keys": ["own-keys@1.0.1", "", { "dependencies": { "get-intrinsic": "^1.2.6", "object-keys": "^1.1.1", "safe-push-apply": "^1.0.0" } }, "sha512-qFOyK5PjiWZd+QQIh+1jhdb9LpxTF0qs7Pm8o5QHYZ0M3vKqSqzsZaEB6oWlxZ+q2sJBMI/Ktgd2N5ZwQoRHfg=="], @@ -2576,6 +2642,8 @@ "proxy-from-env": ["proxy-from-env@1.1.0", "", {}, "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="], + "pump": ["pump@3.0.3", "", { "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" } }, "sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA=="], + "punycode": ["punycode@2.3.1", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="], "punycode.js": ["punycode.js@2.3.1", "", {}, "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA=="], @@ -2690,8 +2758,14 @@ "restore-cursor": ["restore-cursor@5.1.0", "", { "dependencies": { "onetime": "^7.0.0", "signal-exit": "^4.1.0" } }, "sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA=="], + "retry": ["retry@0.13.1", "", {}, "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg=="], + + "retry-request": ["retry-request@7.0.2", "", { "dependencies": { "@types/request": "^2.48.8", "extend": "^3.0.2", "teeny-request": "^9.0.0" } }, "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w=="], + "reusify": ["reusify@1.1.0", "", {}, "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw=="], + "rfc4648": ["rfc4648@1.5.4", "", {}, "sha512-rRg/6Lb+IGfJqO05HZkN50UtY7K/JhxJag1kP23+zyMfrvoB0B7RWv06MbOzoc79RgCdNTiUaNsTT1AJZ7Z+cg=="], + "rfdc": ["rfdc@1.4.1", "", {}, "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA=="], "rimraf": ["rimraf@5.0.10", "", { "dependencies": { "glob": "^10.3.7" }, "bin": { "rimraf": "dist/esm/bin.mjs" } }, "sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ=="], @@ -2806,12 +2880,20 @@ "stop-iteration-iterator": ["stop-iteration-iterator@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "internal-slot": "^1.1.0" } }, "sha512-eLoXW/DHyl62zxY4SCaIgnRhuMr6ri4juEYARS8E6sCEqzKpOiE521Ucofdx+KnDZl5xmvGYaaKCk5FEOxJCoQ=="], + "stream-buffers": ["stream-buffers@3.0.3", "", {}, "sha512-pqMqwQCso0PBJt2PQmDO0cFj0lyqmiwOMiMSkVtRokl7e+ZTRYgDHKnuZNbqjiJXgsg4nuqtD/zxuo9KqTp0Yw=="], + "stream-chain": ["stream-chain@2.2.5", "", {}, "sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA=="], + "stream-events": ["stream-events@1.0.5", "", { "dependencies": { "stubs": "^3.0.0" } }, "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg=="], + "stream-json": ["stream-json@1.9.1", "", { "dependencies": { "stream-chain": "^2.2.5" } }, "sha512-uWkjJ+2Nt/LO9Z/JyKZbMusL8Dkh97uUBTv3AJQ74y07lVahLY4eEFsPsE97pxYBwr8nnjMAIch5eqI0gPShyw=="], + "stream-shift": ["stream-shift@1.0.3", "", {}, "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ=="], + "streamsearch": ["streamsearch@1.1.0", "", {}, "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg=="], + "streamx": ["streamx@2.23.0", "", { "dependencies": { "events-universal": "^1.0.0", "fast-fifo": "^1.3.2", "text-decoder": "^1.1.0" } }, "sha512-kn+e44esVfn2Fa/O0CPFcex27fjIL6MkVae0Mm6q+E6f0hWv578YCERbv+4m02cjxvDsPKLnmxral/rR6lBMAg=="], + "strict-uri-encode": ["strict-uri-encode@2.0.0", "", {}, "sha512-QwiXZgpRcKkhTj2Scnn++4PKtWsH0kpzZ62L2R6c/LUVYv7hVnZqcg2+sMuT6R7Jusu1vviK/MFsu6kNJfWlEQ=="], "string-argv": ["string-argv@0.3.2", "", {}, "sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q=="], @@ -2846,6 +2928,8 @@ "strtok3": ["strtok3@10.3.4", "", { "dependencies": { "@tokenizer/token": "^0.3.0" } }, "sha512-KIy5nylvC5le1OdaaoCJ07L+8iQzJHGH6pWDuzS+d07Cu7n1MZ2x26P8ZKIWfbK02+XIL8Mp4RkWeqdUCrDMfg=="], + "stubs": ["stubs@3.0.0", "", {}, "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw=="], + "style-to-js": ["style-to-js@1.1.21", "", { "dependencies": { "style-to-object": "1.0.14" } }, "sha512-RjQetxJrrUJLQPHbLku6U/ocGtzyjbJMP9lCNK7Ag0CNh690nSH8woqWH9u16nMjYBAok+i7JO1NP2pOy8IsPQ=="], "style-to-object": ["style-to-object@1.0.14", "", { "dependencies": { "inline-style-parser": "0.2.7" } }, "sha512-LIN7rULI0jBscWQYaSswptyderlarFkjQ+t79nzty8tcIAceVomEVlLzH5VP4Cmsv6MtKhs7qaAiwlcp+Mgaxw=="], @@ -2880,10 +2964,18 @@ "tapable": ["tapable@2.3.0", "", {}, "sha512-g9ljZiwki/LfxmQADO3dEY1CbpmXT5Hm2fJ+QaGKwSXUylMybePR7/67YW7jOrrvjEgL1Fmz5kzyAjWVWLlucg=="], + "tar-fs": ["tar-fs@3.1.1", "", { "dependencies": { "pump": "^3.0.0", "tar-stream": "^3.1.5" }, "optionalDependencies": { "bare-fs": "^4.0.1", "bare-path": "^3.0.0" } }, "sha512-LZA0oaPOc2fVo82Txf3gw+AkEd38szODlptMYejQUhndHMLQ9M059uXR+AfS7DNo0NpINvSqDsvyaCrBVkptWg=="], + + "tar-stream": ["tar-stream@3.1.7", "", { "dependencies": { "b4a": "^1.6.4", "fast-fifo": "^1.2.0", "streamx": "^2.15.0" } }, "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ=="], + + "teeny-request": ["teeny-request@9.0.0", "", { "dependencies": { "http-proxy-agent": "^5.0.0", "https-proxy-agent": "^5.0.0", "node-fetch": "^2.6.9", "stream-events": "^1.0.5", "uuid": "^9.0.0" } }, "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g=="], + "terser": ["terser@5.46.0", "", { "dependencies": { "@jridgewell/source-map": "^0.3.3", "acorn": "^8.15.0", "commander": "^2.20.0", "source-map-support": "~0.5.20" }, "bin": { "terser": "bin/terser" } }, "sha512-jTwoImyr/QbOWFFso3YoU3ik0jBBDJ6JTOQiy/J2YxVJdZCc+5u7skhNwiOR3FQIygFqVUPHl7qbbxtjW2K3Qg=="], "terser-webpack-plugin": ["terser-webpack-plugin@5.3.16", "", { "dependencies": { "@jridgewell/trace-mapping": "^0.3.25", "jest-worker": "^27.4.5", "schema-utils": "^4.3.0", "serialize-javascript": "^6.0.2", "terser": "^5.31.1" }, "peerDependencies": { "webpack": "^5.1.0" } }, "sha512-h9oBFCWrq78NyWWVcSwZarJkZ01c2AyGrzs1crmHZO3QUg9D61Wu4NPjBy69n7JqylFF5y+CsUZYmYEIZ3mR+Q=="], + "text-decoder": ["text-decoder@1.2.7", "", { "dependencies": { "b4a": "^1.6.4" } }, "sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ=="], + "thenify": ["thenify@3.3.1", "", { "dependencies": { "any-promise": "^1.0.0" } }, "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw=="], "thenify-all": ["thenify-all@1.6.0", "", { "dependencies": { "thenify": ">= 3.1.0 < 4" } }, "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA=="], @@ -3150,6 +3242,12 @@ "@eslint/eslintrc/ignore": ["ignore@5.3.2", "", {}, "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g=="], + "@google-cloud/storage/fast-xml-parser": ["fast-xml-parser@5.3.6", "", { "dependencies": { "strnum": "^2.1.2" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-QNI3sAvSvaOiaMl8FYU4trnEzCwiRr8XMWgAHzlrWpTSj+QaCSvOf1h82OEP1s4hiAXhnbXSyFWCf4ldZzZRVA=="], + + "@google-cloud/storage/google-auth-library": ["google-auth-library@9.15.1", "", { "dependencies": { "base64-js": "^1.3.0", "ecdsa-sig-formatter": "^1.0.11", "gaxios": "^6.1.1", "gcp-metadata": "^6.1.0", "gtoken": "^7.0.0", "jws": "^4.0.0" } }, "sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng=="], + + "@google-cloud/storage/uuid": ["uuid@8.3.2", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg=="], + "@isaacs/cliui/string-width": ["string-width@5.1.2", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" } }, "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA=="], "@isaacs/cliui/wrap-ansi": ["wrap-ansi@8.1.0", "", { "dependencies": { "ansi-styles": "^6.1.0", "string-width": "^5.0.1", "strip-ansi": "^7.0.1" } }, "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ=="], @@ -3278,16 +3376,24 @@ "@types/express-serve-static-core/@types/node": ["@types/node@25.2.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w=="], + "@types/node-fetch/@types/node": ["@types/node@25.2.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w=="], + "@types/node-forge/@types/node": ["@types/node@25.2.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w=="], "@types/pg/@types/node": ["@types/node@25.2.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w=="], "@types/readable-stream/@types/node": ["@types/node@25.2.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w=="], + "@types/request/@types/node": ["@types/node@25.2.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w=="], + + "@types/request/form-data": ["form-data@2.5.5", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.2", "mime-types": "^2.1.35", "safe-buffer": "^5.2.1" } }, "sha512-jqdObeR2rxZZbPSGL+3VckHMYtu+f9//KXBsVny6JSX/pa38Fy+bGjuG8eW/H6USNQWhLi8Num++cU2yOCNz4A=="], + "@types/send/@types/node": ["@types/node@25.2.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w=="], "@types/serve-static/@types/node": ["@types/node@25.2.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w=="], + "@types/stream-buffers/@types/node": ["@types/node@25.2.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w=="], + "@types/superagent/@types/node": ["@types/node@25.2.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w=="], "@types/ws/@types/node": ["@types/node@25.2.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w=="], @@ -3336,6 +3442,8 @@ "drizzle-kit/esbuild": ["esbuild@0.25.12", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.12", "@esbuild/android-arm": "0.25.12", "@esbuild/android-arm64": "0.25.12", "@esbuild/android-x64": "0.25.12", "@esbuild/darwin-arm64": "0.25.12", "@esbuild/darwin-x64": "0.25.12", "@esbuild/freebsd-arm64": "0.25.12", "@esbuild/freebsd-x64": "0.25.12", "@esbuild/linux-arm": "0.25.12", "@esbuild/linux-arm64": "0.25.12", "@esbuild/linux-ia32": "0.25.12", "@esbuild/linux-loong64": "0.25.12", "@esbuild/linux-mips64el": "0.25.12", "@esbuild/linux-ppc64": "0.25.12", "@esbuild/linux-riscv64": "0.25.12", "@esbuild/linux-s390x": "0.25.12", "@esbuild/linux-x64": "0.25.12", "@esbuild/netbsd-arm64": "0.25.12", "@esbuild/netbsd-x64": "0.25.12", "@esbuild/openbsd-arm64": "0.25.12", "@esbuild/openbsd-x64": "0.25.12", "@esbuild/openharmony-arm64": "0.25.12", "@esbuild/sunos-x64": "0.25.12", "@esbuild/win32-arm64": "0.25.12", "@esbuild/win32-ia32": "0.25.12", "@esbuild/win32-x64": "0.25.12" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg=="], + "duplexify/readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="], + "escodegen/source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="], "eslint/ajv": ["ajv@6.12.6", "", { "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" } }, "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g=="], @@ -3356,14 +3464,22 @@ "form-data/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], - "gaxios/node-fetch": ["node-fetch@3.3.2", "", { "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", "formdata-polyfill": "^4.0.10" } }, "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA=="], + "gaxios/uuid": ["uuid@9.0.1", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA=="], + + "gcp-metadata/gaxios": ["gaxios@7.1.3", "", { "dependencies": { "extend": "^3.0.2", "https-proxy-agent": "^7.0.1", "node-fetch": "^3.3.2", "rimraf": "^5.0.1" } }, "sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ=="], "get-uri/data-uri-to-buffer": ["data-uri-to-buffer@6.0.2", "", {}, "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw=="], "glob/minimatch": ["minimatch@9.0.5", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow=="], + "google-auth-library/gaxios": ["gaxios@7.1.3", "", { "dependencies": { "extend": "^3.0.2", "https-proxy-agent": "^7.0.1", "node-fetch": "^3.3.2", "rimraf": "^5.0.1" } }, "sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ=="], + + "googleapis-common/gaxios": ["gaxios@7.1.3", "", { "dependencies": { "extend": "^3.0.2", "https-proxy-agent": "^7.0.1", "node-fetch": "^3.3.2", "rimraf": "^5.0.1" } }, "sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ=="], + "gradient-string/chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], + "gtoken/gaxios": ["gaxios@7.1.3", "", { "dependencies": { "extend": "^3.0.2", "https-proxy-agent": "^7.0.1", "node-fetch": "^3.3.2", "rimraf": "^5.0.1" } }, "sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ=="], + "hast-util-from-html/parse5": ["parse5@7.3.0", "", { "dependencies": { "entities": "^6.0.0" } }, "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw=="], "hast-util-from-parse5/hastscript": ["hastscript@9.0.1", "", { "dependencies": { "@types/hast": "^3.0.0", "comma-separated-tokens": "^2.0.0", "hast-util-parse-selector": "^4.0.0", "property-information": "^7.0.0", "space-separated-tokens": "^2.0.0" } }, "sha512-g7df9rMFX/SPi34tyGCyUBREQoKkapwdY/T04Qn9TDWfHhAYt4/I0gMVirzK5wEzeUqIjEB+LXC/ypb7Aqno5w=="], @@ -3396,6 +3512,8 @@ "micromatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], + "minimist-options/arrify": ["arrify@1.0.1", "", {}, "sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA=="], + "minio/lodash": ["lodash@4.17.23", "", {}, "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w=="], "minio/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], @@ -3482,12 +3600,20 @@ "sucrase/commander": ["commander@4.1.1", "", {}, "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA=="], + "superagent/mime": ["mime@2.6.0", "", { "bin": { "mime": "cli.js" } }, "sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg=="], + "supertest/cookie-signature": ["cookie-signature@1.2.2", "", {}, "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="], "tailwindcss/postcss-selector-parser": ["postcss-selector-parser@6.1.2", "", { "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" } }, "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg=="], "tailwindcss/resolve": ["resolve@1.22.11", "", { "dependencies": { "is-core-module": "^2.16.1", "path-parse": "^1.0.7", "supports-preserve-symlinks-flag": "^1.0.0" }, "bin": { "resolve": "bin/resolve" } }, "sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ=="], + "teeny-request/http-proxy-agent": ["http-proxy-agent@5.0.0", "", { "dependencies": { "@tootallnate/once": "2", "agent-base": "6", "debug": "4" } }, "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w=="], + + "teeny-request/https-proxy-agent": ["https-proxy-agent@5.0.1", "", { "dependencies": { "agent-base": "6", "debug": "4" } }, "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA=="], + + "teeny-request/uuid": ["uuid@9.0.1", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA=="], + "terser/commander": ["commander@2.20.3", "", {}, "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ=="], "through2/readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="], @@ -3574,6 +3700,12 @@ "@eslint/eslintrc/ajv/json-schema-traverse": ["json-schema-traverse@0.4.1", "", {}, "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="], + "@google-cloud/storage/fast-xml-parser/strnum": ["strnum@2.1.2", "", {}, "sha512-l63NF9y/cLROq/yqKXSLtcMeeyOfnSQlfMSlzFt/K73oIaD8DGaQWd7Z34X9GPiKqP5rbSh84Hl4bOlLcjiSrQ=="], + + "@google-cloud/storage/google-auth-library/gcp-metadata": ["gcp-metadata@6.1.1", "", { "dependencies": { "gaxios": "^6.1.1", "google-logging-utils": "^0.0.2", "json-bigint": "^1.0.0" } }, "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A=="], + + "@google-cloud/storage/google-auth-library/gtoken": ["gtoken@7.1.0", "", { "dependencies": { "gaxios": "^6.0.0", "jws": "^4.0.0" } }, "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw=="], + "@isaacs/cliui/string-width/emoji-regex": ["emoji-regex@9.2.2", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="], "@isaacs/cliui/wrap-ansi/ansi-styles": ["ansi-styles@6.2.3", "", {}, "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg=="], @@ -3636,6 +3768,8 @@ "@types/express/@types/express-serve-static-core/@types/node": ["@types/node@25.2.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w=="], + "@types/request/form-data/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], + "@typescript-eslint/typescript-estree/minimatch/brace-expansion": ["brace-expansion@2.0.2", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ=="], "body-parser/debug/ms": ["ms@2.0.0", "", {}, "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="], @@ -3710,10 +3844,18 @@ "form-data/mime-types/mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="], + "gcp-metadata/gaxios/node-fetch": ["node-fetch@3.3.2", "", { "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", "formdata-polyfill": "^4.0.10" } }, "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA=="], + "glob/minimatch/brace-expansion": ["brace-expansion@2.0.2", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ=="], + "google-auth-library/gaxios/node-fetch": ["node-fetch@3.3.2", "", { "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", "formdata-polyfill": "^4.0.10" } }, "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA=="], + + "googleapis-common/gaxios/node-fetch": ["node-fetch@3.3.2", "", { "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", "formdata-polyfill": "^4.0.10" } }, "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA=="], + "gradient-string/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + "gtoken/gaxios/node-fetch": ["node-fetch@3.3.2", "", { "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", "formdata-polyfill": "^4.0.10" } }, "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA=="], + "hast-util-from-html/parse5/entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="], "hast-util-from-parse5/hastscript/hast-util-parse-selector": ["hast-util-parse-selector@4.0.0", "", { "dependencies": { "@types/hast": "^3.0.0" } }, "sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A=="], @@ -3750,6 +3892,10 @@ "refractor/@types/hast/@types/unist": ["@types/unist@2.0.11", "", {}, "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA=="], + "teeny-request/http-proxy-agent/agent-base": ["agent-base@6.0.2", "", { "dependencies": { "debug": "4" } }, "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ=="], + + "teeny-request/https-proxy-agent/agent-base": ["agent-base@6.0.2", "", { "dependencies": { "debug": "4" } }, "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ=="], + "ts-loader/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], "vizion/async/lodash": ["lodash@4.17.23", "", {}, "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w=="], @@ -3774,6 +3920,8 @@ "@aws-crypto/util/@smithy/util-utf8/@smithy/util-buffer-from/@smithy/is-array-buffer": ["@smithy/is-array-buffer@2.2.0", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA=="], + "@google-cloud/storage/google-auth-library/gcp-metadata/google-logging-utils": ["google-logging-utils@0.0.2", "", {}, "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ=="], + "@nestjs/platform-express/express/accepts/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], "@nestjs/platform-express/express/accepts/negotiator": ["negotiator@0.6.3", "", {}, "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg=="], @@ -3786,6 +3934,8 @@ "@nestjs/platform-express/express/type-is/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], + "@types/request/form-data/mime-types/mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="], + "body-parser/type-is/mime-types/mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="], "markdown-it-html5-embed/markdown-it/argparse/sprintf-js": ["sprintf-js@1.0.3", "", {}, "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g=="], diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 000000000..4e06e95c2 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,31 @@ +# Kubernetes Deployment (Local First) + +This folder contains the first draft Helm charts to run ShipSec Studio on Kubernetes. + +Primary target for this draft: + +- Local Kubernetes on OrbStack +- DinD enabled (temporary) so docker-based components can run via `DOCKER_HOST` + +## Quick Start (OrbStack) + +1. Ensure OrbStack Kubernetes is running. +2. Run: + +```bash +./deploy/scripts/orbstack/install.sh +./deploy/scripts/orbstack/smoke.sh +``` + +## Access (Local Defaults) + +- Backend: `http://localhost:3211/health` +- Frontend: `http://localhost:8090` +- Temporal UI: `http://localhost:8081` +- MinIO Console: `http://localhost:9001` + +## Notes + +- This draft uses `temporalio/auto-setup` for local/dev parity with `docker/docker-compose.full.yml`. Do not treat this as a production Temporal deployment. +- DinD is enabled only to match the current execution model. It is not a production security model. + diff --git a/deploy/helm/shipsec-infra/Chart.yaml b/deploy/helm/shipsec-infra/Chart.yaml new file mode 100644 index 000000000..2a36505ac --- /dev/null +++ b/deploy/helm/shipsec-infra/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: shipsec-infra +description: Local/dev infrastructure dependencies for ShipSec Studio +type: application +version: 0.1.0 +appVersion: "0.1.0" + diff --git a/deploy/helm/shipsec-infra/templates/_helpers.tpl b/deploy/helm/shipsec-infra/templates/_helpers.tpl new file mode 100644 index 000000000..e0a8c0ff8 --- /dev/null +++ b/deploy/helm/shipsec-infra/templates/_helpers.tpl @@ -0,0 +1,8 @@ +{{- define "shipsec-infra.labels" -}} +app.kubernetes.io/name: shipsec-infra +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }} +{{- end -}} + diff --git a/deploy/helm/shipsec-infra/templates/loki.yaml b/deploy/helm/shipsec-infra/templates/loki.yaml new file mode 100644 index 000000000..51913ceac --- /dev/null +++ b/deploy/helm/shipsec-infra/templates/loki.yaml @@ -0,0 +1,56 @@ +{{- if .Values.loki.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: shipsec-loki + namespace: {{ .Values.global.namespace }} + labels: + {{- include "shipsec-infra.labels" . | nindent 4 }} + app.kubernetes.io/component: loki +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: loki + template: + metadata: + labels: + app.kubernetes.io/component: loki + spec: + containers: + - name: loki + image: {{ .Values.loki.image | quote }} + args: ["-config.file=/etc/loki/local-config.yaml"] + ports: + - name: http + containerPort: 3100 + readinessProbe: + httpGet: + path: /ready + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /ready + port: http + initialDelaySeconds: 30 + periodSeconds: 20 +--- +apiVersion: v1 +kind: Service +metadata: + name: shipsec-loki + namespace: {{ .Values.global.namespace }} + labels: + {{- include "shipsec-infra.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: http + port: 3100 + targetPort: http + selector: + app.kubernetes.io/component: loki +{{- end }} + diff --git a/deploy/helm/shipsec-infra/templates/minio.yaml b/deploy/helm/shipsec-infra/templates/minio.yaml new file mode 100644 index 000000000..65eb8d751 --- /dev/null +++ b/deploy/helm/shipsec-infra/templates/minio.yaml @@ -0,0 +1,95 @@ +{{- if .Values.minio.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: shipsec-minio-secret + namespace: {{ .Values.global.namespace }} + labels: + {{- include "shipsec-infra.labels" . | nindent 4 }} +type: Opaque +stringData: + MINIO_ROOT_USER: {{ .Values.minio.rootUser | quote }} + MINIO_ROOT_PASSWORD: {{ .Values.minio.rootPassword | quote }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: shipsec-minio + namespace: {{ .Values.global.namespace }} + labels: + {{- include "shipsec-infra.labels" . | nindent 4 }} + app.kubernetes.io/component: minio +spec: + serviceName: shipsec-minio + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: minio + template: + metadata: + labels: + app.kubernetes.io/component: minio + spec: + containers: + - name: minio + image: {{ .Values.minio.image | quote }} + args: ["server", "/data", "--console-address", ":9001"] + envFrom: + - secretRef: + name: shipsec-minio-secret + ports: + - name: api + containerPort: 9000 + - name: console + containerPort: 9001 + readinessProbe: + httpGet: + path: /minio/health/ready + port: api + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /minio/health/live + port: api + initialDelaySeconds: 15 + periodSeconds: 20 + volumeMounts: + - name: data + mountPath: /data + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.minio.persistence.size }} +--- +apiVersion: v1 +kind: Service +metadata: + name: shipsec-minio + namespace: {{ .Values.global.namespace }} + labels: + {{- include "shipsec-infra.labels" . | nindent 4 }} +spec: + {{- $svcType := "ClusterIP" -}} + {{- $apiPort := 9000 -}} + {{- $consolePort := 9001 -}} + {{- with .Values.minio.service -}} + {{- if .type }}{{- $svcType = .type }}{{- end -}} + {{- if .apiPort }}{{- $apiPort = .apiPort }}{{- end -}} + {{- if .consolePort }}{{- $consolePort = .consolePort }}{{- end -}} + {{- end }} + type: {{ $svcType }} + ports: + - name: api + port: {{ $apiPort }} + targetPort: api + - name: console + port: {{ $consolePort }} + targetPort: console + selector: + app.kubernetes.io/component: minio +{{- end }} diff --git a/deploy/helm/shipsec-infra/templates/postgres.yaml b/deploy/helm/shipsec-infra/templates/postgres.yaml new file mode 100644 index 000000000..dac8c19af --- /dev/null +++ b/deploy/helm/shipsec-infra/templates/postgres.yaml @@ -0,0 +1,109 @@ +{{- if .Values.postgres.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: shipsec-postgres-secret + namespace: {{ .Values.global.namespace }} + labels: + {{- include "shipsec-infra.labels" . | nindent 4 }} +type: Opaque +stringData: + POSTGRES_USER: {{ .Values.postgres.user | quote }} + POSTGRES_PASSWORD: {{ .Values.postgres.password | quote }} + POSTGRES_DB: {{ .Values.postgres.database | quote }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: shipsec-postgres-initdb + namespace: {{ .Values.global.namespace }} + labels: + {{- include "shipsec-infra.labels" . | nindent 4 }} +data: + create-temporal-db.sh: | + #!/bin/bash + set -e + psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + CREATE DATABASE temporal; + GRANT ALL PRIVILEGES ON DATABASE temporal TO $POSTGRES_USER; + EOSQL +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: shipsec-postgres + namespace: {{ .Values.global.namespace }} + labels: + {{- include "shipsec-infra.labels" . | nindent 4 }} + app.kubernetes.io/component: postgres +spec: + serviceName: shipsec-postgres + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: postgres + template: + metadata: + labels: + app.kubernetes.io/component: postgres + spec: + containers: + - name: postgres + image: {{ .Values.postgres.image | quote }} + ports: + - name: postgres + containerPort: 5432 + env: + # GKE (and some other environments) mount ext4 volumes with a `lost+found` + # directory at the volume root. Postgres initdb fails if the data dir is + # not empty, so we point PGDATA at a subdirectory. + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + envFrom: + - secretRef: + name: shipsec-postgres-secret + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + - name: initdb + mountPath: /docker-entrypoint-initdb.d + readinessProbe: + exec: + command: ["sh", "-c", "pg_isready -U $POSTGRES_USER"] + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + exec: + command: ["sh", "-c", "pg_isready -U $POSTGRES_USER"] + initialDelaySeconds: 15 + periodSeconds: 10 + volumes: + - name: initdb + configMap: + name: shipsec-postgres-initdb + defaultMode: 0755 + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.postgres.persistence.size }} +--- +apiVersion: v1 +kind: Service +metadata: + name: shipsec-postgres + namespace: {{ .Values.global.namespace }} + labels: + {{- include "shipsec-infra.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: postgres + port: 5432 + targetPort: postgres + selector: + app.kubernetes.io/component: postgres +{{- end }} diff --git a/deploy/helm/shipsec-infra/templates/redis.yaml b/deploy/helm/shipsec-infra/templates/redis.yaml new file mode 100644 index 000000000..148e490e6 --- /dev/null +++ b/deploy/helm/shipsec-infra/templates/redis.yaml @@ -0,0 +1,53 @@ +{{- if .Values.redis.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: shipsec-redis + namespace: {{ .Values.global.namespace }} + labels: + {{- include "shipsec-infra.labels" . | nindent 4 }} + app.kubernetes.io/component: redis +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: redis + template: + metadata: + labels: + app.kubernetes.io/component: redis + spec: + containers: + - name: redis + image: {{ .Values.redis.image | quote }} + ports: + - name: redis + containerPort: 6379 + readinessProbe: + exec: + command: ["redis-cli", "ping"] + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + exec: + command: ["redis-cli", "ping"] + initialDelaySeconds: 15 + periodSeconds: 10 +--- +apiVersion: v1 +kind: Service +metadata: + name: shipsec-redis + namespace: {{ .Values.global.namespace }} + labels: + {{- include "shipsec-infra.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: redis + port: 6379 + targetPort: redis + selector: + app.kubernetes.io/component: redis +{{- end }} + diff --git a/deploy/helm/shipsec-infra/templates/redpanda.yaml b/deploy/helm/shipsec-infra/templates/redpanda.yaml new file mode 100644 index 000000000..0617ce291 --- /dev/null +++ b/deploy/helm/shipsec-infra/templates/redpanda.yaml @@ -0,0 +1,90 @@ +{{- if .Values.redpanda.enabled }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: shipsec-redpanda + namespace: {{ .Values.global.namespace }} + labels: + {{- include "shipsec-infra.labels" . | nindent 4 }} + app.kubernetes.io/component: redpanda +spec: + serviceName: shipsec-redpanda + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: redpanda + template: + metadata: + labels: + app.kubernetes.io/component: redpanda + spec: + initContainers: + - name: volume-permissions + image: busybox:1.36 + command: ["sh", "-c", "chown -R 101:101 /var/lib/redpanda/data"] + securityContext: + runAsUser: 0 + volumeMounts: + - name: data + mountPath: /var/lib/redpanda/data + containers: + - name: redpanda + image: {{ .Values.redpanda.image | quote }} + args: + - redpanda + - start + - --mode=dev-container + - --smp=1 + - --reserve-memory=0M + - --overprovisioned + - --node-id=0 + - --check=false + - --advertise-kafka-addr=shipsec-redpanda.{{ .Values.global.namespace }}.svc.cluster.local:9092 + ports: + - name: kafka + containerPort: 9092 + - name: admin + containerPort: 9644 + readinessProbe: + httpGet: + path: /v1/status/ready + port: admin + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /v1/status/ready + port: admin + initialDelaySeconds: 30 + periodSeconds: 20 + volumeMounts: + - name: data + mountPath: /var/lib/redpanda/data + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.redpanda.persistence.size }} +--- +apiVersion: v1 +kind: Service +metadata: + name: shipsec-redpanda + namespace: {{ .Values.global.namespace }} + labels: + {{- include "shipsec-infra.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: kafka + port: 9092 + targetPort: kafka + - name: admin + port: 9644 + targetPort: admin + selector: + app.kubernetes.io/component: redpanda +{{- end }} diff --git a/deploy/helm/shipsec-infra/templates/temporal.yaml b/deploy/helm/shipsec-infra/templates/temporal.yaml new file mode 100644 index 000000000..b9085ab9a --- /dev/null +++ b/deploy/helm/shipsec-infra/templates/temporal.yaml @@ -0,0 +1,131 @@ +{{- if .Values.temporal.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: shipsec-temporal + namespace: {{ .Values.global.namespace }} + labels: + {{- include "shipsec-infra.labels" . | nindent 4 }} + app.kubernetes.io/component: temporal +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: temporal + template: + metadata: + labels: + app.kubernetes.io/component: temporal + spec: + containers: + - name: temporal + image: {{ .Values.temporal.image | quote }} + ports: + - name: grpc + containerPort: 7233 + env: + - name: DB + value: postgres12 + - name: DB_PORT + value: "5432" + - name: DB_NAME + value: temporal + - name: POSTGRES_USER + value: {{ .Values.temporal.postgresUser | default .Values.postgres.user | quote }} + - name: POSTGRES_PWD + value: {{ .Values.temporal.postgresPassword | default .Values.postgres.password | quote }} + - name: POSTGRES_SEEDS + value: {{ .Values.temporal.postgresHost | default "shipsec-postgres" | quote }} + - name: AUTO_SETUP + value: "true" + readinessProbe: + tcpSocket: + port: grpc + initialDelaySeconds: 20 + periodSeconds: 10 + livenessProbe: + tcpSocket: + port: grpc + initialDelaySeconds: 60 + periodSeconds: 20 +--- +apiVersion: v1 +kind: Service +metadata: + name: shipsec-temporal + namespace: {{ .Values.global.namespace }} + labels: + {{- include "shipsec-infra.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: grpc + port: 7233 + targetPort: grpc + selector: + app.kubernetes.io/component: temporal +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: shipsec-temporal-ui + namespace: {{ .Values.global.namespace }} + labels: + {{- include "shipsec-infra.labels" . | nindent 4 }} + app.kubernetes.io/component: temporal-ui +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: temporal-ui + template: + metadata: + labels: + app.kubernetes.io/component: temporal-ui + spec: + containers: + - name: temporal-ui + image: {{ .Values.temporal.uiImage | quote }} + ports: + - name: http + containerPort: 8080 + env: + - name: TEMPORAL_ADDRESS + value: shipsec-temporal:7233 + - name: TEMPORAL_NAMESPACE + value: default + readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 30 + periodSeconds: 20 +--- +apiVersion: v1 +kind: Service +metadata: + name: shipsec-temporal-ui + namespace: {{ .Values.global.namespace }} + labels: + {{- include "shipsec-infra.labels" . | nindent 4 }} +spec: + {{- $uiSvcType := "ClusterIP" -}} + {{- $uiSvcPort := 8080 -}} + {{- with .Values.temporal.uiService -}} + {{- if .type }}{{- $uiSvcType = .type }}{{- end -}} + {{- if .port }}{{- $uiSvcPort = .port }}{{- end -}} + {{- end }} + type: {{ $uiSvcType }} + ports: + - name: http + port: {{ $uiSvcPort }} + targetPort: http + selector: + app.kubernetes.io/component: temporal-ui +{{- end }} diff --git a/deploy/helm/shipsec-infra/values.yaml b/deploy/helm/shipsec-infra/values.yaml new file mode 100644 index 000000000..e4ff74299 --- /dev/null +++ b/deploy/helm/shipsec-infra/values.yaml @@ -0,0 +1,47 @@ +global: + namespace: shipsec-system + +postgres: + enabled: true + image: postgres:16-alpine + user: shipsec + password: shipsec + database: shipsec + persistence: + enabled: true + size: 5Gi + +redis: + enabled: true + image: redis:7-alpine + persistence: + enabled: false + size: 1Gi + +minio: + enabled: true + image: minio/minio:RELEASE.2024-10-02T17-50-41Z + rootUser: minioadmin + rootPassword: minioadmin + persistence: + enabled: true + size: 10Gi + +temporal: + enabled: true + image: temporalio/auto-setup:latest + uiImage: temporalio/ui:latest + +redpanda: + enabled: true + image: redpandadata/redpanda:v24.2.5 + persistence: + enabled: true + size: 5Gi + +loki: + enabled: false + image: grafana/loki:3.2.1 + persistence: + enabled: false + size: 5Gi diff --git a/deploy/helm/shipsec-infra/values/cloud-generic.yaml b/deploy/helm/shipsec-infra/values/cloud-generic.yaml new file mode 100644 index 000000000..d1c4fd630 --- /dev/null +++ b/deploy/helm/shipsec-infra/values/cloud-generic.yaml @@ -0,0 +1,22 @@ +# Managed services — disabled (Cloud SQL, Memorystore) +postgres: + enabled: false + +redis: + enabled: false + +# Still runs in-cluster +minio: + enabled: true + +temporal: + enabled: true + postgresHost: '10.25.225.3' + postgresUser: shipsec + postgresPassword: shipsec-dev-2026 + +redpanda: + enabled: true + +loki: + enabled: true diff --git a/deploy/helm/shipsec-infra/values/gke-dev.yaml b/deploy/helm/shipsec-infra/values/gke-dev.yaml new file mode 100644 index 000000000..e455bdb8f --- /dev/null +++ b/deploy/helm/shipsec-infra/values/gke-dev.yaml @@ -0,0 +1,14 @@ +global: + namespace: shipsec-system + +# Keep infra in-cluster for the first GKE pass to move fast. +# Move to managed services (Cloud SQL, Memorystore, GCS) later. + +minio: + service: + type: ClusterIP + +temporal: + uiService: + type: ClusterIP + diff --git a/deploy/helm/shipsec-infra/values/gke-managed.yaml b/deploy/helm/shipsec-infra/values/gke-managed.yaml new file mode 100644 index 000000000..a72d1ba2f --- /dev/null +++ b/deploy/helm/shipsec-infra/values/gke-managed.yaml @@ -0,0 +1,27 @@ +# GKE with managed services (Cloud SQL, Memorystore). +# Layer on top of gke-dev.yaml: +# --values values/gke-dev.yaml --values values/gke-managed.yaml + +global: + namespace: shipsec-system + +# Replaced by Cloud SQL +postgres: + enabled: false + +# Replaced by Memorystore +redis: + enabled: false + +# Keep MinIO in-cluster (org policy blocks HMAC key creation for GCS S3 compat) +minio: + service: + type: ClusterIP + +# Temporal stays in-cluster but points at Cloud SQL +temporal: + postgresHost: "10.25.225.3" + postgresUser: "shipsec" + postgresPassword: "shipsec-dev-2026" + uiService: + type: ClusterIP diff --git a/deploy/helm/shipsec-infra/values/local-orbstack.yaml b/deploy/helm/shipsec-infra/values/local-orbstack.yaml new file mode 100644 index 000000000..400d84809 --- /dev/null +++ b/deploy/helm/shipsec-infra/values/local-orbstack.yaml @@ -0,0 +1,14 @@ +global: + namespace: shipsec-system + +temporal: + uiService: + type: LoadBalancer + port: 8081 + +minio: + service: + type: LoadBalancer + apiPort: 9000 + consolePort: 9001 + diff --git a/deploy/helm/shipsec-infra/values/vps.yaml b/deploy/helm/shipsec-infra/values/vps.yaml new file mode 100644 index 000000000..9a207973e --- /dev/null +++ b/deploy/helm/shipsec-infra/values/vps.yaml @@ -0,0 +1,11 @@ +global: + namespace: shipsec-system + +minio: + service: + type: ClusterIP + +temporal: + uiService: + type: ClusterIP + diff --git a/deploy/helm/shipsec/Chart.yaml b/deploy/helm/shipsec/Chart.yaml new file mode 100644 index 000000000..a5b75d8a0 --- /dev/null +++ b/deploy/helm/shipsec/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: shipsec +description: ShipSec Studio app (backend, worker, frontend) with optional DinD +type: application +version: 0.1.0 +appVersion: "0.1.0" + diff --git a/deploy/helm/shipsec/templates/_helpers.tpl b/deploy/helm/shipsec/templates/_helpers.tpl new file mode 100644 index 000000000..d6737af9d --- /dev/null +++ b/deploy/helm/shipsec/templates/_helpers.tpl @@ -0,0 +1,16 @@ +{{- define "shipsec.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "shipsec.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" -}} +{{- end -}} + +{{- define "shipsec.labels" -}} +app.kubernetes.io/name: {{ include "shipsec.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +helm.sh/chart: {{ include "shipsec.chart" . }} +{{- end -}} + diff --git a/deploy/helm/shipsec/templates/app-secret.local.yaml b/deploy/helm/shipsec/templates/app-secret.local.yaml new file mode 100644 index 000000000..14bf44a74 --- /dev/null +++ b/deploy/helm/shipsec/templates/app-secret.local.yaml @@ -0,0 +1,33 @@ +{{- if .Values.secrets.create }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.secrets.name }} + namespace: {{ .Values.global.namespaces.system }} + labels: + {{- include "shipsec.labels" . | nindent 4 }} +type: Opaque +stringData: + DATABASE_URL: {{ .Values.secrets.databaseUrl | quote }} + MINIO_ROOT_USER: {{ .Values.secrets.minioRootUser | quote }} + MINIO_ROOT_PASSWORD: {{ .Values.secrets.minioRootPassword | quote }} + MINIO_ACCESS_KEY: {{ .Values.secrets.minioRootUser | quote }} + MINIO_SECRET_KEY: {{ .Values.secrets.minioRootPassword | quote }} + SECRET_STORE_MASTER_KEY: {{ .Values.secrets.secretStoreMasterKey | quote }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.secrets.name }} + namespace: {{ .Values.global.namespaces.workers }} + labels: + {{- include "shipsec.labels" . | nindent 4 }} +type: Opaque +stringData: + DATABASE_URL: {{ .Values.secrets.databaseUrl | quote }} + MINIO_ROOT_USER: {{ .Values.secrets.minioRootUser | quote }} + MINIO_ROOT_PASSWORD: {{ .Values.secrets.minioRootPassword | quote }} + MINIO_ACCESS_KEY: {{ .Values.secrets.minioRootUser | quote }} + MINIO_SECRET_KEY: {{ .Values.secrets.minioRootPassword | quote }} + SECRET_STORE_MASTER_KEY: {{ .Values.secrets.secretStoreMasterKey | quote }} +{{- end }} diff --git a/deploy/helm/shipsec/templates/backend-deployment.yaml b/deploy/helm/shipsec/templates/backend-deployment.yaml new file mode 100644 index 000000000..e6f3858b2 --- /dev/null +++ b/deploy/helm/shipsec/templates/backend-deployment.yaml @@ -0,0 +1,63 @@ +{{- if .Values.backend.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: shipsec-backend + namespace: {{ .Values.global.namespaces.system }} + labels: + {{- include "shipsec.labels" . | nindent 4 }} + app.kubernetes.io/component: backend +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: backend + template: + metadata: + labels: + {{- include "shipsec.labels" . | nindent 8 }} + app.kubernetes.io/component: backend + spec: + containers: + - name: backend + image: "{{ .Values.backend.image.repository }}:{{ .Values.backend.image.tag }}" + imagePullPolicy: {{ .Values.backend.image.pullPolicy }} + ports: + - name: http + containerPort: 3211 + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.name }} + key: DATABASE_URL + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.name }} + key: MINIO_ROOT_USER + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.name }} + key: MINIO_ROOT_PASSWORD + {{- range $k, $v := .Values.backend.env }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + readinessProbe: + httpGet: + path: /api/v1/health + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /api/v1/health + port: http + initialDelaySeconds: 15 + periodSeconds: 10 + resources: + {{- toYaml .Values.backend.resources | nindent 10 }} +{{- end }} diff --git a/deploy/helm/shipsec/templates/backend-service.yaml b/deploy/helm/shipsec/templates/backend-service.yaml new file mode 100644 index 000000000..9a00700a7 --- /dev/null +++ b/deploy/helm/shipsec/templates/backend-service.yaml @@ -0,0 +1,20 @@ +{{- if .Values.backend.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: shipsec-backend + namespace: {{ .Values.global.namespaces.system }} + labels: + {{- include "shipsec.labels" . | nindent 4 }} + app.kubernetes.io/component: backend +spec: + type: {{ .Values.backend.service.type }} + ports: + - name: http + port: {{ .Values.backend.service.port }} + targetPort: http + selector: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: backend +{{- end }} + diff --git a/deploy/helm/shipsec/templates/dind-deployment.yaml b/deploy/helm/shipsec/templates/dind-deployment.yaml new file mode 100644 index 000000000..bc48159b1 --- /dev/null +++ b/deploy/helm/shipsec/templates/dind-deployment.yaml @@ -0,0 +1,48 @@ +{{- if .Values.execution.dind.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: shipsec-dind + namespace: {{ .Values.global.namespaces.workloads }} + labels: + {{- include "shipsec.labels" . | nindent 4 }} + app.kubernetes.io/component: dind +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: dind + template: + metadata: + labels: + {{- include "shipsec.labels" . | nindent 8 }} + app.kubernetes.io/component: dind + spec: + containers: + - name: dind + image: docker:27-dind + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + args: + - "--host=tcp://0.0.0.0:{{ .Values.execution.dind.port }}" + - "--storage-driver=overlay2" + env: + - name: DOCKER_TLS_CERTDIR + value: "" + ports: + - name: docker + containerPort: {{ .Values.execution.dind.port }} + volumeMounts: + - name: docker-storage + mountPath: /var/lib/docker + volumes: + - name: docker-storage + {{- if .Values.execution.dind.storage.enabled }} + persistentVolumeClaim: + claimName: shipsec-dind-pvc + {{- else }} + emptyDir: {} + {{- end }} +{{- end }} diff --git a/deploy/helm/shipsec/templates/dind-pvc.yaml b/deploy/helm/shipsec/templates/dind-pvc.yaml new file mode 100644 index 000000000..32a11ef1d --- /dev/null +++ b/deploy/helm/shipsec/templates/dind-pvc.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.execution.dind.enabled .Values.execution.dind.storage.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: shipsec-dind-pvc + namespace: {{ .Values.global.namespaces.workloads }} + labels: + {{- include "shipsec.labels" . | nindent 4 }} + app.kubernetes.io/component: dind +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.execution.dind.storage.size }} +{{- end }} + diff --git a/deploy/helm/shipsec/templates/dind-service.yaml b/deploy/helm/shipsec/templates/dind-service.yaml new file mode 100644 index 000000000..349c0d245 --- /dev/null +++ b/deploy/helm/shipsec/templates/dind-service.yaml @@ -0,0 +1,20 @@ +{{- if .Values.execution.dind.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: shipsec-dind + namespace: {{ .Values.global.namespaces.workloads }} + labels: + {{- include "shipsec.labels" . | nindent 4 }} + app.kubernetes.io/component: dind +spec: + type: ClusterIP + ports: + - name: docker + port: {{ .Values.execution.dind.port }} + targetPort: docker + selector: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: dind +{{- end }} + diff --git a/deploy/helm/shipsec/templates/frontend-deployment.yaml b/deploy/helm/shipsec/templates/frontend-deployment.yaml new file mode 100644 index 000000000..88d0ce375 --- /dev/null +++ b/deploy/helm/shipsec/templates/frontend-deployment.yaml @@ -0,0 +1,44 @@ +{{- if .Values.frontend.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: shipsec-frontend + namespace: {{ .Values.global.namespaces.system }} + labels: + {{- include "shipsec.labels" . | nindent 4 }} + app.kubernetes.io/component: frontend +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: frontend + template: + metadata: + labels: + {{- include "shipsec.labels" . | nindent 8 }} + app.kubernetes.io/component: frontend + spec: + containers: + - name: frontend + image: "{{ .Values.frontend.image.repository }}:{{ .Values.frontend.image.tag }}" + imagePullPolicy: {{ .Values.frontend.image.pullPolicy }} + ports: + - name: http + containerPort: 8080 + readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 30 + periodSeconds: 20 + resources: + {{- toYaml .Values.frontend.resources | nindent 10 }} +{{- end }} + diff --git a/deploy/helm/shipsec/templates/frontend-service.yaml b/deploy/helm/shipsec/templates/frontend-service.yaml new file mode 100644 index 000000000..4cb8ba89f --- /dev/null +++ b/deploy/helm/shipsec/templates/frontend-service.yaml @@ -0,0 +1,20 @@ +{{- if .Values.frontend.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: shipsec-frontend + namespace: {{ .Values.global.namespaces.system }} + labels: + {{- include "shipsec.labels" . | nindent 4 }} + app.kubernetes.io/component: frontend +spec: + type: {{ .Values.frontend.service.type }} + ports: + - name: http + port: {{ .Values.frontend.service.port }} + targetPort: http + selector: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: frontend +{{- end }} + diff --git a/deploy/helm/shipsec/templates/ingress.yaml b/deploy/helm/shipsec/templates/ingress.yaml new file mode 100644 index 000000000..883b327cf --- /dev/null +++ b/deploy/helm/shipsec/templates/ingress.yaml @@ -0,0 +1,45 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: shipsec-ingress + namespace: {{ .Values.global.namespaces.system }} + labels: + {{- include "shipsec.labels" . | nindent 4 }} + annotations: + nginx.ingress.kubernetes.io/proxy-body-size: "50m" + nginx.ingress.kubernetes.io/proxy-read-timeout: "300" + nginx.ingress.kubernetes.io/proxy-send-timeout: "300" + {{- if .Values.ingress.websocket }} + nginx.ingress.kubernetes.io/enable-websocket: "true" + {{- end }} + {{- if .Values.ingress.tls.enabled }} + cert-manager.io/cluster-issuer: {{ .Values.ingress.tls.clusterIssuer }} + {{- end }} +spec: + ingressClassName: nginx + {{- if .Values.ingress.tls.enabled }} + tls: + - hosts: + - {{ .Values.ingress.host }} + secretName: {{ .Values.ingress.tls.secretName }} + {{- end }} + rules: + - host: {{ .Values.ingress.host }} + http: + paths: + - path: /api + pathType: Prefix + backend: + service: + name: shipsec-backend + port: + number: 3211 + - path: / + pathType: Prefix + backend: + service: + name: shipsec-frontend + port: + number: 8080 +{{- end }} diff --git a/deploy/helm/shipsec/templates/worker-deployment.yaml b/deploy/helm/shipsec/templates/worker-deployment.yaml new file mode 100644 index 000000000..c4734f9ec --- /dev/null +++ b/deploy/helm/shipsec/templates/worker-deployment.yaml @@ -0,0 +1,79 @@ +{{- if .Values.worker.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: shipsec-worker + namespace: {{ .Values.global.namespaces.workers }} + labels: + {{- include "shipsec.labels" . | nindent 4 }} + app.kubernetes.io/component: worker +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: worker + template: + metadata: + labels: + {{- include "shipsec.labels" . | nindent 8 }} + app.kubernetes.io/component: worker + spec: + {{- if eq .Values.execution.mode "k8s" }} + serviceAccountName: shipsec-worker + {{- end }} + containers: + - name: worker + image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag }}" + imagePullPolicy: {{ .Values.worker.image.pullPolicy }} + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.name }} + key: DATABASE_URL + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.name }} + key: MINIO_ACCESS_KEY + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.name }} + key: MINIO_SECRET_KEY + - name: SECRET_STORE_MASTER_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.name }} + key: SECRET_STORE_MASTER_KEY + {{- if eq .Values.execution.mode "k8s" }} + - name: EXECUTION_MODE + value: "k8s" + - name: K8S_JOB_NAMESPACE + value: {{ .Values.execution.k8s.jobNamespace | quote }} + - name: K8S_JOB_IMAGE_PULL_POLICY + value: {{ .Values.execution.k8s.imagePullPolicy | quote }} + {{- if .Values.execution.k8s.imagePullSecret }} + - name: K8S_IMAGE_PULL_SECRET + value: {{ .Values.execution.k8s.imagePullSecret | quote }} + {{- end }} + {{- if .Values.execution.k8s.gcsBucket }} + - name: GCS_VOLUME_BUCKET + value: {{ .Values.execution.k8s.gcsBucket | quote }} + {{- end }} + {{- if .Values.execution.k8s.jobServiceAccount }} + - name: K8S_JOB_SERVICE_ACCOUNT + value: {{ .Values.execution.k8s.jobServiceAccount | quote }} + {{- end }} + {{- else if .Values.execution.workerDockerHost }} + - name: DOCKER_HOST + value: {{ .Values.execution.workerDockerHost | quote }} + {{- end }} + {{- range $k, $v := .Values.worker.env }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + resources: + {{- toYaml .Values.worker.resources | nindent 10 }} +{{- end }} diff --git a/deploy/helm/shipsec/templates/worker-rbac.yaml b/deploy/helm/shipsec/templates/worker-rbac.yaml new file mode 100644 index 000000000..fbe1b305d --- /dev/null +++ b/deploy/helm/shipsec/templates/worker-rbac.yaml @@ -0,0 +1,64 @@ +{{- if and .Values.worker.enabled (eq .Values.execution.mode "k8s") }} +# ServiceAccount for the worker to create K8s Jobs +apiVersion: v1 +kind: ServiceAccount +metadata: + name: shipsec-worker + namespace: {{ .Values.global.namespaces.workers }} + labels: + {{- include "shipsec.labels" . | nindent 4 }} + app.kubernetes.io/component: worker + {{- if .Values.execution.k8s.workerGcpSa }} + annotations: + iam.gke.io/gcp-service-account: {{ .Values.execution.k8s.workerGcpSa | quote }} + {{- end }} +--- +# Role in the workloads namespace — worker creates Jobs and ConfigMaps here +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: shipsec-job-runner + namespace: {{ .Values.global.namespaces.workloads }} + labels: + {{- include "shipsec.labels" . | nindent 4 }} +rules: +- apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create", "get", "list", "watch", "delete"] +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["create", "get", "update", "delete"] +- apiGroups: [""] + resources: ["pods", "pods/log", "pods/attach"] + verbs: ["get", "list", "watch"] +--- +# Bind the worker SA (in shipsec-workers) to the Role (in shipsec-workloads) +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: shipsec-worker-job-runner + namespace: {{ .Values.global.namespaces.workloads }} + labels: + {{- include "shipsec.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: shipsec-job-runner +subjects: +- kind: ServiceAccount + name: shipsec-worker + namespace: {{ .Values.global.namespaces.workers }} +{{- if .Values.execution.k8s.jobRunnerGcpSa }} +--- +# ServiceAccount for job pods (GCS FUSE CSI access via Workload Identity) +apiVersion: v1 +kind: ServiceAccount +metadata: + name: shipsec-job-runner + namespace: {{ .Values.global.namespaces.workloads }} + labels: + {{- include "shipsec.labels" . | nindent 4 }} + annotations: + iam.gke.io/gcp-service-account: {{ .Values.execution.k8s.jobRunnerGcpSa | quote }} +{{- end }} +{{- end }} diff --git a/deploy/helm/shipsec/values.yaml b/deploy/helm/shipsec/values.yaml new file mode 100644 index 000000000..8ac338bba --- /dev/null +++ b/deploy/helm/shipsec/values.yaml @@ -0,0 +1,126 @@ +global: + namespaces: + system: shipsec-system + workers: shipsec-workers + workloads: shipsec-workloads + +secrets: + create: true + name: shipsec-app-secrets + databaseUrl: postgresql://shipsec:shipsec@shipsec-postgres.shipsec-system.svc.cluster.local:5432/shipsec + minioRootUser: minioadmin + minioRootPassword: minioadmin + +backend: + enabled: true + image: + repository: ghcr.io/shipsecai/studio-backend + tag: latest + pullPolicy: IfNotPresent + service: + type: ClusterIP + port: 3211 + env: + NODE_ENV: production + SHIPSEC_ENV: local + PORT: '3211' + ENABLE_INGEST_SERVICES: 'false' + TEMPORAL_ADDRESS: shipsec-temporal.shipsec-system.svc.cluster.local:7233 + TEMPORAL_NAMESPACE: shipsec-dev + TEMPORAL_TASK_QUEUE: shipsec-dev + MINIO_ENDPOINT: shipsec-minio.shipsec-system.svc.cluster.local + MINIO_PORT: '9000' + LOKI_URL: http://shipsec-loki.shipsec-system.svc.cluster.local:3100 + TERMINAL_REDIS_URL: redis://shipsec-redis.shipsec-system.svc.cluster.local:6379 + LOG_KAFKA_BROKERS: shipsec-redpanda.shipsec-system.svc.cluster.local:9092 + LOG_KAFKA_TOPIC: telemetry.logs + LOG_KAFKA_CLIENT_ID: shipsec-backend + EVENT_KAFKA_TOPIC: telemetry.events + EVENT_KAFKA_CLIENT_ID: shipsec-backend-events + EVENT_KAFKA_GROUP_ID: shipsec-event-ingestor + AUTH_PROVIDER: local + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 1000m + memory: 2Gi + +worker: + enabled: true + image: + repository: ghcr.io/shipsecai/studio-worker + tag: latest + pullPolicy: IfNotPresent + env: + NODE_ENV: production + SHIPSEC_ENV: local + ENABLE_INGEST_SERVICES: 'false' + TEMPORAL_ADDRESS: shipsec-temporal.shipsec-system.svc.cluster.local:7233 + TEMPORAL_NAMESPACE: shipsec-dev + TEMPORAL_TASK_QUEUE: shipsec-dev + MINIO_ENDPOINT: shipsec-minio.shipsec-system.svc.cluster.local + MINIO_PORT: '9000' + MINIO_BUCKET_NAME: shipsec-files + LOKI_URL: http://shipsec-loki.shipsec-system.svc.cluster.local:3100 + TERMINAL_REDIS_URL: redis://shipsec-redis.shipsec-system.svc.cluster.local:6379 + TERMINAL_REDIS_MAXLEN: '5000' + LOG_KAFKA_BROKERS: shipsec-redpanda.shipsec-system.svc.cluster.local:9092 + LOG_KAFKA_TOPIC: telemetry.logs + LOG_KAFKA_CLIENT_ID: shipsec-worker + EVENT_KAFKA_TOPIC: telemetry.events + EVENT_KAFKA_CLIENT_ID: shipsec-worker-events + BACKEND_URL: http://shipsec-backend.shipsec-system.svc.cluster.local:3211 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 2000m + memory: 4Gi + +frontend: + enabled: true + image: + repository: ghcr.io/shipsecai/studio-frontend + tag: latest + pullPolicy: IfNotPresent + service: + type: ClusterIP + port: 8080 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + +ingress: + enabled: false + host: studio-next.shipsec.ai + websocket: true + tls: + enabled: false + clusterIssuer: letsencrypt-prod + secretName: shipsec-tls + +execution: + # "docker" = use Docker CLI (local dev / DIND), "k8s" = K8s Jobs (GKE / production) + mode: docker + dind: + enabled: false + serviceName: shipsec-dind + namespace: shipsec-workloads + port: 2375 + storage: + enabled: true + size: 20Gi + workerDockerHost: '' + k8s: + # Namespace where component Jobs are created + jobNamespace: shipsec-workloads + imagePullPolicy: IfNotPresent + # Name of a K8s Secret with Docker registry credentials for pulling component images + imagePullSecret: '' diff --git a/deploy/helm/shipsec/values/cloud-generic.yaml b/deploy/helm/shipsec/values/cloud-generic.yaml new file mode 100644 index 000000000..d48dfc0d2 --- /dev/null +++ b/deploy/helm/shipsec/values/cloud-generic.yaml @@ -0,0 +1,16 @@ +secrets: + create: false + +backend: + service: + type: ClusterIP + +frontend: + service: + type: ClusterIP + +execution: + dind: + enabled: false + workerDockerHost: "" + diff --git a/deploy/helm/shipsec/values/dind.yaml b/deploy/helm/shipsec/values/dind.yaml new file mode 100644 index 000000000..a1f7c195f --- /dev/null +++ b/deploy/helm/shipsec/values/dind.yaml @@ -0,0 +1,5 @@ +execution: + dind: + enabled: true + workerDockerHost: tcp://shipsec-dind.shipsec-workloads.svc.cluster.local:2375 + diff --git a/deploy/helm/shipsec/values/gke-dev.yaml b/deploy/helm/shipsec/values/gke-dev.yaml new file mode 100644 index 000000000..6726cdd5e --- /dev/null +++ b/deploy/helm/shipsec/values/gke-dev.yaml @@ -0,0 +1,14 @@ +global: + namespaces: + system: shipsec-system + workers: shipsec-workers + workloads: shipsec-workloads + +backend: + service: + type: LoadBalancer + +frontend: + service: + type: LoadBalancer + diff --git a/deploy/helm/shipsec/values/gke-managed.yaml b/deploy/helm/shipsec/values/gke-managed.yaml new file mode 100644 index 000000000..fe477ac2a --- /dev/null +++ b/deploy/helm/shipsec/values/gke-managed.yaml @@ -0,0 +1,95 @@ +# GKE with managed services (Cloud SQL, Memorystore). +# Layer on top of gke-dev.yaml: +# --values values/gke-dev.yaml --values values/gke-managed.yaml + +global: + namespaces: + system: shipsec-system + workers: shipsec-workers + workloads: shipsec-workloads + +secrets: + create: true + name: shipsec-app-secrets + databaseUrl: 'postgresql://shipsec:shipsec-dev-2026@10.25.225.3:5432/shipsec' + minioRootUser: minioadmin + minioRootPassword: minioadmin + secretStoreMasterKey: '0123456789abcdef0123456789abcdef' + +backend: + image: + repository: us-central1-docker.pkg.dev/shipsec/shipsec-studio/backend + tag: 718736ca-20260209130018 + service: + type: LoadBalancer + env: + NODE_ENV: production + SHIPSEC_ENV: local + PORT: '3211' + ENABLE_INGEST_SERVICES: 'true' + TEMPORAL_ADDRESS: shipsec-temporal.shipsec-system.svc.cluster.local:7233 + TEMPORAL_NAMESPACE: shipsec-dev + TEMPORAL_TASK_QUEUE: shipsec-dev + MINIO_ENDPOINT: shipsec-minio.shipsec-system.svc.cluster.local + MINIO_PORT: '9000' + LOKI_URL: http://shipsec-loki.shipsec-system.svc.cluster.local:3100 + TERMINAL_REDIS_URL: 'redis://10.25.224.3:6379' + LOG_KAFKA_BROKERS: shipsec-redpanda.shipsec-system.svc.cluster.local:9092 + LOG_KAFKA_TOPIC: telemetry.logs + LOG_KAFKA_CLIENT_ID: shipsec-backend + EVENT_KAFKA_TOPIC: telemetry.events + EVENT_KAFKA_CLIENT_ID: shipsec-backend-events + EVENT_KAFKA_GROUP_ID: shipsec-event-ingestor + AUTH_PROVIDER: local + +worker: + image: + repository: us-central1-docker.pkg.dev/shipsec/shipsec-studio/worker + tag: 49d5de9a-wk-fix2-20260218003437 + env: + NODE_ENV: production + SHIPSEC_ENV: local + ENABLE_INGEST_SERVICES: 'false' + TEMPORAL_ADDRESS: shipsec-temporal.shipsec-system.svc.cluster.local:7233 + TEMPORAL_NAMESPACE: shipsec-dev + TEMPORAL_TASK_QUEUE: shipsec-dev + MINIO_ENDPOINT: shipsec-minio.shipsec-system.svc.cluster.local + MINIO_PORT: '9000' + MINIO_BUCKET_NAME: shipsec-files + LOKI_URL: http://shipsec-loki.shipsec-system.svc.cluster.local:3100 + TERMINAL_REDIS_URL: 'redis://10.25.224.3:6379' + TERMINAL_REDIS_MAXLEN: '5000' + LOG_KAFKA_BROKERS: shipsec-redpanda.shipsec-system.svc.cluster.local:9092 + LOG_KAFKA_TOPIC: telemetry.logs + LOG_KAFKA_CLIENT_ID: shipsec-worker + EVENT_KAFKA_TOPIC: telemetry.events + EVENT_KAFKA_CLIENT_ID: shipsec-worker-events + BACKEND_URL: http://shipsec-backend.shipsec-system.svc.cluster.local:3211 + +frontend: + image: + repository: us-central1-docker.pkg.dev/shipsec/shipsec-studio/frontend + tag: 21a84c73-fe-20260212115953 + service: + type: LoadBalancer + +ingress: + enabled: true + host: studio-next.shipsec.ai + websocket: true + tls: + enabled: true + clusterIssuer: letsencrypt-prod + secretName: shipsec-tls + +# K8s Jobs for component execution — no DIND needed +execution: + mode: k8s + k8s: + jobNamespace: shipsec-workloads + imagePullPolicy: IfNotPresent + imagePullSecret: ghcr-creds + gcsBucket: shipsec-volumes-shipsec-dev + jobServiceAccount: shipsec-job-runner + jobRunnerGcpSa: shipsec-job-runner@shipsec.iam.gserviceaccount.com + workerGcpSa: shipsec-worker@shipsec.iam.gserviceaccount.com diff --git a/deploy/helm/shipsec/values/local-orbstack.yaml b/deploy/helm/shipsec/values/local-orbstack.yaml new file mode 100644 index 000000000..b7cb1ad0b --- /dev/null +++ b/deploy/helm/shipsec/values/local-orbstack.yaml @@ -0,0 +1,16 @@ +global: + namespaces: + system: shipsec-system + workers: shipsec-workers + workloads: shipsec-workloads + +backend: + service: + type: LoadBalancer + port: 3211 + +frontend: + service: + type: LoadBalancer + port: 8090 + diff --git a/deploy/helm/shipsec/values/no-dind.yaml b/deploy/helm/shipsec/values/no-dind.yaml new file mode 100644 index 000000000..98304b034 --- /dev/null +++ b/deploy/helm/shipsec/values/no-dind.yaml @@ -0,0 +1,5 @@ +execution: + dind: + enabled: false + workerDockerHost: "" + diff --git a/deploy/helm/shipsec/values/vps.yaml b/deploy/helm/shipsec/values/vps.yaml new file mode 100644 index 000000000..a234c394f --- /dev/null +++ b/deploy/helm/shipsec/values/vps.yaml @@ -0,0 +1,15 @@ +global: + namespaces: + system: shipsec-system + workers: shipsec-workers + workloads: shipsec-workloads + +backend: + service: + type: ClusterIP + port: 3211 + +frontend: + service: + type: ClusterIP + port: 8080 diff --git a/deploy/scripts/gcp/README.md b/deploy/scripts/gcp/README.md new file mode 100644 index 000000000..0dd19e657 --- /dev/null +++ b/deploy/scripts/gcp/README.md @@ -0,0 +1,47 @@ +# GCP (GKE) quickstart + +This is the "fast path" to get ShipSec Studio running on GKE Standard in `us-central1`. + +It intentionally keeps dependencies **in-cluster** for the first cloud pass. + +## Prereqs + +- `gcloud`, `kubectl`, `helm`, `docker` +- A GKE Standard cluster already created (we used `shipsec-dev` in `us-central1-a`) +- Artifact Registry repo `shipsec-studio` in `us-central1` + +## Install + +```bash +bash deploy/scripts/gcp/install.sh +``` + +Override defaults: + +```bash +PROJECT_ID=shipsec REGION=us-central1 ZONE=us-central1-a CLUSTER_NAME=shipsec-dev IMAGE_TAG=dev1 bash deploy/scripts/gcp/install.sh +``` + +## Smoke + +```bash +bash deploy/scripts/gcp/smoke.sh +``` + +## Notes + +- This path uses DinD (privileged) for now. Treat it as trusted-tenant only. +- Frontend is built with `VITE_API_URL` pointing to the backend LoadBalancer IP. +- If you build from an Apple Silicon machine, you must push `linux/amd64` images to GKE nodes. Otherwise pods will crash with `exec format error`. `install.sh` enforces `--platform linux/amd64` and uses a unique `IMAGE_TAG` by default. + +## kubectl setup (on your machine) + +```bash +gcloud components install gke-gcloud-auth-plugin --quiet +gcloud config set project shipsec +gcloud config set compute/region us-central1 +gcloud config set compute/zone us-central1-a +gcloud container clusters get-credentials shipsec-dev --zone us-central1-a --project shipsec +kubectl config current-context +kubectl get nodes +``` diff --git a/deploy/scripts/gcp/install.sh b/deploy/scripts/gcp/install.sh new file mode 100755 index 000000000..a51e53d82 --- /dev/null +++ b/deploy/scripts/gcp/install.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" + +PROJECT_ID="${PROJECT_ID:-shipsec}" +REGION="${REGION:-us-central1}" +ZONE="${ZONE:-us-central1-a}" +CLUSTER_NAME="${CLUSTER_NAME:-shipsec-dev}" +KUBE_CONTEXT="gke_${PROJECT_ID}_${ZONE}_${CLUSTER_NAME}" + +SYSTEM_NS="${SYSTEM_NS:-shipsec-system}" +WORKERS_NS="${WORKERS_NS:-shipsec-workers}" +WORKLOADS_NS="${WORKLOADS_NS:-shipsec-workloads}" + +AR_REPO="${AR_REPO:-shipsec-studio}" +GIT_SHA="$(git -C "${ROOT_DIR}" rev-parse --short HEAD)" +# Default tag includes a timestamp to avoid amd64/arm64 tag collisions and to +# ensure GKE nodes pull the new image. +IMAGE_TAG="${IMAGE_TAG:-${GIT_SHA}-$(date +%Y%m%d%H%M%S)}" + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "[shipsec] Missing required command: $1" >&2 + exit 1 + fi +} + +require_cmd gcloud +require_cmd kubectl +require_cmd helm +require_cmd docker + +echo "[shipsec] Configuring gcloud defaults..." +gcloud config set project "${PROJECT_ID}" >/dev/null +gcloud config set compute/region "${REGION}" >/dev/null +gcloud config set compute/zone "${ZONE}" >/dev/null + +echo "[shipsec] Fetching GKE credentials..." +gcloud container clusters get-credentials "${CLUSTER_NAME}" --zone "${ZONE}" --project "${PROJECT_ID}" >/dev/null + +echo "[shipsec] Ensuring Artifact Registry pull permissions for nodes..." +PROJECT_NUMBER="$(gcloud projects describe "${PROJECT_ID}" --format='value(projectNumber)')" +NODE_SA="${PROJECT_NUMBER}-compute@developer.gserviceaccount.com" +gcloud projects add-iam-policy-binding "${PROJECT_ID}" \ + --member="serviceAccount:${NODE_SA}" \ + --role="roles/artifactregistry.reader" \ + --quiet >/dev/null || true + +echo "[shipsec] Configuring docker auth for Artifact Registry..." +gcloud auth configure-docker "${REGION}-docker.pkg.dev" --quiet >/dev/null + +BACKEND_IMAGE="${REGION}-docker.pkg.dev/${PROJECT_ID}/${AR_REPO}/backend:${IMAGE_TAG}" +WORKER_IMAGE="${REGION}-docker.pkg.dev/${PROJECT_ID}/${AR_REPO}/worker:${IMAGE_TAG}" +FRONTEND_IMAGE="${REGION}-docker.pkg.dev/${PROJECT_ID}/${AR_REPO}/frontend:${IMAGE_TAG}" + +echo "[shipsec] Building + pushing backend/worker images (linux/amd64)..." +cd "${ROOT_DIR}" +docker buildx build --platform linux/amd64 --target backend -t "${BACKEND_IMAGE}" --push . +docker buildx build --platform linux/amd64 --target worker -t "${WORKER_IMAGE}" --push . + +echo "[shipsec] Creating namespaces (idempotent)..." +kubectl --context "${KUBE_CONTEXT}" get namespace "${SYSTEM_NS}" >/dev/null 2>&1 || kubectl --context "${KUBE_CONTEXT}" create namespace "${SYSTEM_NS}" +kubectl --context "${KUBE_CONTEXT}" get namespace "${WORKERS_NS}" >/dev/null 2>&1 || kubectl --context "${KUBE_CONTEXT}" create namespace "${WORKERS_NS}" +kubectl --context "${KUBE_CONTEXT}" get namespace "${WORKLOADS_NS}" >/dev/null 2>&1 || kubectl --context "${KUBE_CONTEXT}" create namespace "${WORKLOADS_NS}" + +echo "[shipsec] Installing infra chart (in-cluster deps, fast path)..." +helm upgrade --install shipsec-infra "${ROOT_DIR}/deploy/helm/shipsec-infra" \ + --namespace "${SYSTEM_NS}" \ + --kube-context "${KUBE_CONTEXT}" \ + --values "${ROOT_DIR}/deploy/helm/shipsec-infra/values.yaml" \ + --values "${ROOT_DIR}/deploy/helm/shipsec-infra/values/gke-dev.yaml" + +echo "[shipsec] Installing app chart (backend/worker first; frontend later)..." +helm upgrade --install shipsec "${ROOT_DIR}/deploy/helm/shipsec" \ + --namespace "${SYSTEM_NS}" \ + --kube-context "${KUBE_CONTEXT}" \ + --values "${ROOT_DIR}/deploy/helm/shipsec/values.yaml" \ + --values "${ROOT_DIR}/deploy/helm/shipsec/values/gke-dev.yaml" \ + --values "${ROOT_DIR}/deploy/helm/shipsec/values/dind.yaml" \ + --set "frontend.enabled=false" \ + --set "backend.image.repository=${REGION}-docker.pkg.dev/${PROJECT_ID}/${AR_REPO}/backend" \ + --set "backend.image.tag=${IMAGE_TAG}" \ + --set "backend.image.pullPolicy=IfNotPresent" \ + --set "worker.image.repository=${REGION}-docker.pkg.dev/${PROJECT_ID}/${AR_REPO}/worker" \ + --set "worker.image.tag=${IMAGE_TAG}" \ + --set "worker.image.pullPolicy=IfNotPresent" + +echo "[shipsec] Waiting for backend service external IP..." +BACKEND_IP="" +for _ in $(seq 1 60); do + BACKEND_IP="$(kubectl --context "${KUBE_CONTEXT}" -n "${SYSTEM_NS}" get svc shipsec-backend -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || true)" + if [[ -n "${BACKEND_IP}" ]]; then + break + fi + sleep 5 +done + +if [[ -z "${BACKEND_IP}" ]]; then + echo "[shipsec] Backend LoadBalancer IP not assigned yet. You can check:" >&2 + echo " kubectl --context ${KUBE_CONTEXT} -n ${SYSTEM_NS} get svc shipsec-backend -o wide" >&2 + exit 1 +fi + +echo "[shipsec] Backend external IP: ${BACKEND_IP}" + +echo "[shipsec] Building + pushing frontend image (linux/amd64; VITE_API_URL points to backend LB)..." +docker buildx build --platform linux/amd64 \ + --target frontend \ + -t "${FRONTEND_IMAGE}" \ + --build-arg "VITE_API_URL=http://${BACKEND_IP}:3211" \ + --build-arg "VITE_BACKEND_URL=http://${BACKEND_IP}:3211" \ + --push \ + . + +echo "[shipsec] Enabling frontend deployment..." +helm upgrade --install shipsec "${ROOT_DIR}/deploy/helm/shipsec" \ + --namespace "${SYSTEM_NS}" \ + --kube-context "${KUBE_CONTEXT}" \ + --values "${ROOT_DIR}/deploy/helm/shipsec/values.yaml" \ + --values "${ROOT_DIR}/deploy/helm/shipsec/values/gke-dev.yaml" \ + --values "${ROOT_DIR}/deploy/helm/shipsec/values/dind.yaml" \ + --set "frontend.enabled=true" \ + --set "backend.image.repository=${REGION}-docker.pkg.dev/${PROJECT_ID}/${AR_REPO}/backend" \ + --set "backend.image.tag=${IMAGE_TAG}" \ + --set "backend.image.pullPolicy=IfNotPresent" \ + --set "worker.image.repository=${REGION}-docker.pkg.dev/${PROJECT_ID}/${AR_REPO}/worker" \ + --set "worker.image.tag=${IMAGE_TAG}" \ + --set "worker.image.pullPolicy=IfNotPresent" \ + --set "frontend.image.repository=${REGION}-docker.pkg.dev/${PROJECT_ID}/${AR_REPO}/frontend" \ + --set "frontend.image.tag=${IMAGE_TAG}" \ + --set "frontend.image.pullPolicy=IfNotPresent" + +echo "[shipsec] Done. Check services:" +echo " kubectl --context ${KUBE_CONTEXT} -n ${SYSTEM_NS} get svc -o wide" diff --git a/deploy/scripts/gcp/smoke.sh b/deploy/scripts/gcp/smoke.sh new file mode 100755 index 000000000..fb0c7370a --- /dev/null +++ b/deploy/scripts/gcp/smoke.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail + +PROJECT_ID="${PROJECT_ID:-shipsec}" +ZONE="${ZONE:-us-central1-a}" +CLUSTER_NAME="${CLUSTER_NAME:-shipsec-dev}" +KUBE_CONTEXT="gke_${PROJECT_ID}_${ZONE}_${CLUSTER_NAME}" + +SYSTEM_NS="${SYSTEM_NS:-shipsec-system}" + +echo "[shipsec] Pods:" +kubectl --context "${KUBE_CONTEXT}" get pods -A + +echo "[shipsec] Waiting for core deployments..." +kubectl --context "${KUBE_CONTEXT}" wait --namespace "${SYSTEM_NS}" --for=condition=available deployment/shipsec-backend --timeout=300s +kubectl --context "${KUBE_CONTEXT}" wait --namespace "${SYSTEM_NS}" --for=condition=available deployment/shipsec-frontend --timeout=300s +kubectl --context "${KUBE_CONTEXT}" wait --namespace "${SYSTEM_NS}" --for=condition=available deployment/shipsec-temporal --timeout=420s +kubectl --context "${KUBE_CONTEXT}" wait --namespace "${SYSTEM_NS}" --for=condition=available deployment/shipsec-temporal-ui --timeout=300s +kubectl --context "${KUBE_CONTEXT}" wait --namespace "${SYSTEM_NS}" --for=condition=available deployment/shipsec-redis --timeout=300s +kubectl --context "${KUBE_CONTEXT}" wait --namespace "${SYSTEM_NS}" --for=condition=ready pod -l app.kubernetes.io/component=postgres --timeout=420s +kubectl --context "${KUBE_CONTEXT}" wait --namespace "${SYSTEM_NS}" --for=condition=ready pod -l app.kubernetes.io/component=minio --timeout=420s +kubectl --context "${KUBE_CONTEXT}" wait --namespace "${SYSTEM_NS}" --for=condition=ready pod -l app.kubernetes.io/component=redpanda --timeout=420s + +echo "[shipsec] Waiting for DinD..." +kubectl --context "${KUBE_CONTEXT}" wait --namespace shipsec-workloads --for=condition=available deployment/shipsec-dind --timeout=420s + +echo "[shipsec] Services:" +kubectl --context "${KUBE_CONTEXT}" --namespace "${SYSTEM_NS}" get svc -o wide + diff --git a/deploy/scripts/orbstack/install.sh b/deploy/scripts/orbstack/install.sh new file mode 100755 index 000000000..fae2e3fd1 --- /dev/null +++ b/deploy/scripts/orbstack/install.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" + +SYSTEM_NS="${SYSTEM_NS:-shipsec-system}" +WORKERS_NS="${WORKERS_NS:-shipsec-workers}" +WORKLOADS_NS="${WORKLOADS_NS:-shipsec-workloads}" + +echo "[shipsec] Creating namespaces (idempotent)..." +kubectl get namespace "${SYSTEM_NS}" >/dev/null 2>&1 || kubectl create namespace "${SYSTEM_NS}" +kubectl get namespace "${WORKERS_NS}" >/dev/null 2>&1 || kubectl create namespace "${WORKERS_NS}" +kubectl get namespace "${WORKLOADS_NS}" >/dev/null 2>&1 || kubectl create namespace "${WORKLOADS_NS}" + +echo "[shipsec] Installing infra chart..." +helm upgrade --install shipsec-infra "${ROOT_DIR}/deploy/helm/shipsec-infra" \ + --namespace "${SYSTEM_NS}" \ + --values "${ROOT_DIR}/deploy/helm/shipsec-infra/values/local-orbstack.yaml" + +echo "[shipsec] Installing app chart..." +helm upgrade --install shipsec "${ROOT_DIR}/deploy/helm/shipsec" \ + --namespace "${SYSTEM_NS}" \ + --values "${ROOT_DIR}/deploy/helm/shipsec/values/local-orbstack.yaml" \ + --values "${ROOT_DIR}/deploy/helm/shipsec/values/dind.yaml" + +echo "[shipsec] Done." diff --git a/deploy/scripts/orbstack/smoke.sh b/deploy/scripts/orbstack/smoke.sh new file mode 100755 index 000000000..7e052c536 --- /dev/null +++ b/deploy/scripts/orbstack/smoke.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +SYSTEM_NS="${SYSTEM_NS:-shipsec-system}" + +echo "[shipsec] Pods:" +kubectl get pods -A + +echo "[shipsec] Waiting for backend to be Ready..." +kubectl wait --namespace "${SYSTEM_NS}" --for=condition=available deployment/shipsec-backend --timeout=180s + +echo "[shipsec] Checking backend health..." +curl -fsS http://localhost:3211/health >/dev/null + +echo "[shipsec] OK" + diff --git a/deploy/scripts/orbstack/uninstall.sh b/deploy/scripts/orbstack/uninstall.sh new file mode 100755 index 000000000..59cc72b63 --- /dev/null +++ b/deploy/scripts/orbstack/uninstall.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail + +SYSTEM_NS="${SYSTEM_NS:-shipsec-system}" + +echo "[shipsec] Uninstalling app chart..." +helm uninstall shipsec --namespace "${SYSTEM_NS}" || true + +echo "[shipsec] Uninstalling infra chart..." +helm uninstall shipsec-infra --namespace "${SYSTEM_NS}" || true + +echo "[shipsec] Done." + diff --git a/deploy/scripts/vps/install.sh b/deploy/scripts/vps/install.sh new file mode 100755 index 000000000..7a719f715 --- /dev/null +++ b/deploy/scripts/vps/install.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" + +SYSTEM_NS="${SYSTEM_NS:-shipsec-system}" +WORKERS_NS="${WORKERS_NS:-shipsec-workers}" +WORKLOADS_NS="${WORKLOADS_NS:-shipsec-workloads}" +KUBE_CONTEXT="${KUBE_CONTEXT:-kind-shipsec}" +KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-shipsec}" +SHIPSEC_BUILD_FRONTEND="${SHIPSEC_BUILD_FRONTEND:-1}" + +if command -v kind >/dev/null 2>&1; then + if ! kind get clusters 2>/dev/null | grep -q "^${KIND_CLUSTER_NAME}$"; then + echo "[shipsec] Creating kind cluster: ${KIND_CLUSTER_NAME}" + kind create cluster --name "${KIND_CLUSTER_NAME}" --wait 180s + fi +fi + +echo "[shipsec] Creating namespaces (idempotent)..." +kubectl --context "${KUBE_CONTEXT}" get namespace "${SYSTEM_NS}" >/dev/null 2>&1 || kubectl --context "${KUBE_CONTEXT}" create namespace "${SYSTEM_NS}" +kubectl --context "${KUBE_CONTEXT}" get namespace "${WORKERS_NS}" >/dev/null 2>&1 || kubectl --context "${KUBE_CONTEXT}" create namespace "${WORKERS_NS}" +kubectl --context "${KUBE_CONTEXT}" get namespace "${WORKLOADS_NS}" >/dev/null 2>&1 || kubectl --context "${KUBE_CONTEXT}" create namespace "${WORKLOADS_NS}" + +IMAGE_OVERRIDES=() +if [[ "${SHIPSEC_BUILD_IMAGES:-0}" == "1" ]]; then + echo "[shipsec] Building images locally (SHIPSEC_BUILD_IMAGES=1)..." + cd "${ROOT_DIR}" + docker build -t shipsec-backend:dev --target backend . + docker build -t shipsec-worker:dev --target worker . + if [[ "${SHIPSEC_BUILD_FRONTEND}" == "1" ]]; then + docker build -t shipsec-frontend:dev --target frontend . + else + echo "[shipsec] Skipping frontend image build (SHIPSEC_BUILD_FRONTEND=0)" + fi + + if command -v kind >/dev/null 2>&1; then + echo "[shipsec] Loading images into kind..." + kind load docker-image shipsec-backend:dev --name "${KIND_CLUSTER_NAME}" + kind load docker-image shipsec-worker:dev --name "${KIND_CLUSTER_NAME}" + if [[ "${SHIPSEC_BUILD_FRONTEND}" == "1" ]]; then + kind load docker-image shipsec-frontend:dev --name "${KIND_CLUSTER_NAME}" + fi + fi + + IMAGE_OVERRIDES+=("--set" "backend.image.repository=shipsec-backend") + IMAGE_OVERRIDES+=("--set" "backend.image.tag=dev") + IMAGE_OVERRIDES+=("--set" "backend.image.pullPolicy=IfNotPresent") + IMAGE_OVERRIDES+=("--set" "worker.image.repository=shipsec-worker") + IMAGE_OVERRIDES+=("--set" "worker.image.tag=dev") + IMAGE_OVERRIDES+=("--set" "worker.image.pullPolicy=IfNotPresent") + if [[ "${SHIPSEC_BUILD_FRONTEND}" == "1" ]]; then + IMAGE_OVERRIDES+=("--set" "frontend.image.repository=shipsec-frontend") + IMAGE_OVERRIDES+=("--set" "frontend.image.tag=dev") + IMAGE_OVERRIDES+=("--set" "frontend.image.pullPolicy=IfNotPresent") + fi +fi + +echo "[shipsec] Installing infra chart (in-cluster deps for VPS test)..." +helm upgrade --install shipsec-infra "${ROOT_DIR}/deploy/helm/shipsec-infra" \ + --namespace "${SYSTEM_NS}" \ + --kube-context "${KUBE_CONTEXT}" \ + --values "${ROOT_DIR}/deploy/helm/shipsec-infra/values.yaml" \ + --values "${ROOT_DIR}/deploy/helm/shipsec-infra/values/vps.yaml" + +echo "[shipsec] Installing app chart (DinD enabled for now)..." +helm upgrade --install shipsec "${ROOT_DIR}/deploy/helm/shipsec" \ + --namespace "${SYSTEM_NS}" \ + --kube-context "${KUBE_CONTEXT}" \ + --values "${ROOT_DIR}/deploy/helm/shipsec/values.yaml" \ + --values "${ROOT_DIR}/deploy/helm/shipsec/values/vps.yaml" \ + --values "${ROOT_DIR}/deploy/helm/shipsec/values/dind.yaml" \ + "${IMAGE_OVERRIDES[@]}" + +cat <<'EOF' + +[shipsec] Install complete. + +Recommended access pattern on a VPS (simple, no LB/Ingress required): + +1) Backend: + kubectl -n shipsec-system port-forward svc/shipsec-backend 3211:3211 + +2) Frontend: + kubectl -n shipsec-system port-forward svc/shipsec-frontend 8090:8080 + +3) Temporal UI: + kubectl -n shipsec-system port-forward svc/shipsec-temporal-ui 8081:8081 + +4) MinIO console: + kubectl -n shipsec-system port-forward svc/shipsec-minio 9001:9001 + +Then SSH tunnel from your laptop: + ssh -L 3211:localhost:3211 -L 8090:localhost:8090 -L 8081:localhost:8081 -L 9001:localhost:9001 clevervps + +EOF diff --git a/deploy/scripts/vps/smoke.sh b/deploy/scripts/vps/smoke.sh new file mode 100755 index 000000000..d2eefa294 --- /dev/null +++ b/deploy/scripts/vps/smoke.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +SYSTEM_NS="${SYSTEM_NS:-shipsec-system}" +KUBE_CONTEXT="${KUBE_CONTEXT:-kind-shipsec}" + +echo "[shipsec] Pods:" +kubectl --context "${KUBE_CONTEXT}" get pods -A + +echo "[shipsec] Waiting for core deployments..." +kubectl --context "${KUBE_CONTEXT}" wait --namespace "${SYSTEM_NS}" --for=condition=available deployment/shipsec-backend --timeout=240s +if kubectl --context "${KUBE_CONTEXT}" --namespace "${SYSTEM_NS}" get deployment/shipsec-frontend >/dev/null 2>&1; then + kubectl --context "${KUBE_CONTEXT}" wait --namespace "${SYSTEM_NS}" --for=condition=available deployment/shipsec-frontend --timeout=240s +fi +kubectl --context "${KUBE_CONTEXT}" wait --namespace "${SYSTEM_NS}" --for=condition=available deployment/shipsec-temporal --timeout=300s +kubectl --context "${KUBE_CONTEXT}" wait --namespace "${SYSTEM_NS}" --for=condition=available deployment/shipsec-temporal-ui --timeout=240s +kubectl --context "${KUBE_CONTEXT}" wait --namespace "${SYSTEM_NS}" --for=condition=available deployment/shipsec-redis --timeout=240s +kubectl --context "${KUBE_CONTEXT}" wait --namespace "${SYSTEM_NS}" --for=condition=ready pod -l app.kubernetes.io/component=postgres --timeout=300s +kubectl --context "${KUBE_CONTEXT}" wait --namespace "${SYSTEM_NS}" --for=condition=ready pod -l app.kubernetes.io/component=minio --timeout=300s +kubectl --context "${KUBE_CONTEXT}" wait --namespace "${SYSTEM_NS}" --for=condition=ready pod -l app.kubernetes.io/component=redpanda --timeout=300s + +echo "[shipsec] Waiting for DinD..." +kubectl --context "${KUBE_CONTEXT}" wait --namespace shipsec-workloads --for=condition=available deployment/shipsec-dind --timeout=300s + +echo "[shipsec] OK (deployments/pods Ready). To verify HTTP endpoints, use port-forward as printed by install.sh." diff --git a/docker/PRODUCTION.md b/docker/PRODUCTION.md index dd5908d09..f6b8f33cb 100644 --- a/docker/PRODUCTION.md +++ b/docker/PRODUCTION.md @@ -4,20 +4,22 @@ This guide covers deploying the analytics infrastructure with security and SaaS ## Overview -| Environment | Security | Multitenancy | Use Case | -|-------------|----------|--------------|----------| -| Development | Disabled | No | Local development, fast iteration | -| Production | Enabled | Yes (Strict) | Multi-tenant SaaS deployment | +| Environment | Security | Multitenancy | Use Case | +| ----------- | -------- | ------------ | --------------------------------- | +| Development | Disabled | No | Local development, fast iteration | +| Production | Enabled | Yes (Strict) | Multi-tenant SaaS deployment | ## SaaS Multitenancy Model **Key Principles:** + - Each customer gets complete data isolation by default - No shared dashboards - sharing is explicitly opt-in - Each customer has their own index pattern (`{customer_id}-*`) - Tenants, roles, and users are created dynamically via backend **Index Naming Convention:** + ``` {customer_id}-analytics-* # Analytics data {customer_id}-workflows-* # Workflow results @@ -40,18 +42,18 @@ docker compose -f docker-compose.infra.yml -f docker-compose.prod.yml up -d ## Files Overview -| File | Purpose | -|------|---------| -| `docker-compose.infra.yml` | Base infrastructure (dev mode, PM2 on host) | -| `docker-compose.full.yml` | Full stack containerized (simple prod, no security) | -| `docker-compose.prod.yml` | Security overlay (combines with infra.yml for SaaS) | -| `nginx/nginx.dev.conf` | Nginx routing to host (PM2 services) | -| `nginx/nginx.prod.conf` | Nginx routing to containers | -| `opensearch-dashboards.yml` | Dashboards config (dev) | -| `opensearch-dashboards.prod.yml` | Dashboards config (prod with multitenancy) | -| `scripts/generate-certs.sh` | TLS certificate generator | -| `opensearch-security/` | Security plugin configuration | -| `certs/` | Generated certificates (gitignored) | +| File | Purpose | +| -------------------------------- | --------------------------------------------------- | +| `docker-compose.infra.yml` | Base infrastructure (dev mode, PM2 on host) | +| `docker-compose.full.yml` | Full stack containerized (simple prod, no security) | +| `docker-compose.prod.yml` | Security overlay (combines with infra.yml for SaaS) | +| `nginx/nginx.dev.conf` | Nginx routing to host (PM2 services) | +| `nginx/nginx.prod.conf` | Nginx routing to containers | +| `opensearch-dashboards.yml` | Dashboards config (dev) | +| `opensearch-dashboards.prod.yml` | Dashboards config (prod with multitenancy) | +| `scripts/generate-certs.sh` | TLS certificate generator | +| `opensearch-security/` | Security plugin configuration | +| `certs/` | Generated certificates (gitignored) | See [README.md](README.md) for detailed usage of each compose file. @@ -60,6 +62,7 @@ See [README.md](README.md) for detailed usage of each compose file. When a new customer is onboarded, the backend must create: ### 1. Create Customer Tenant + ```bash PUT /_plugins/_security/api/tenants/{customer_id} { @@ -68,6 +71,7 @@ PUT /_plugins/_security/api/tenants/{customer_id} ``` ### 2. Create Customer Role (with Index Isolation) + ```bash PUT /_plugins/_security/api/roles/customer_{customer_id}_rw { @@ -84,6 +88,7 @@ PUT /_plugins/_security/api/roles/customer_{customer_id}_rw ``` ### 3. Create Customer User + ```bash PUT /_plugins/_security/api/internalusers/{user_email} { @@ -97,6 +102,7 @@ PUT /_plugins/_security/api/internalusers/{user_email} ``` ### 4. Map User to Role + ```bash PUT /_plugins/_security/api/rolesmapping/customer_{customer_id}_rw { @@ -116,6 +122,7 @@ The `scripts/generate-certs.sh` script generates: - **admin.pem / admin-key.pem** - Admin certificate for cluster management For production: + - Use a proper CA (Let's Encrypt, internal PKI) - Store private keys in a secrets manager (Vault, AWS Secrets Manager) - Set up certificate rotation before expiration @@ -124,16 +131,17 @@ For production: Only two system users are defined (in `internal_users.yml`): -| User | Purpose | -|------|---------| -| `admin` | Platform operations - DO NOT give to customers | -| `kibanaserver` | Dashboards backend communication | +| User | Purpose | +| -------------- | ---------------------------------------------- | +| `admin` | Platform operations - DO NOT give to customers | +| `kibanaserver` | Dashboards backend communication | Customer users are created dynamically via the Security REST API. ### Password Hashing Generate password hashes for users: + ```bash docker run -it opensearchproject/opensearch:2.11.1 \ /usr/share/opensearch/plugins/opensearch-security/tools/hash.sh -p YOUR_PASSWORD @@ -155,10 +163,10 @@ curl -u user@customer.com:password \ ## Environment Variables -| Variable | Required | Description | -|----------|----------|-------------| -| `OPENSEARCH_ADMIN_PASSWORD` | Yes | Admin user password | -| `OPENSEARCH_DASHBOARDS_PASSWORD` | Yes | kibanaserver user password | +| Variable | Required | Description | +| -------------------------------- | -------- | -------------------------- | +| `OPENSEARCH_ADMIN_PASSWORD` | Yes | Admin user password | +| `OPENSEARCH_DASHBOARDS_PASSWORD` | Yes | kibanaserver user password | ## Updating Security Configuration @@ -179,12 +187,14 @@ docker exec -it shipsec-opensearch \ ### Container fails to start Check logs: + ```bash docker logs shipsec-opensearch docker logs shipsec-opensearch-dashboards ``` Common issues: + - Certificate permissions (should be 600 for keys, 644 for certs) - Missing environment variables - Incorrect certificate paths @@ -206,6 +216,7 @@ curl -k -u admin:PASSWORD https://localhost:9200/_cluster/health ### Cross-tenant data leak If a customer can see another customer's data: + 1. Verify index_patterns in role are correctly scoped to `{customer_id}-*` 2. Check role mapping is correct 3. Ensure user's backend_roles match their customer ID @@ -213,11 +224,13 @@ If a customer can see another customer's data: ## Switching Between Environments **Development (no security):** + ```bash docker compose -f docker-compose.infra.yml up -d ``` **Production (with security):** + ```bash docker compose -f docker-compose.infra.yml -f docker-compose.prod.yml up -d ``` diff --git a/docker/SECURE-DEV-MODE.md b/docker/SECURE-DEV-MODE.md index f20acccbc..7ce177b84 100644 --- a/docker/SECURE-DEV-MODE.md +++ b/docker/SECURE-DEV-MODE.md @@ -25,25 +25,25 @@ just dev-insecure ### Docker Compose Files -| File | Purpose | -|------|---------| -| `docker-compose.infra.yml` | Base infrastructure (Postgres, Redis, Temporal, etc.) | -| `docker-compose.dev-secure.yml` | Security overlay for development | -| `docker-compose.prod.yml` | Production security configuration | +| File | Purpose | +| ------------------------------- | ----------------------------------------------------- | +| `docker-compose.infra.yml` | Base infrastructure (Postgres, Redis, Temporal, etc.) | +| `docker-compose.dev-secure.yml` | Security overlay for development | +| `docker-compose.prod.yml` | Production security configuration | ### Security Configuration Files Located in `docker/opensearch-security/`: -| File | Purpose | -|------|---------| -| `config.yml` | Authentication/authorization backends (proxy auth) | -| `internal_users.yml` | System users (admin, kibanaserver, worker) | -| `roles.yml` | Role definitions with index permissions | -| `roles_mapping.yml` | User-to-role mappings | -| `action_groups.yml` | Permission groups for roles | -| `tenants.yml` | Tenant definitions | -| `audit.yml` | Audit logging configuration | +| File | Purpose | +| -------------------- | -------------------------------------------------- | +| `config.yml` | Authentication/authorization backends (proxy auth) | +| `internal_users.yml` | System users (admin, kibanaserver, worker) | +| `roles.yml` | Role definitions with index permissions | +| `roles_mapping.yml` | User-to-role mappings | +| `action_groups.yml` | Permission groups for roles | +| `tenants.yml` | Tenant definitions | +| `audit.yml` | Audit logging configuration | ### TLS Certificates @@ -57,13 +57,14 @@ Certificates are auto-generated on first run and stored in `docker/certs/`: For development convenience, default passwords are set: -| User | Password | Purpose | -|------|----------|---------| -| `admin` | `admin` | Platform administrator | -| `kibanaserver` | `admin` | Dashboards backend communication | -| `worker` | `admin` | Worker service for indexing | +| User | Password | Purpose | +| -------------- | -------- | -------------------------------- | +| `admin` | `admin` | Platform administrator | +| `kibanaserver` | `admin` | Dashboards backend communication | +| `worker` | `admin` | Worker service for indexing | **Important**: Change these in production via environment variables: + - `OPENSEARCH_ADMIN_PASSWORD` - `OPENSEARCH_DASHBOARDS_PASSWORD` @@ -81,6 +82,7 @@ For development convenience, default passwords are set: ### Dynamic Provisioning When a new customer is onboarded, the backend creates: + 1. A tenant for their organization 2. A role with permissions scoped to their indices 3. User-to-role mappings diff --git a/docker/docker-compose.dev-ports.yml b/docker/docker-compose.dev-ports.yml index 0da3bd056..535bd61e6 100644 --- a/docker/docker-compose.dev-ports.yml +++ b/docker/docker-compose.dev-ports.yml @@ -12,43 +12,43 @@ services: postgres: ports: - - "127.0.0.1:5433:5432" + - '127.0.0.1:5433:5432' temporal: ports: - - "127.0.0.1:7233:7233" + - '127.0.0.1:7233:7233' temporal-ui: ports: - - "127.0.0.1:8081:8080" + - '127.0.0.1:8081:8080' minio: ports: - - "127.0.0.1:9000:9000" - - "127.0.0.1:9001:9001" + - '127.0.0.1:9000:9000' + - '127.0.0.1:9001:9001' redis: ports: - - "127.0.0.1:6379:6379" + - '127.0.0.1:6379:6379' loki: ports: - - "127.0.0.1:3100:3100" + - '127.0.0.1:3100:3100' redpanda: ports: - - "127.0.0.1:9092:9092" - - "127.0.0.1:9644:9644" + - '127.0.0.1:9092:9092' + - '127.0.0.1:9644:9644' redpanda-console: ports: - - "127.0.0.1:8082:8080" + - '127.0.0.1:8082:8080' opensearch: ports: - - "127.0.0.1:9200:9200" - - "127.0.0.1:9600:9600" + - '127.0.0.1:9200:9200' + - '127.0.0.1:9600:9600' opensearch-dashboards: ports: - - "127.0.0.1:5601:5601" + - '127.0.0.1:5601:5601' diff --git a/docker/docker-compose.dev-secure.yml b/docker/docker-compose.dev-secure.yml index 1411e3822..a6ff6931b 100644 --- a/docker/docker-compose.dev-secure.yml +++ b/docker/docker-compose.dev-secure.yml @@ -17,7 +17,7 @@ services: opensearch: # Custom entrypoint for proxy auth config templating - entrypoint: ["/usr/share/opensearch/config/opensearch-security/docker-entrypoint-security.sh"] + entrypoint: ['/usr/share/opensearch/config/opensearch-security/docker-entrypoint-security.sh'] environment: # Enable security plugin (override infra.yml settings) - DISABLE_SECURITY_PLUGIN=false @@ -34,7 +34,11 @@ services: # Custom config file with admin_dn as YAML array (env vars don't support arrays) - ./opensearch.dev-secure.yml:/usr/share/opensearch/config/opensearch.yml:ro healthcheck: - test: ["CMD-SHELL", "curl -sf -u admin:$${OPENSEARCH_ADMIN_PASSWORD:-admin} --cacert /usr/share/opensearch/config/certs/root-ca.pem https://localhost:9200/_cluster/health || exit 1"] + test: + [ + 'CMD-SHELL', + 'curl -sf -u admin:$${OPENSEARCH_ADMIN_PASSWORD:-admin} --cacert /usr/share/opensearch/config/certs/root-ca.pem https://localhost:9200/_cluster/health || exit 1', + ] interval: 30s timeout: 10s retries: 10 @@ -51,7 +55,11 @@ services: - ./certs:/usr/share/opensearch-dashboards/config/certs:ro healthcheck: # Check if server responds (401 is fine - means server is up, security just requires auth) - test: ["CMD-SHELL", "curl -sf -o /dev/null -w '%{http_code}' http://localhost:5601/analytics/api/status | grep -qE '(200|401)' || exit 1"] + test: + [ + 'CMD-SHELL', + "curl -sf -o /dev/null -w '%{http_code}' http://localhost:5601/analytics/api/status | grep -qE '(200|401)' || exit 1", + ] interval: 30s timeout: 10s retries: 10 diff --git a/docker/docker-compose.infra.yml b/docker/docker-compose.infra.yml index 1e9e619c9..517883705 100644 --- a/docker/docker-compose.infra.yml +++ b/docker/docker-compose.infra.yml @@ -9,12 +9,12 @@ services: POSTGRES_MULTIPLE_DATABASES: temporal # Internal only - use docker-compose.dev-ports.yml overlay for local dev access expose: - - "5432" + - '5432' volumes: - postgres_data:/var/lib/postgresql/data - ./init-db:/docker-entrypoint-initdb.d healthcheck: - test: ["CMD-SHELL", "pg_isready -U shipsec"] + test: ['CMD-SHELL', 'pg_isready -U shipsec'] interval: 5s timeout: 3s retries: 10 @@ -35,7 +35,7 @@ services: - POSTGRES_SEEDS=postgres - AUTO_SETUP=true expose: - - "7233" + - '7233' volumes: - temporal_data:/var/lib/temporal restart: unless-stopped @@ -49,7 +49,7 @@ services: - TEMPORAL_ADDRESS=temporal:7233 - TEMPORAL_CORS_ORIGINS=http://localhost:5173 expose: - - "8080" + - '8080' restart: unless-stopped minio: @@ -60,13 +60,13 @@ services: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin expose: - - "9000" - - "9001" + - '9000' + - '9001' volumes: - minio_data:/data restart: unless-stopped healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + test: ['CMD', 'curl', '-f', 'http://localhost:9000/minio/health/live'] interval: 30s timeout: 10s retries: 5 @@ -75,12 +75,12 @@ services: image: redis:latest container_name: shipsec-redis expose: - - "6379" + - '6379' volumes: - redis_data:/data restart: unless-stopped healthcheck: - test: ["CMD", "redis-cli", "ping"] + test: ['CMD', 'redis-cli', 'ping'] interval: 30s timeout: 10s retries: 5 @@ -90,13 +90,13 @@ services: container_name: shipsec-loki command: -config.file=/etc/loki/local-config.yaml expose: - - "3100" + - '3100' volumes: - ./loki/loki-config.yaml:/etc/loki/local-config.yaml - loki_data:/loki restart: unless-stopped healthcheck: - test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:3100/ready"] + test: ['CMD', 'wget', '--no-verbose', '--tries=1', '--spider', 'http://localhost:3100/ready'] interval: 30s timeout: 10s retries: 5 @@ -115,13 +115,13 @@ services: - --check=false - --advertise-kafka-addr=localhost:9092 expose: - - "9092" - - "9644" + - '9092' + - '9644' volumes: - redpanda_data:/var/lib/redpanda/data restart: unless-stopped healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9644/v1/status/ready"] + test: ['CMD', 'curl', '-f', 'http://localhost:9644/v1/status/ready'] interval: 30s timeout: 10s retries: 5 @@ -134,7 +134,7 @@ services: environment: CONFIG_FILEPATH: /etc/redpanda/console-config.yaml expose: - - "8080" + - '8080' volumes: - ./redpanda-console-config.yaml:/etc/redpanda/console-config.yaml:ro restart: unless-stopped @@ -145,7 +145,7 @@ services: environment: - discovery.type=single-node - bootstrap.memory_lock=true - - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - 'OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m' - DISABLE_SECURITY_PLUGIN=true - DISABLE_INSTALL_DEMO_CONFIG=true ulimits: @@ -158,13 +158,13 @@ services: # Ports exposed only within Docker network (not to host) # Use docker-compose.dev-ports.yml overlay for local dev access expose: - - "9200" - - "9600" + - '9200' + - '9600' volumes: - opensearch_data:/usr/share/opensearch/data restart: unless-stopped healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:9200/_cluster/health || exit 1"] + test: ['CMD-SHELL', 'curl -f http://localhost:9200/_cluster/health || exit 1'] interval: 30s timeout: 10s retries: 5 @@ -185,12 +185,12 @@ services: # Use docker-compose.dev-ports.yml overlay for local dev access # Production uses nginx reverse proxy at /analytics expose: - - "5601" + - '5601' volumes: - ./opensearch-dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml:ro restart: unless-stopped healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:5601/analytics/api/status || exit 1"] + test: ['CMD-SHELL', 'curl -f http://localhost:5601/analytics/api/status || exit 1'] interval: 30s timeout: 10s retries: 5 @@ -204,8 +204,8 @@ services: condition: service_healthy volumes: - ./opensearch-init.sh:/init.sh:ro - entrypoint: ["/bin/sh", "/init.sh"] - restart: "no" + entrypoint: ['/bin/sh', '/init.sh'] + restart: 'no' # Nginx reverse proxy - unified entry point # DEV MODE: Uses nginx.dev.conf which points to host.docker.internal for PM2 services @@ -216,14 +216,14 @@ services: opensearch-dashboards: condition: service_healthy ports: - - "80:80" + - '80:80' volumes: - ./nginx/nginx.dev.conf:/etc/nginx/nginx.conf:ro extra_hosts: - - "host.docker.internal:host-gateway" + - 'host.docker.internal:host-gateway' restart: unless-stopped healthcheck: - test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost/health"] + test: ['CMD', 'wget', '--no-verbose', '--tries=1', '--spider', 'http://localhost/health'] interval: 30s timeout: 10s retries: 5 diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index fe81b009b..8225aca5d 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -17,7 +17,7 @@ services: opensearch: # Custom entrypoint for proxy auth config templating - entrypoint: ["/usr/share/opensearch/config/opensearch-security/docker-entrypoint-security.sh"] + entrypoint: ['/usr/share/opensearch/config/opensearch-security/docker-entrypoint-security.sh'] environment: # Remove security disable flags (override dev settings) - DISABLE_SECURITY_PLUGIN=false @@ -49,7 +49,11 @@ services: - ./certs:/usr/share/opensearch/config/certs:ro - ./opensearch-security:/usr/share/opensearch/config/opensearch-security:ro healthcheck: - test: ["CMD-SHELL", "curl -sf -u admin:$${OPENSEARCH_ADMIN_PASSWORD:-admin} --cacert /usr/share/opensearch/config/certs/root-ca.pem https://localhost:9200/_cluster/health || exit 1"] + test: + [ + 'CMD-SHELL', + 'curl -sf -u admin:$${OPENSEARCH_ADMIN_PASSWORD:-admin} --cacert /usr/share/opensearch/config/certs/root-ca.pem https://localhost:9200/_cluster/health || exit 1', + ] interval: 30s timeout: 10s retries: 10 @@ -65,7 +69,11 @@ services: - ./certs:/usr/share/opensearch-dashboards/config/certs:ro healthcheck: # Check if server responds (401 is fine - means server is up, security just requires auth) - test: ["CMD-SHELL", "curl -sf -o /dev/null -w '%{http_code}' http://localhost:5601/analytics/api/status | grep -qE '(200|401)' || exit 1"] + test: + [ + 'CMD-SHELL', + "curl -sf -o /dev/null -w '%{http_code}' http://localhost:5601/analytics/api/status | grep -qE '(200|401)' || exit 1", + ] interval: 30s timeout: 10s retries: 10 @@ -86,8 +94,8 @@ services: - ./nginx/nginx.prod.conf:/etc/nginx/nginx.conf:ro - ./certs:/etc/nginx/certs:ro ports: - - "80:80" - - "443:443" + - '80:80' + - '443:443' volumes: opensearch_data: diff --git a/docker/opensearch-dashboards.prod.yml b/docker/opensearch-dashboards.prod.yml index c9007b136..31be59f3f 100644 --- a/docker/opensearch-dashboards.prod.yml +++ b/docker/opensearch-dashboards.prod.yml @@ -6,34 +6,35 @@ # - Multitenancy for tenant isolation # - TLS for secure communication with OpenSearch -server.host: "0.0.0.0" +server.host: '0.0.0.0' server.port: 5601 # Base path configuration for reverse proxy -server.basePath: "/analytics" +server.basePath: '/analytics' server.rewriteBasePath: true # OpenSearch connection (HTTPS for production) -opensearch.hosts: ["https://opensearch:9200"] +opensearch.hosts: ['https://opensearch:9200'] # TLS Configuration - trust the CA certificate opensearch.ssl.verificationMode: certificate -opensearch.ssl.certificateAuthorities: ["/usr/share/opensearch-dashboards/config/certs/root-ca.pem"] +opensearch.ssl.certificateAuthorities: ['/usr/share/opensearch-dashboards/config/certs/root-ca.pem'] # Authentication - proxy auth from nginx (primary) + basic auth for kibanaserver # Note: OpenSearch Dashboards doesn't support env var interpolation in YAML # In production, use a secrets manager or pre-process this file -opensearch.username: "kibanaserver" -opensearch.password: "admin" -opensearch.requestHeadersAllowlist: ["securitytenant", "Authorization", "x-forwarded-for", "x-proxy-user", "x-proxy-roles"] +opensearch.username: 'kibanaserver' +opensearch.password: 'admin' +opensearch.requestHeadersAllowlist: + ['securitytenant', 'Authorization', 'x-forwarded-for', 'x-proxy-user', 'x-proxy-roles'] # Proxy Authentication Configuration # Nginx sets x-proxy-user/x-proxy-roles headers after validating user session via auth_request. # Dashboards trusts these headers (no login form). Users must log in via the main app first. # The kibanaserver user (above) is still used for Dashboards' own backend connection to OpenSearch. -opensearch_security.auth.type: "proxy" -opensearch_security.proxycache.user_header: "x-proxy-user" -opensearch_security.proxycache.roles_header: "x-proxy-roles" +opensearch_security.auth.type: 'proxy' +opensearch_security.proxycache.user_header: 'x-proxy-user' +opensearch_security.proxycache.roles_header: 'x-proxy-roles' # Security Plugin Configuration - SaaS Multitenancy # Each customer gets their own isolated tenant - no shared data by default @@ -41,19 +42,19 @@ opensearch_security.proxycache.roles_header: "x-proxy-roles" opensearch_security.multitenancy.enabled: true opensearch_security.multitenancy.tenants.enable_global: false opensearch_security.multitenancy.tenants.enable_private: false -opensearch_security.multitenancy.tenants.preferred: ["Custom"] +opensearch_security.multitenancy.tenants.preferred: ['Custom'] opensearch_security.multitenancy.show_roles: false opensearch_security.multitenancy.enable_filter: false -opensearch_security.readonly_mode.roles: ["kibana_read_only"] +opensearch_security.readonly_mode.roles: ['kibana_read_only'] opensearch_security.cookie.secure: true -opensearch_security.cookie.isSameSite: "Strict" +opensearch_security.cookie.isSameSite: 'Strict' # Session configuration opensearch_security.session.ttl: 3600000 opensearch_security.session.keepalive: true # Default landing page - Discover instead of Home (which shows all plugin links) -uiSettings.overrides.defaultRoute: "/app/discover" +uiSettings.overrides.defaultRoute: '/app/discover' # Logging logging.dest: stdout diff --git a/docker/opensearch-dashboards.yml b/docker/opensearch-dashboards.yml index 7c24007a3..6192c410d 100644 --- a/docker/opensearch-dashboards.yml +++ b/docker/opensearch-dashboards.yml @@ -10,15 +10,15 @@ # 4. Add: opensearch_security.multitenancy.enabled: true # 5. Add: opensearch_security.multitenancy.tenants.preferred: ["Private", "Global"] -server.host: "0.0.0.0" +server.host: '0.0.0.0' server.port: 5601 # Base path configuration for reverse proxy -server.basePath: "/analytics" +server.basePath: '/analytics' server.rewriteBasePath: true # OpenSearch connection -opensearch.hosts: ["http://opensearch:9200"] +opensearch.hosts: ['http://opensearch:9200'] # Logging logging.dest: stdout @@ -27,7 +27,7 @@ logging.quiet: false logging.verbose: false # Default landing page - Discover instead of Home (which shows all plugin links) -uiSettings.overrides.defaultRoute: "/app/discover" +uiSettings.overrides.defaultRoute: '/app/discover' # CSP - relaxed for development (inline scripts needed by dashboards) csp.strict: false diff --git a/docker/opensearch-security/action_groups.yml b/docker/opensearch-security/action_groups.yml index 1b4a5179a..c3bb9b38d 100644 --- a/docker/opensearch-security/action_groups.yml +++ b/docker/opensearch-security/action_groups.yml @@ -12,7 +12,7 @@ --- _meta: - type: "actiongroups" + type: 'actiongroups' config_version: 2 # ============================================================================= @@ -24,38 +24,38 @@ security_findings_write: reserved: false static: false allowed_actions: - - "indices:data/write/index" - - "indices:data/write/bulk*" - - "indices:data/write/update" - - "indices:data/write/delete" - - "indices:admin/create" - - "indices:admin/mapping/put" - description: "Write access to security findings indices" + - 'indices:data/write/index' + - 'indices:data/write/bulk*' + - 'indices:data/write/update' + - 'indices:data/write/delete' + - 'indices:admin/create' + - 'indices:admin/mapping/put' + description: 'Write access to security findings indices' security_findings_read: reserved: false static: false allowed_actions: - - "indices:data/read/search*" - - "indices:data/read/get*" - - "indices:data/read/mget*" - - "indices:data/read/msearch*" - - "indices:data/read/scroll*" - - "indices:admin/mappings/get" - - "indices:admin/resolve/index" - description: "Read access to security findings indices" + - 'indices:data/read/search*' + - 'indices:data/read/get*' + - 'indices:data/read/mget*' + - 'indices:data/read/msearch*' + - 'indices:data/read/scroll*' + - 'indices:admin/mappings/get' + - 'indices:admin/resolve/index' + description: 'Read access to security findings indices' # Dashboard access for customers dashboards_read: reserved: false static: false allowed_actions: - - "kibana_all_read" - description: "Read-only access to dashboards" + - 'kibana_all_read' + description: 'Read-only access to dashboards' dashboards_write: reserved: false static: false allowed_actions: - - "kibana_all_write" - description: "Write access to dashboards" + - 'kibana_all_write' + description: 'Write access to dashboards' diff --git a/docker/opensearch-security/allowlist.yml b/docker/opensearch-security/allowlist.yml index cd934c4ab..ac5d82527 100644 --- a/docker/opensearch-security/allowlist.yml +++ b/docker/opensearch-security/allowlist.yml @@ -5,7 +5,7 @@ --- _meta: - type: "allowlist" + type: 'allowlist' config_version: 2 config: diff --git a/docker/opensearch-security/audit.yml b/docker/opensearch-security/audit.yml index f8fae3211..685a6872c 100644 --- a/docker/opensearch-security/audit.yml +++ b/docker/opensearch-security/audit.yml @@ -4,7 +4,7 @@ --- _meta: - type: "audit" + type: 'audit' config_version: 2 config: @@ -22,9 +22,9 @@ config: exclude_sensitive_headers: true # Ignore system indices ignore_users: - - "kibanaserver" + - 'kibanaserver' ignore_requests: - - "SearchRequest" - - "indices:data/read/*" + - 'SearchRequest' + - 'indices:data/read/*' compliance: enabled: false diff --git a/docker/opensearch-security/config.yml b/docker/opensearch-security/config.yml index 7af3c434f..af5bdfc8b 100644 --- a/docker/opensearch-security/config.yml +++ b/docker/opensearch-security/config.yml @@ -6,7 +6,7 @@ --- _meta: - type: "config" + type: 'config' config_version: 2 config: @@ -30,8 +30,8 @@ config: type: proxy challenge: false config: - user_header: "x-proxy-user" - roles_header: "x-proxy-roles" + user_header: 'x-proxy-user' + roles_header: 'x-proxy-roles' authentication_backend: type: noop diff --git a/docker/opensearch-security/internal_users.yml b/docker/opensearch-security/internal_users.yml index fdb93d833..f345eb60f 100644 --- a/docker/opensearch-security/internal_users.yml +++ b/docker/opensearch-security/internal_users.yml @@ -22,7 +22,7 @@ --- _meta: - type: "internalusers" + type: 'internalusers' config_version: 2 # ============================================================================= @@ -32,33 +32,33 @@ _meta: # Platform admin - for internal operations only admin: # Default password: admin (CHANGE IN PRODUCTION!) - hash: "$2a$12$VcCDgh2NDk07JGN0rjGbM.Ad41qVR/YFJcgHp0UGns5JDymv..TOG" + hash: '$2a$12$VcCDgh2NDk07JGN0rjGbM.Ad41qVR/YFJcgHp0UGns5JDymv..TOG' reserved: true backend_roles: - - "admin" + - 'admin' attributes: - role: "system" - description: "Platform administrator - internal use only" + role: 'system' + description: 'Platform administrator - internal use only' # Dashboards server user - used by OpenSearch Dashboards kibanaserver: # Default password: admin (matches OPENSEARCH_DASHBOARDS_PASSWORD default) - hash: "$2a$12$VcCDgh2NDk07JGN0rjGbM.Ad41qVR/YFJcgHp0UGns5JDymv..TOG" + hash: '$2a$12$VcCDgh2NDk07JGN0rjGbM.Ad41qVR/YFJcgHp0UGns5JDymv..TOG' reserved: true attributes: - role: "system" - description: "Dashboards backend communication user" + role: 'system' + description: 'Dashboards backend communication user' # Worker service user - for indexing security findings from worker processes worker: # Default password: worker (CHANGE IN PRODUCTION!) - hash: "$2a$12$VcCDgh2NDk07JGN0rjGbM.Ad41qVR/YFJcgHp0UGns5JDymv..TOG" + hash: '$2a$12$VcCDgh2NDk07JGN0rjGbM.Ad41qVR/YFJcgHp0UGns5JDymv..TOG' reserved: false backend_roles: - - "worker_write" + - 'worker_write' attributes: - role: "system" - description: "Worker service for indexing security findings" + role: 'system' + description: 'Worker service for indexing security findings' # ============================================================================= # CUSTOMER USERS diff --git a/docker/opensearch-security/nodes_dn.yml b/docker/opensearch-security/nodes_dn.yml index 566555f15..c1cbbaccc 100644 --- a/docker/opensearch-security/nodes_dn.yml +++ b/docker/opensearch-security/nodes_dn.yml @@ -5,7 +5,7 @@ --- _meta: - type: "nodesdn" + type: 'nodesdn' config_version: 2 # Allow all nodes with certificates signed by our CA diff --git a/docker/opensearch-security/roles.yml b/docker/opensearch-security/roles.yml index f2d44c4d7..8fa19f745 100644 --- a/docker/opensearch-security/roles.yml +++ b/docker/opensearch-security/roles.yml @@ -26,7 +26,7 @@ --- _meta: - type: "roles" + type: 'roles' config_version: 2 # ============================================================================= @@ -37,34 +37,34 @@ _meta: platform_admin: reserved: true cluster_permissions: - - "*" + - '*' index_permissions: - index_patterns: - - "*" + - '*' allowed_actions: - - "*" + - '*' tenant_permissions: - tenant_patterns: - - "*" + - '*' allowed_actions: - - "kibana_all_write" + - 'kibana_all_write' # Worker write role - for indexing security findings from worker processes # Write-only access to security-findings-* indices (no read of other orgs' data) worker_write: reserved: false - description: "Worker service role for indexing security findings" + description: 'Worker service role for indexing security findings' cluster_permissions: - - "cluster_composite_ops_ro" - - "indices:data/write/*" + - 'cluster_composite_ops_ro' + - 'indices:data/write/*' index_permissions: - index_patterns: - - "security-findings-*" + - 'security-findings-*' allowed_actions: - - "write" - - "create_index" - - "indices:data/write/*" - - "indices:admin/mapping/put" + - 'write' + - 'create_index' + - 'indices:data/write/*' + - 'indices:admin/mapping/put' # ============================================================================= # CUSTOMER ROLE TEMPLATE @@ -76,99 +76,99 @@ worker_write: # Index pattern: {customer_id}-* customer_template_rw: reserved: false - description: "Template for customer read-write roles - DO NOT USE DIRECTLY" + description: 'Template for customer read-write roles - DO NOT USE DIRECTLY' cluster_permissions: - - "cluster_composite_ops_ro" - - "indices:data/read/scroll*" + - 'cluster_composite_ops_ro' + - 'indices:data/read/scroll*' # Required for Dashboards saved objects (bulk writes to .kibana_* tenant indices) - - "indices:data/write/bulk" + - 'indices:data/write/bulk' # Alerting: monitor CRUD, execution, alerts, and destinations (legacy endpoints) - - "cluster:admin/opendistro/alerting/monitor/get" - - "cluster:admin/opendistro/alerting/monitor/search" - - "cluster:admin/opendistro/alerting/monitor/write" - - "cluster:admin/opendistro/alerting/monitor/execute" - - "cluster:admin/opendistro/alerting/alerts/get" - - "cluster:admin/opendistro/alerting/alerts/ack" - - "cluster:admin/opendistro/alerting/destination/get" - - "cluster:admin/opendistro/alerting/destination/write" - - "cluster:admin/opendistro/alerting/destination/delete" + - 'cluster:admin/opendistro/alerting/monitor/get' + - 'cluster:admin/opendistro/alerting/monitor/search' + - 'cluster:admin/opendistro/alerting/monitor/write' + - 'cluster:admin/opendistro/alerting/monitor/execute' + - 'cluster:admin/opendistro/alerting/alerts/get' + - 'cluster:admin/opendistro/alerting/alerts/ack' + - 'cluster:admin/opendistro/alerting/destination/get' + - 'cluster:admin/opendistro/alerting/destination/write' + - 'cluster:admin/opendistro/alerting/destination/delete' # Notifications plugin (OpenSearch 2.x): channel features + config CRUD - - "cluster:admin/opensearch/notifications/features" - - "cluster:admin/opensearch/notifications/configs/get" - - "cluster:admin/opensearch/notifications/configs/create" - - "cluster:admin/opensearch/notifications/configs/update" - - "cluster:admin/opensearch/notifications/configs/delete" + - 'cluster:admin/opensearch/notifications/features' + - 'cluster:admin/opensearch/notifications/configs/get' + - 'cluster:admin/opensearch/notifications/configs/create' + - 'cluster:admin/opensearch/notifications/configs/update' + - 'cluster:admin/opensearch/notifications/configs/delete' index_permissions: - index_patterns: - - "CUSTOMER_ID_PLACEHOLDER-*" + - 'CUSTOMER_ID_PLACEHOLDER-*' allowed_actions: - - "read" - - "write" - - "create_index" - - "indices:data/read/*" - - "indices:data/write/*" - - "indices:admin/mapping/put" + - 'read' + - 'write' + - 'create_index' + - 'indices:data/read/*' + - 'indices:data/write/*' + - 'indices:admin/mapping/put' - index_patterns: - - ".kibana*" + - '.kibana*' allowed_actions: - - "read" - - "write" - - "create_index" - - "indices:data/read/*" - - "indices:data/write/*" - - "indices:admin/mapping/put" + - 'read' + - 'write' + - 'create_index' + - 'indices:data/read/*' + - 'indices:data/write/*' + - 'indices:admin/mapping/put' tenant_permissions: - tenant_patterns: - - "CUSTOMER_ID_PLACEHOLDER" + - 'CUSTOMER_ID_PLACEHOLDER' allowed_actions: - - "kibana_all_write" + - 'kibana_all_write' # Template: Customer read-only access (for viewers) # Actual role name: customer_{customer_id}_ro # Index pattern: {customer_id}-* customer_template_ro: reserved: false - description: "Template for customer read-only roles - DO NOT USE DIRECTLY" + description: 'Template for customer read-only roles - DO NOT USE DIRECTLY' cluster_permissions: - - "cluster_composite_ops_ro" + - 'cluster_composite_ops_ro' # Required for Dashboards saved objects (bulk writes to .kibana_* tenant indices) - - "indices:data/write/bulk" + - 'indices:data/write/bulk' # Alerting: monitor CRUD, execution, alerts, and destinations (legacy endpoints) - - "cluster:admin/opendistro/alerting/monitor/get" - - "cluster:admin/opendistro/alerting/monitor/search" - - "cluster:admin/opendistro/alerting/monitor/write" - - "cluster:admin/opendistro/alerting/monitor/execute" - - "cluster:admin/opendistro/alerting/alerts/get" - - "cluster:admin/opendistro/alerting/alerts/ack" - - "cluster:admin/opendistro/alerting/destination/get" - - "cluster:admin/opendistro/alerting/destination/write" - - "cluster:admin/opendistro/alerting/destination/delete" + - 'cluster:admin/opendistro/alerting/monitor/get' + - 'cluster:admin/opendistro/alerting/monitor/search' + - 'cluster:admin/opendistro/alerting/monitor/write' + - 'cluster:admin/opendistro/alerting/monitor/execute' + - 'cluster:admin/opendistro/alerting/alerts/get' + - 'cluster:admin/opendistro/alerting/alerts/ack' + - 'cluster:admin/opendistro/alerting/destination/get' + - 'cluster:admin/opendistro/alerting/destination/write' + - 'cluster:admin/opendistro/alerting/destination/delete' # Notifications plugin (OpenSearch 2.x): channel features + config CRUD - - "cluster:admin/opensearch/notifications/features" - - "cluster:admin/opensearch/notifications/configs/get" - - "cluster:admin/opensearch/notifications/configs/create" - - "cluster:admin/opensearch/notifications/configs/update" - - "cluster:admin/opensearch/notifications/configs/delete" + - 'cluster:admin/opensearch/notifications/features' + - 'cluster:admin/opensearch/notifications/configs/get' + - 'cluster:admin/opensearch/notifications/configs/create' + - 'cluster:admin/opensearch/notifications/configs/update' + - 'cluster:admin/opensearch/notifications/configs/delete' index_permissions: - index_patterns: - - "CUSTOMER_ID_PLACEHOLDER-*" + - 'CUSTOMER_ID_PLACEHOLDER-*' allowed_actions: - - "read" - - "indices:data/read/*" + - 'read' + - 'indices:data/read/*' - index_patterns: - - ".kibana*" + - '.kibana*' allowed_actions: - - "read" - - "write" - - "create_index" - - "indices:data/read/*" - - "indices:data/write/*" - - "indices:admin/mapping/put" + - 'read' + - 'write' + - 'create_index' + - 'indices:data/read/*' + - 'indices:data/write/*' + - 'indices:admin/mapping/put' tenant_permissions: - tenant_patterns: - - "CUSTOMER_ID_PLACEHOLDER" + - 'CUSTOMER_ID_PLACEHOLDER' allowed_actions: - - "kibana_all_write" + - 'kibana_all_write' # ============================================================================= # DASHBOARDS INTERNAL ROLES diff --git a/docker/opensearch-security/roles_mapping.yml b/docker/opensearch-security/roles_mapping.yml index 69636259d..7b5f0ac2a 100644 --- a/docker/opensearch-security/roles_mapping.yml +++ b/docker/opensearch-security/roles_mapping.yml @@ -18,7 +18,7 @@ --- _meta: - type: "rolesmapping" + type: 'rolesmapping' config_version: 2 # ============================================================================= @@ -29,35 +29,35 @@ _meta: platform_admin: reserved: true users: - - "admin" + - 'admin' backend_roles: - - "platform_admin" - description: "Platform administrators with full system access" + - 'platform_admin' + description: 'Platform administrators with full system access' # Dashboards server mapping kibana_server: reserved: true users: - - "kibanaserver" - description: "OpenSearch Dashboards server user" + - 'kibanaserver' + description: 'OpenSearch Dashboards server user' # Security REST API access - for admin operations security_rest_api_access: reserved: true users: - - "admin" + - 'admin' backend_roles: - - "platform_admin" - description: "Access to Security REST API for tenant/role management" + - 'platform_admin' + description: 'Access to Security REST API for tenant/role management' # Worker service mapping - for indexing security findings worker_write: reserved: false users: - - "worker" + - 'worker' backend_roles: - - "worker_write" - description: "Worker service for indexing security findings" + - 'worker_write' + description: 'Worker service for indexing security findings' # ============================================================================= # CUSTOMER ROLE MAPPINGS diff --git a/docker/opensearch-security/tenants.yml b/docker/opensearch-security/tenants.yml index ae9d03937..3819ae9c8 100644 --- a/docker/opensearch-security/tenants.yml +++ b/docker/opensearch-security/tenants.yml @@ -16,7 +16,7 @@ --- _meta: - type: "tenants" + type: 'tenants' config_version: 2 # NOTE: Customer tenants are created dynamically by the application backend @@ -25,4 +25,4 @@ _meta: # Admin tenant - for platform operators only (not customers) __platform_admin: reserved: true - description: "Platform administration - internal use only" + description: 'Platform administration - internal use only' diff --git a/docker/opensearch.dev-secure.yml b/docker/opensearch.dev-secure.yml index f9f0494ac..123af29cc 100644 --- a/docker/opensearch.dev-secure.yml +++ b/docker/opensearch.dev-secure.yml @@ -25,11 +25,11 @@ plugins.security.allow_default_init_securityindex: true # Admin DN - Required for securityadmin.sh and REST API access plugins.security.authcz.admin_dn: - - "CN=admin,OU=ShipSec,O=ShipSecAI,L=SF,ST=CA,C=US" + - 'CN=admin,OU=ShipSec,O=ShipSecAI,L=SF,ST=CA,C=US' plugins.security.audit.type: internal_opensearch plugins.security.enable_snapshot_restore_privilege: true plugins.security.check_snapshot_restore_write_privileges: true plugins.security.restapi.roles_enabled: - - "all_access" - - "security_rest_api_access" + - 'all_access' + - 'security_rest_api_access' diff --git a/docs/docs.json b/docs/docs.json index 046e40145..49e6bc91f 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -15,18 +15,11 @@ "groups": [ { "group": "Getting Started", - "pages": [ - "index", - "quickstart", - "installation", - "command-reference" - ] + "pages": ["index", "quickstart", "installation", "command-reference"] }, { "group": "Architecture", - "pages": [ - "architecture" - ] + "pages": ["architecture"] }, { "group": "Components", diff --git a/docs/workflows/execution-status.md b/docs/workflows/execution-status.md index 1ac61aba3..1f0b60d8b 100644 --- a/docs/workflows/execution-status.md +++ b/docs/workflows/execution-status.md @@ -4,17 +4,17 @@ This document describes the different execution statuses a workflow run can have ## Status Overview -| Status | Color | Description | -|--------|-------|-------------| -| `QUEUED` | Blue | Workflow is waiting to be executed | -| `RUNNING` | Blue | Workflow is actively executing | -| `COMPLETED` | Green | Workflow finished successfully - all nodes completed | -| `FAILED` | Red | Workflow failed - at least one node failed or workflow crashed | -| `CANCELLED` | Gray | Workflow was cancelled by user | -| `TERMINATED` | Gray | Workflow was forcefully terminated | -| `TIMED_OUT` | Amber | Workflow exceeded maximum execution time | -| `AWAITING_INPUT` | Purple | Workflow is paused waiting for human input | -| `STALE` | Amber | Orphaned record - data inconsistency (see below) | +| Status | Color | Description | +| ---------------- | ------ | -------------------------------------------------------------- | +| `QUEUED` | Blue | Workflow is waiting to be executed | +| `RUNNING` | Blue | Workflow is actively executing | +| `COMPLETED` | Green | Workflow finished successfully - all nodes completed | +| `FAILED` | Red | Workflow failed - at least one node failed or workflow crashed | +| `CANCELLED` | Gray | Workflow was cancelled by user | +| `TERMINATED` | Gray | Workflow was forcefully terminated | +| `TIMED_OUT` | Amber | Workflow exceeded maximum execution time | +| `AWAITING_INPUT` | Purple | Workflow is paused waiting for human input | +| `STALE` | Amber | Orphaned record - data inconsistency (see below) | ## Status Transitions @@ -30,50 +30,64 @@ QUEUED → RUNNING → COMPLETED ## Detailed Status Descriptions ### QUEUED + The workflow run has been created and is waiting to start execution. This is the initial state before the Temporal worker picks up the workflow. ### RUNNING + The workflow is actively executing. At least one node has started processing. ### COMPLETED + All nodes in the workflow have finished successfully. This is a terminal state. **Conditions:** + - All expected nodes have `COMPLETED` trace events - No `FAILED` trace events ### FAILED + The workflow encountered an error during execution. This is a terminal state. **Conditions:** + - At least one node has a `FAILED` trace event, OR - Some nodes started but not all completed (workflow crashed/lost) ### CANCELLED + The user manually cancelled the workflow execution. This is a terminal state. ### TERMINATED + The workflow was forcefully terminated (e.g., via Temporal API). This is a terminal state. ### TIMED_OUT + The workflow exceeded its maximum allowed execution time. This is a terminal state. ### AWAITING_INPUT + The workflow has reached a human input node and is waiting for user interaction. The workflow will resume to `RUNNING` when input is provided. ### STALE + **Special Status - Data Inconsistency Warning** The run record exists in the database but there's no evidence it ever executed: + - No trace events in the database - Temporal has no record of this workflow **Common Causes:** + 1. **Fresh Temporal instance with old database** - The Temporal server was reset/reinstalled but the application database retained old run records 2. **Failed workflow start** - The backend created a run record but the Temporal workflow failed to start (network error, Temporal unavailable, etc.) 3. **Data migration issues** - Database was migrated without corresponding Temporal data **Recommended Action:** + - Review these records and delete them if they represent stale data - Investigate why the data inconsistency occurred to prevent future occurrences diff --git a/frontend/src/services/api.ts b/frontend/src/services/api.ts index 03d26798e..54b9fec04 100644 --- a/frontend/src/services/api.ts +++ b/frontend/src/services/api.ts @@ -97,7 +97,10 @@ function resolveApiBaseUrl() { } } - return 'http://localhost:3211'; + // No explicit API URL — use same-origin relative paths. + // Works with path-based routing (/api/v1/* routed to backend via Ingress). + // For local dev, set VITE_API_URL=http://localhost:3211 in frontend/.env + return ''; } export const API_BASE_URL = resolveApiBaseUrl(); @@ -604,7 +607,8 @@ export const api = { }, ): Promise => { const headers = await getAuthHeaders(); - const url = new URL(`${API_V1_URL}/workflows/runs/${executionId}/terminal`); + const path = `${API_V1_URL}/workflows/runs/${executionId}/terminal`; + const url = new URL(path, window.location.origin); if (params?.nodeRef) url.searchParams.set('nodeRef', params.nodeRef); if (params?.stream) url.searchParams.set('stream', params.stream); if (params?.cursor) url.searchParams.set('cursor', params.cursor); diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index 65a65eec0..21bfb5e9f 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -28,7 +28,7 @@ export default defineConfig({ port: frontendPort, strictPort: true, open: false, - allowedHosts: ['studio.shipsec.ai', 'frontend'], + allowedHosts: ['studio.shipsec.ai', 'studio-next.shipsec.ai', 'frontend'], proxy: { '/api/': { target: `http://localhost:${backendPort}`, @@ -43,6 +43,6 @@ export default defineConfig({ }, }, preview: { - allowedHosts: ['studio.shipsec.ai', 'frontend'], + allowedHosts: ['studio.shipsec.ai', 'studio-next.shipsec.ai', 'frontend'], }, }); diff --git a/infra/gcp/APPLY.md b/infra/gcp/APPLY.md new file mode 100644 index 000000000..99e0f21e8 --- /dev/null +++ b/infra/gcp/APPLY.md @@ -0,0 +1,111 @@ +# Apply Guide (Terraform) + +This repo uses `terraform` locally. (OpenTofu works too, but is not assumed to be installed.) + +## 0) Auth (required) + +Terraform's GCP provider uses Application Default Credentials (ADC) by default. + +```bash +gcloud auth login +gcloud auth application-default login +gcloud config set project shipsec +gcloud config set compute/region us-central1 +gcloud config set compute/zone us-central1-a +``` + +Verify: + +```bash +gcloud auth application-default print-access-token >/dev/null && echo adc:present +``` + +### Non-interactive fallback (recommended in CI) + +If you can't use ADC (for example in headless sessions), you can use a short-lived token: + +```bash +export TF_VAR_access_token="$(gcloud auth print-access-token)" +``` + +## 1) Bootstrap Terraform state bucket (run once) + +Pick a globally unique bucket name, then: + +```bash +cd infra/gcp/bootstrap +terraform init +terraform apply \ + -var project_id=shipsec \ + -var region=us-central1 \ + -var state_bucket_name=shipsec-tfstate +``` + +## 2) Dev environment (fast) + +```bash +cd infra/gcp/envs/dev +terraform init \ + -backend-config="bucket=shipsec-tfstate" \ + -backend-config="prefix=infra/gcp/dev" + +terraform apply \ + -var project_id=shipsec \ + -var region=us-central1 \ + -var zone=us-central1-a \ + -var cluster_name=shipsec-dev +``` + +### Dev environment (no remote state; works without ADC) + +If you don't have ADC configured but want to apply once, use `envs/dev-local`: + +```bash +export TF_VAR_access_token="$(gcloud auth print-access-token)" +cd infra/gcp/envs/dev-local +terraform init +terraform apply \ + -var project_id=shipsec \ + -var region=us-central1 \ + -var zone=us-central1-a \ + -var cluster_name=shipsec-dev-tf \ + -var node_count=1 +``` + +Get credentials: + +```bash +gcloud container clusters get-credentials shipsec-dev --zone us-central1-a --project shipsec +kubectl get nodes +``` + +## 3) Prod environment (baseline) + +`prod` creates a regional cluster with private nodes and Cloud NAT, plus separate node pools: + +- `system-pool`: backend/worker/control plane pods +- `exec-pool`: execution workloads (tainted `shipsec.io/exec=true:NoSchedule`) + +```bash +cd infra/gcp/envs/prod +terraform init \ + -backend-config="bucket=shipsec-tfstate" \ + -backend-config="prefix=infra/gcp/prod" + +terraform apply \ + -var project_id=shipsec \ + -var region=us-central1 \ + -var cluster_name=shipsec-prod +``` + +Then fetch credentials: + +```bash +gcloud container clusters get-credentials shipsec-prod --region us-central1 --project shipsec +kubectl get nodes +``` + +## Notes + +- If your org policies require it, add a project `environment` tag. It's not required for GKE itself. +- This file intentionally does not include any credentials, service account keys, or secrets. diff --git a/infra/gcp/README.md b/infra/gcp/README.md new file mode 100644 index 000000000..1a803b866 --- /dev/null +++ b/infra/gcp/README.md @@ -0,0 +1,55 @@ +# GCP Infra (Terraform/OpenTofu) + +This directory is intended for the **private** repo only. + +Goals: + +- Provision GCP infrastructure (network, GKE, Artifact Registry) with sane defaults. +- Keep app deployment (Helm) separate from infrastructure provisioning. +- Support a fast `dev` environment and a safer `prod` environment. + +## Layout + +- `infra/gcp/bootstrap/`: creates a GCS bucket for Terraform state (run once per project). +- `infra/gcp/envs/dev/`: fast dev cluster (zonal, public nodes by default). +- `infra/gcp/envs/prod/`: production-ready baseline (regional, private nodes, Cloud NAT, node pool split). + +## Prereqs + +- `gcloud` authenticated to the right project +- Application Default Credentials for Terraform/OpenTofu: + +```bash +gcloud auth application-default login +gcloud config set project shipsec +``` + +## Quickstart (recommended) + +1. Bootstrap state bucket: + +```bash +cd infra/gcp/bootstrap +terraform init +terraform apply -var project_id=shipsec -var region=us-central1 +``` + +2. Create `dev` cluster: + +```bash +cd infra/gcp/envs/dev +terraform init -backend-config="bucket=shipsec-tfstate" -backend-config="prefix=infra/gcp/dev" +terraform apply -var project_id=shipsec -var region=us-central1 -var zone=us-central1-a +``` + +3. Fetch kube credentials: + +```bash +gcloud container clusters get-credentials shipsec-dev --zone us-central1-a --project shipsec +kubectl get nodes +``` + +## Notes + +- `prod` uses private nodes and Cloud NAT by default. That is closer to real production, but costs more. +- Artifact Registry is created in the chosen region for pushing images. diff --git a/infra/gcp/bootstrap/.terraform.lock.hcl b/infra/gcp/bootstrap/.terraform.lock.hcl new file mode 100644 index 000000000..18f208087 --- /dev/null +++ b/infra/gcp/bootstrap/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "7.18.0" + constraints = ">= 5.20.0" + hashes = [ + "h1:Hqg6g5/5hFRK73xBE7ANAeuQbuw8ibuPrzXP7OOPxrk=", + "zh:041dc216f7352e36af65d4a6d4a38d24fec4c05193a4f4c8cf69138e29dc9421", + "zh:454c675e0487f011764eb0cd15d7b1e43d06a4e80ed056aeb4ad11df31368f81", + "zh:4e76c8a1e5645f1e2c258c8074d4e9ecfc1d6383d207d03f492df16da389a120", + "zh:60c96075fc082d9584b9cb8f48f0d23f90fd4344e6141a417580c6bad1b21957", + "zh:ad82cece07a0816153e3fc6cb6d7672c6c009742dc802ab434a83d0731d94ae7", + "zh:aebbf8a0bd3af0b6c705d5d85ec51891f533b83dcbae7249e64a252efc6fd862", + "zh:bfbb19a5b46950eaf0a83cea09a5992d1b0e96792130faeb6c733609dc2913df", + "zh:c196b4c82d0252fa751ee3cd84433bc483b7ce7d6fcab5db0413dbaa9f218650", + "zh:db1c83777bc6d7fc195be83712a9f503e9a5a1f7326fd6968d9812acc53f2056", + "zh:dcd58beeac9d1889e5532cfcd3bd8dec5568ab06b0a81427bc9b35931b6f0178", + "zh:eaeedc86c2d01630a3166ae98d5138e9bf9463b2a606d7f7df6d465d1501f28f", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/infra/gcp/bootstrap/main.tf b/infra/gcp/bootstrap/main.tf new file mode 100644 index 000000000..c947d8840 --- /dev/null +++ b/infra/gcp/bootstrap/main.tf @@ -0,0 +1,24 @@ +resource "google_storage_bucket" "tfstate" { + name = var.state_bucket_name + location = var.region + uniform_bucket_level_access = true + force_destroy = false + + versioning { + enabled = true + } + + lifecycle_rule { + condition { + num_newer_versions = 20 + } + action { + type = "Delete" + } + } +} + +output "state_bucket_name" { + value = google_storage_bucket.tfstate.name +} + diff --git a/infra/gcp/bootstrap/variables.tf b/infra/gcp/bootstrap/variables.tf new file mode 100644 index 000000000..af3048015 --- /dev/null +++ b/infra/gcp/bootstrap/variables.tf @@ -0,0 +1,22 @@ +variable "project_id" { + type = string + description = "GCP project id (e.g. shipsec)." +} + +variable "region" { + type = string + description = "GCP region (e.g. us-central1)." +} + +variable "access_token" { + type = string + description = "Optional short-lived OAuth access token (bypasses ADC)." + default = null + sensitive = true +} + +variable "state_bucket_name" { + type = string + description = "Globally unique GCS bucket name for Terraform state." + default = "shipsec-tfstate" +} diff --git a/infra/gcp/bootstrap/versions.tf b/infra/gcp/bootstrap/versions.tf new file mode 100644 index 000000000..adaf6f7db --- /dev/null +++ b/infra/gcp/bootstrap/versions.tf @@ -0,0 +1,16 @@ +terraform { + required_version = ">= 1.5.0" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 5.20.0" + } + } +} + +provider "google" { + project = var.project_id + region = var.region + access_token = var.access_token +} diff --git a/infra/gcp/envs/dev/.terraform.lock.hcl b/infra/gcp/envs/dev/.terraform.lock.hcl new file mode 100644 index 000000000..cf2b1697b --- /dev/null +++ b/infra/gcp/envs/dev/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "7.19.0" + constraints = ">= 5.20.0" + hashes = [ + "h1:fsiBePQ2WgTqyORF6Klc/GDgV8JHUTMJALf6V4xJU3A=", + "zh:06da157d858384b2383414447c1bf6cf319ad72ea87d7030c6ca18b9bb774f73", + "zh:2f1d7c3461a6b59ffcf0eed2f3764e2f0a2c70464927e561d968d82112e3600d", + "zh:4705ce487e6b2c52376e1f9bc0dc650e8326ab3e20d0673c9fed62e1313d2d67", + "zh:5cd9a4ee36d3d7ffbabb90c83cb7cce54cf0f10c912db4be7492ebc1a78611b3", + "zh:688622dbac98fe95115518ff3d9324cf71ffdf124ca6e66b2269f43d9f8e7ceb", + "zh:7a5c07ae0728c7a57a63d848411c91550fd3bfe662f60821b50d3370be360134", + "zh:8a6472dec8082d7225a811c8ee0bf550c7a9c36e86cfd19b10363106f2dfbb80", + "zh:8e11d4c27e70500aaa1335cb721ad64c4b0e41b3c7398d6fe58a3d92f10ea213", + "zh:9a119c27e27bad73cdd8c0544f8a68a84bdac3de0129f13a87a6890ed19c6035", + "zh:dd12460d2b8b4497b5a7c46bb486ace9859d2fc642782989df315e618596d1e4", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:fc35c660777b377978e5f2d008db6181ff2f98777cdd215effc11d665e99e0bc", + ] +} diff --git a/infra/gcp/envs/dev/main.tf b/infra/gcp/envs/dev/main.tf new file mode 100644 index 000000000..c89db6ce9 --- /dev/null +++ b/infra/gcp/envs/dev/main.tf @@ -0,0 +1,376 @@ +# -------------------------------------------------------------------------- +# Adopt the existing shipsec-dev GKE cluster into Terraform. +# The cluster was created imperatively on the default VPC, so we reference +# the network/subnet as data sources rather than managing them. +# -------------------------------------------------------------------------- + +locals { + services = toset([ + "cloudresourcemanager.googleapis.com", + "serviceusage.googleapis.com", + "iam.googleapis.com", + "compute.googleapis.com", + "container.googleapis.com", + "artifactregistry.googleapis.com", + "secretmanager.googleapis.com", + "sqladmin.googleapis.com", + "redis.googleapis.com", + "servicenetworking.googleapis.com", + ]) +} + +resource "google_project_service" "enabled" { + for_each = local.services + project = var.project_id + service = each.value + + disable_on_destroy = false +} + +resource "google_artifact_registry_repository" "docker" { + project = var.project_id + location = var.region + repository_id = var.artifact_repo_name + format = "DOCKER" + + depends_on = [google_project_service.enabled] +} + +# The cluster lives on the default VPC — we don't manage it, just reference it. +data "google_compute_network" "default" { + project = var.project_id + name = "default" +} + +data "google_compute_subnetwork" "default" { + project = var.project_id + region = var.region + name = "default" +} + +resource "google_container_cluster" "gke" { + project = var.project_id + name = var.cluster_name + location = var.zone + + deletion_protection = false + initial_node_count = 1 + + release_channel { + channel = "REGULAR" + } + + network = data.google_compute_network.default.id + subnetwork = data.google_compute_subnetwork.default.id + + ip_allocation_policy { + cluster_secondary_range_name = "gke-shipsec-dev-pods-0a61f82c" + } + + addons_config { + gcs_fuse_csi_driver_config { + enabled = true + } + } + + workload_identity_config { + workload_pool = "${var.project_id}.svc.id.goog" + } + + # initial_node_count drifts to 0 after remove_default_node_pool removes it. + # node_config/node_pool are managed by the separate google_container_node_pool resource. + lifecycle { + ignore_changes = [initial_node_count, node_config, node_pool] + } + + depends_on = [google_project_service.enabled] +} + +resource "google_container_node_pool" "default_pool" { + project = var.project_id + name = "default-pool" + cluster = google_container_cluster.gke.name + location = var.zone + + initial_node_count = var.node_count + + node_config { + machine_type = var.node_machine_type + disk_type = "pd-balanced" + disk_size_gb = var.node_disk_gb + image_type = "COS_CONTAINERD" + + oauth_scopes = [ + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + "https://www.googleapis.com/auth/service.management.readonly", + "https://www.googleapis.com/auth/servicecontrol", + "https://www.googleapis.com/auth/trace.append", + ] + } +} + +# ========================================================================== +# Managed Services: Cloud SQL, Memorystore, GCS +# ========================================================================== + +# Private Service Access — allows Cloud SQL and Memorystore to get private IPs +# on the default VPC so GKE pods can reach them without public IPs. +resource "google_compute_global_address" "private_ip_range" { + project = var.project_id + name = "shipsec-private-ip-range" + purpose = "VPC_PEERING" + address_type = "INTERNAL" + prefix_length = 20 + network = data.google_compute_network.default.id + + depends_on = [google_project_service.enabled] +} + +resource "google_service_networking_connection" "private_vpc" { + network = data.google_compute_network.default.id + service = "servicenetworking.googleapis.com" + reserved_peering_ranges = [google_compute_global_address.private_ip_range.name] + + depends_on = [google_project_service.enabled] +} + +# --- Cloud SQL (PostgreSQL 16) --- +resource "google_sql_database_instance" "postgres" { + project = var.project_id + name = "${var.cluster_name}-pg" + region = var.region + database_version = "POSTGRES_16" + + deletion_protection = false + + settings { + tier = var.cloudsql_tier + edition = "ENTERPRISE" + availability_type = "ZONAL" + disk_size = 10 + disk_type = "PD_SSD" + disk_autoresize = true + + ip_configuration { + ipv4_enabled = false + private_network = data.google_compute_network.default.id + enable_private_path_for_google_cloud_services = true + } + + backup_configuration { + enabled = true + start_time = "03:00" + point_in_time_recovery_enabled = true + transaction_log_retention_days = 7 + backup_retention_settings { + retained_backups = 7 + } + } + } + + depends_on = [google_service_networking_connection.private_vpc] +} + +resource "google_sql_database" "shipsec" { + project = var.project_id + instance = google_sql_database_instance.postgres.name + name = "shipsec" +} + +resource "google_sql_database" "temporal" { + project = var.project_id + instance = google_sql_database_instance.postgres.name + name = "temporal" +} + +resource "google_sql_user" "shipsec" { + project = var.project_id + instance = google_sql_database_instance.postgres.name + name = "shipsec" + password = var.db_password +} + +# --- Memorystore (Redis) --- +resource "google_redis_instance" "redis" { + project = var.project_id + name = "${var.cluster_name}-redis" + region = var.region + tier = "BASIC" + memory_size_gb = var.redis_memory_gb + + authorized_network = data.google_compute_network.default.id + connect_mode = "PRIVATE_SERVICE_ACCESS" + + redis_version = "REDIS_7_2" + + depends_on = [google_service_networking_connection.private_vpc] +} + +# --- GCS (replaces MinIO for artifact/file storage) --- +resource "google_storage_bucket" "artifacts" { + project = var.project_id + name = "${var.project_id}-artifacts-${var.cluster_name}" + location = var.region + force_destroy = true + + uniform_bucket_level_access = true + + versioning { + enabled = false + } + + lifecycle_rule { + condition { + age = 90 + } + action { + type = "Delete" + } + } +} + +# Service account for GCS access via Workload Identity +resource "google_service_account" "storage" { + project = var.project_id + account_id = "${var.cluster_name}-storage" + display_name = "Storage SA for ${var.cluster_name}" +} + +resource "google_storage_bucket_iam_member" "storage_admin" { + bucket = google_storage_bucket.artifacts.name + role = "roles/storage.objectAdmin" + member = "serviceAccount:${google_service_account.storage.email}" +} + +# Workload Identity binding: allow K8s SA "storage" in shipsec-system +# namespace to impersonate this GCP SA. +resource "google_service_account_iam_member" "storage_wi" { + service_account_id = google_service_account.storage.name + role = "roles/iam.workloadIdentityUser" + member = "serviceAccount:${var.project_id}.svc.id.goog[shipsec-system/storage]" +} + +# ========================================================================== +# GCS FUSE Volume Support (for K8s job pods) +# ========================================================================== + +# GCS bucket for job volumes (auto-cleanup after 7 days) +resource "google_storage_bucket" "volumes" { + project = var.project_id + name = "${var.project_id}-volumes-${var.cluster_name}" + location = var.region + uniform_bucket_level_access = true + force_destroy = true + + lifecycle_rule { + condition { + age = 7 + } + action { + type = "Delete" + } + } +} + +# GCP SA for the worker pod (Workload Identity → GCS SDK access) +resource "google_service_account" "worker" { + project = var.project_id + account_id = "shipsec-worker" + display_name = "ShipSec Worker" +} + +# Workload Identity: shipsec-workers/shipsec-worker KSA → shipsec-worker GCP SA +resource "google_service_account_iam_member" "worker_wi" { + service_account_id = google_service_account.worker.name + role = "roles/iam.workloadIdentityUser" + member = "serviceAccount:${var.project_id}.svc.id.goog[shipsec-workers/shipsec-worker]" +} + +# Worker SA → volumes bucket access (reads inputs, reads outputs via SDK) +resource "google_storage_bucket_iam_member" "worker_volumes" { + bucket = google_storage_bucket.volumes.name + role = "roles/storage.objectUser" + member = "serviceAccount:${google_service_account.worker.email}" +} + +# GCP SA for job pods (mounted via GCS FUSE CSI) +resource "google_service_account" "job_runner" { + project = var.project_id + account_id = "shipsec-job-runner" + display_name = "ShipSec K8s Job Runner" +} + +# Job runner SA → volumes bucket access +resource "google_storage_bucket_iam_member" "job_runner_volumes" { + bucket = google_storage_bucket.volumes.name + role = "roles/storage.objectUser" + member = "serviceAccount:${google_service_account.job_runner.email}" +} + +# Workload Identity: K8s SA → GCP SA (for job pods in shipsec-workloads) +resource "google_service_account_iam_member" "job_runner_wi" { + service_account_id = google_service_account.job_runner.name + role = "roles/iam.workloadIdentityUser" + member = "serviceAccount:${var.project_id}.svc.id.goog[shipsec-workloads/shipsec-job-runner]" +} + +# Existing storage SA also needs access to volumes bucket (worker reads/writes via SDK) +resource "google_storage_bucket_iam_member" "storage_volumes" { + bucket = google_storage_bucket.volumes.name + role = "roles/storage.objectUser" + member = "serviceAccount:${google_service_account.storage.email}" +} + +# ========================================================================== +# Outputs +# ========================================================================== + +output "artifact_registry_repo" { + value = "${var.region}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.docker.repository_id}" +} + +output "cluster_location" { + value = var.zone +} + +output "cluster_name" { + value = google_container_cluster.gke.name +} + +# Cloud SQL +output "database_url" { + value = "postgresql://${google_sql_user.shipsec.name}:${var.db_password}@${google_sql_database_instance.postgres.private_ip_address}:5432/shipsec" + sensitive = true +} + +output "cloudsql_private_ip" { + value = google_sql_database_instance.postgres.private_ip_address +} + +# Memorystore +output "redis_url" { + value = "redis://${google_redis_instance.redis.host}:${google_redis_instance.redis.port}" +} + +# GCS (via Workload Identity) +output "gcs_bucket" { + value = google_storage_bucket.artifacts.name +} + +output "gcs_storage_sa_email" { + value = google_service_account.storage.email +} + +output "gcs_volumes_bucket" { + value = google_storage_bucket.volumes.name +} + +output "job_runner_sa_email" { + value = google_service_account.job_runner.email +} + +output "worker_sa_email" { + value = google_service_account.worker.email +} diff --git a/infra/gcp/envs/dev/variables.tf b/infra/gcp/envs/dev/variables.tf new file mode 100644 index 000000000..d4217b435 --- /dev/null +++ b/infra/gcp/envs/dev/variables.tf @@ -0,0 +1,73 @@ +variable "project_id" { + type = string + description = "GCP project id (e.g. shipsec)." +} + +variable "access_token" { + type = string + description = "Optional short-lived OAuth access token (bypasses ADC)." + default = null + sensitive = true +} + +variable "region" { + type = string + description = "GCP region (e.g. us-central1)." + default = "us-central1" +} + +variable "zone" { + type = string + description = "GCP zone for a zonal dev cluster (e.g. us-central1-a)." + default = "us-central1-a" +} + +variable "cluster_name" { + type = string + description = "GKE cluster name." + default = "shipsec-dev" +} + +variable "artifact_repo_name" { + type = string + description = "Artifact Registry repo name (Docker)." + default = "shipsec-studio" +} + +variable "node_machine_type" { + type = string + description = "Machine type for dev nodes." + default = "e2-standard-4" +} + +variable "node_count" { + type = number + description = "Initial node count for the dev node pool." + default = 2 +} + +variable "node_disk_gb" { + type = number + description = "Boot disk size (GB)." + default = 100 +} + +# --- Managed services --- + +variable "cloudsql_tier" { + type = string + description = "Cloud SQL machine tier." + default = "db-custom-1-3840" +} + +variable "db_password" { + type = string + description = "Password for the shipsec Cloud SQL user." + sensitive = true +} + +variable "redis_memory_gb" { + type = number + description = "Memorystore Redis memory in GB." + default = 1 +} diff --git a/infra/gcp/envs/dev/versions.tf b/infra/gcp/envs/dev/versions.tf new file mode 100644 index 000000000..3ca969c46 --- /dev/null +++ b/infra/gcp/envs/dev/versions.tf @@ -0,0 +1,19 @@ +terraform { + required_version = ">= 1.5.0" + + backend "gcs" {} + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 5.20.0" + } + } +} + +provider "google" { + project = var.project_id + region = var.region + zone = var.zone + access_token = var.access_token +} diff --git a/infra/gcp/envs/prod/main.tf b/infra/gcp/envs/prod/main.tf new file mode 100644 index 000000000..01237a2ce --- /dev/null +++ b/infra/gcp/envs/prod/main.tf @@ -0,0 +1,194 @@ +locals { + services = toset([ + "cloudresourcemanager.googleapis.com", + "serviceusage.googleapis.com", + "iam.googleapis.com", + "container.googleapis.com", + "artifactregistry.googleapis.com", + "secretmanager.googleapis.com", + "compute.googleapis.com", + ]) +} + +resource "google_project_service" "enabled" { + for_each = local.services + project = var.project_id + service = each.value + disable_on_destroy = false +} + +resource "google_artifact_registry_repository" "docker" { + project = var.project_id + location = var.region + repository_id = var.artifact_repo_name + format = "DOCKER" + + depends_on = [google_project_service.enabled] +} + +resource "google_compute_network" "vpc" { + project = var.project_id + name = "${var.cluster_name}-vpc" + auto_create_subnetworks = false + + depends_on = [google_project_service.enabled] +} + +resource "google_compute_subnetwork" "subnet" { + project = var.project_id + region = var.region + name = "${var.cluster_name}-subnet" + network = google_compute_network.vpc.id + ip_cidr_range = "10.110.0.0/16" + + private_ip_google_access = true + + secondary_ip_range { + range_name = "pods" + ip_cidr_range = "10.120.0.0/16" + } + + secondary_ip_range { + range_name = "services" + ip_cidr_range = "10.130.0.0/20" + } +} + +resource "google_compute_router" "router" { + project = var.project_id + region = var.region + name = "${var.cluster_name}-router" + network = google_compute_network.vpc.id +} + +resource "google_compute_router_nat" "nat" { + project = var.project_id + region = var.region + name = "${var.cluster_name}-nat" + router = google_compute_router.router.name + + nat_ip_allocate_option = "AUTO_ONLY" + source_subnetwork_ip_ranges_to_nat = "LIST_OF_SUBNETWORKS" + + subnetwork { + name = google_compute_subnetwork.subnet.id + source_ip_ranges_to_nat = ["ALL_IP_RANGES"] + } + + log_config { + enable = true + filter = "ERRORS_ONLY" + } +} + +resource "google_container_cluster" "gke" { + project = var.project_id + name = var.cluster_name + location = var.region + + deletion_protection = true + remove_default_node_pool = true + initial_node_count = 1 + + release_channel { + channel = "REGULAR" + } + + network = google_compute_network.vpc.id + subnetwork = google_compute_subnetwork.subnet.id + + ip_allocation_policy { + cluster_secondary_range_name = "pods" + services_secondary_range_name = "services" + } + + workload_identity_config { + workload_pool = "${var.project_id}.svc.id.goog" + } + + private_cluster_config { + enable_private_nodes = true + enable_private_endpoint = false + master_ipv4_cidr_block = "172.16.0.0/28" + } + + dynamic "master_authorized_networks_config" { + for_each = length(var.master_authorized_cidrs) > 0 ? [1] : [] + content { + dynamic "cidr_blocks" { + for_each = var.master_authorized_cidrs + content { + cidr_block = cidr_blocks.value.cidr_block + display_name = cidr_blocks.value.display_name + } + } + } + } + + depends_on = [google_project_service.enabled] +} + +resource "google_container_node_pool" "system" { + project = var.project_id + name = "system-pool" + cluster = google_container_cluster.gke.name + location = var.region + + autoscaling { + min_node_count = var.system_pool_min + max_node_count = var.system_pool_max + } + + node_config { + machine_type = var.system_pool_machine_type + disk_type = "pd-balanced" + disk_size_gb = var.node_disk_gb + image_type = "COS_CONTAINERD" + + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform", + ] + } +} + +resource "google_container_node_pool" "exec" { + project = var.project_id + name = "exec-pool" + cluster = google_container_cluster.gke.name + location = var.region + + autoscaling { + min_node_count = var.exec_pool_min + max_node_count = var.exec_pool_max + } + + node_config { + machine_type = var.exec_pool_machine_type + disk_type = "pd-balanced" + disk_size_gb = var.node_disk_gb + image_type = "COS_CONTAINERD" + + taint { + key = "shipsec.io/exec" + value = "true" + effect = "NO_SCHEDULE" + } + + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform", + ] + } +} + +output "artifact_registry_repo" { + value = "${var.region}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.docker.repository_id}" +} + +output "cluster_location" { + value = var.region +} + +output "cluster_name" { + value = google_container_cluster.gke.name +} + diff --git a/infra/gcp/envs/prod/variables.tf b/infra/gcp/envs/prod/variables.tf new file mode 100644 index 000000000..d07f0347c --- /dev/null +++ b/infra/gcp/envs/prod/variables.tf @@ -0,0 +1,80 @@ +variable "project_id" { + type = string + description = "GCP project id (e.g. shipsec)." +} + +variable "access_token" { + type = string + description = "Optional short-lived OAuth access token (bypasses ADC)." + default = null + sensitive = true +} + +variable "region" { + type = string + description = "GCP region for a regional prod cluster (e.g. us-central1)." + default = "us-central1" +} + +variable "cluster_name" { + type = string + description = "GKE cluster name." + default = "shipsec-prod" +} + +variable "artifact_repo_name" { + type = string + description = "Artifact Registry repo name (Docker)." + default = "shipsec-studio" +} + +variable "master_authorized_cidrs" { + type = list(object({ + cidr_block = string + display_name = string + })) + description = "CIDRs allowed to reach the control plane endpoint." + default = [] +} + +variable "system_pool_machine_type" { + type = string + description = "Machine type for the system node pool." + default = "e2-standard-4" +} + +variable "exec_pool_machine_type" { + type = string + description = "Machine type for the execution node pool." + default = "e2-standard-4" +} + +variable "system_pool_min" { + type = number + description = "Min nodes for system pool." + default = 2 +} + +variable "system_pool_max" { + type = number + description = "Max nodes for system pool." + default = 5 +} + +variable "exec_pool_min" { + type = number + description = "Min nodes for exec pool." + default = 1 +} + +variable "exec_pool_max" { + type = number + description = "Max nodes for exec pool." + default = 4 +} + +variable "node_disk_gb" { + type = number + description = "Boot disk size (GB)." + default = 100 +} diff --git a/infra/gcp/envs/prod/versions.tf b/infra/gcp/envs/prod/versions.tf new file mode 100644 index 000000000..0efbb7d9f --- /dev/null +++ b/infra/gcp/envs/prod/versions.tf @@ -0,0 +1,18 @@ +terraform { + required_version = ">= 1.5.0" + + backend "gcs" {} + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 5.20.0" + } + } +} + +provider "google" { + project = var.project_id + region = var.region + access_token = var.access_token +} diff --git a/packages/backend-client/src/api-client.ts b/packages/backend-client/src/api-client.ts index 4378d1ed8..d0743c2d2 100644 --- a/packages/backend-client/src/api-client.ts +++ b/packages/backend-client/src/api-client.ts @@ -36,7 +36,7 @@ export class ShipSecApiClient { private baseUrl: string; constructor(config: ClientConfig = {}) { - this.baseUrl = config.baseUrl || 'http://localhost:3211'; + this.baseUrl = config.baseUrl || ''; this.client = createClient({ baseUrl: this.baseUrl, diff --git a/packages/component-sdk/src/runner.ts b/packages/component-sdk/src/runner.ts index dd366bbdb..828af2b56 100644 --- a/packages/component-sdk/src/runner.ts +++ b/packages/component-sdk/src/runner.ts @@ -516,6 +516,29 @@ async function runDockerWithPty( }); } +/** + * Override hook for the docker runner. When set, all `kind: 'docker'` executions + * are routed through this function instead of the built-in runComponentInDocker. + * + * Used by the worker to plug in a K8s Job runner at startup: + * setDockerRunnerOverride(runComponentInK8sJob) + */ +type DockerRunnerOverrideFn = ( + runner: DockerRunnerConfig, + params: I, + context: ExecutionContext, +) => Promise; + +let dockerRunnerOverride: DockerRunnerOverrideFn | null = null; + +export function setDockerRunnerOverride(fn: DockerRunnerOverrideFn): void { + dockerRunnerOverride = fn; +} + +export function clearDockerRunnerOverride(): void { + dockerRunnerOverride = null; +} + export async function runComponentWithRunner( runner: RunnerConfig, execute: (params: I, context: ExecutionContext) => Promise, @@ -526,6 +549,9 @@ export async function runComponentWithRunner( case 'inline': return runComponentInline(execute, params, context); case 'docker': + if (dockerRunnerOverride) { + return dockerRunnerOverride(runner, params, context); + } return runComponentInDocker(runner, params, context); case 'remote': context.logger.info(`[Runner] remote execution stub for ${runner.endpoint}`); diff --git a/packages/component-sdk/src/types.ts b/packages/component-sdk/src/types.ts index a01bf5d7c..fe2baab6f 100644 --- a/packages/component-sdk/src/types.ts +++ b/packages/component-sdk/src/types.ts @@ -15,7 +15,7 @@ import type { HttpInstrumentationOptions, HttpRequestInput } from './http/types' export type { ExecutionContextMetadata } from './interfaces'; -export type RunnerKind = 'inline' | 'docker' | 'remote'; +export type RunnerKind = 'inline' | 'docker' | 'remote' | 'k8s'; export interface InlineRunnerConfig { kind: 'inline'; diff --git a/worker/package.json b/worker/package.json index bc75fb3da..7df4a1a3c 100644 --- a/worker/package.json +++ b/worker/package.json @@ -23,8 +23,10 @@ "@ai-sdk/mcp": "^1.0.13", "@ai-sdk/openai": "^3.0.18", "@aws-sdk/client-s3": "^3.975.0", + "@google-cloud/storage": "^7.14.0", "@googleapis/admin": "^29.0.0", "@grpc/grpc-js": "^1.14.3", + "@kubernetes/client-node": "^1.4.0", "@modelcontextprotocol/sdk": "^1.25.1", "@okta/okta-sdk-nodejs": "^7.3.0", "@opensearch-project/opensearch": "^3.5.1", diff --git a/worker/src/components/ai/__tests__/opencode.test.ts b/worker/src/components/ai/__tests__/opencode.test.ts index c14798390..1b2eac633 100644 --- a/worker/src/components/ai/__tests__/opencode.test.ts +++ b/worker/src/components/ai/__tests__/opencode.test.ts @@ -1,14 +1,14 @@ import { describe, it, expect, vi, beforeEach, afterAll } from 'bun:test'; import { componentRegistry } from '@shipsec/component-sdk'; import * as SDK from '@shipsec/component-sdk'; // Import for spying -import { IsolatedContainerVolume } from '../../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../../utils/isolated-volume'; import * as utils from '../utils'; import '../opencode'; // Register the component -// Mock IsolatedContainerVolume +// Mock createIsolatedVolume vi.mock('../../../utils/isolated-volume', () => { return { - IsolatedContainerVolume: vi.fn().mockImplementation(() => ({ + createIsolatedVolume: vi.fn().mockImplementation(() => ({ initialize: vi.fn().mockResolvedValue('mock-volume-name'), cleanup: vi.fn().mockResolvedValue(undefined), getVolumeConfig: vi @@ -84,8 +84,8 @@ describe('shipsec.opencode.agent', () => { expect(result.report).toContain('# Report'); - expect(IsolatedContainerVolume).toHaveBeenCalled(); - const volumeInstance = (IsolatedContainerVolume as any).mock.results[0].value; + expect(createIsolatedVolume).toHaveBeenCalled(); + const volumeInstance = (createIsolatedVolume as any).mock.results[0].value; const initCall = volumeInstance.initialize.mock.calls[0][0]; expect(initCall['context.json']).toContain('"alertId": "123"'); @@ -131,8 +131,8 @@ describe('shipsec.opencode.agent', () => { await component.execute({ inputs, params }, context as any); - expect(IsolatedContainerVolume).toHaveBeenCalled(); - const volumeInstance = (IsolatedContainerVolume as any).mock.results[0].value; + expect(createIsolatedVolume).toHaveBeenCalled(); + const volumeInstance = (createIsolatedVolume as any).mock.results[0].value; const initCall = volumeInstance.initialize.mock.calls[0][0]; const config = JSON.parse(initCall['opencode.jsonc']); diff --git a/worker/src/components/ai/opencode.ts b/worker/src/components/ai/opencode.ts index e83fd0ebe..d7818995f 100644 --- a/worker/src/components/ai/opencode.ts +++ b/worker/src/components/ai/opencode.ts @@ -11,7 +11,7 @@ import { param, } from '@shipsec/component-sdk'; import { LLMProviderSchema, llmProviderContractName } from '@shipsec/contracts'; -import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../utils/isolated-volume'; import { DEFAULT_GATEWAY_URL, getGatewaySessionToken } from './utils'; const inputSchema = inputs({ @@ -241,7 +241,7 @@ Please investigate the issue and generate a detailed report. // 4. Setup Isolated Volume const tenantId = (context as any).tenantId ?? 'default-tenant'; - const volume = new IsolatedContainerVolume(tenantId, context.runId); + const volume = createIsolatedVolume(tenantId, context.runId); try { // 5. Execute Docker Container diff --git a/worker/src/components/core/mcp-group-runtime.ts b/worker/src/components/core/mcp-group-runtime.ts index 7ee16001a..43ca38309 100644 --- a/worker/src/components/core/mcp-group-runtime.ts +++ b/worker/src/components/core/mcp-group-runtime.ts @@ -3,7 +3,7 @@ import type { ExecutionContext } from '@shipsec/component-sdk'; import { Client } from '@modelcontextprotocol/sdk/client/index.js'; import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; import { startMcpDockerServer } from './mcp-runtime'; -import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../utils/isolated-volume'; /** * Schema for MCP Group Templates (code-defined) @@ -169,8 +169,8 @@ export async function executeMcpGroupNode( ); const endpoints: McpServerEndpoint[] = []; - const volumes: ReturnType[] = []; - let volume: IsolatedContainerVolume | null = null; + const volumes: ReturnType['getVolumeConfig']>[] = []; + let volume: ReturnType | null = null; try { // Create volume if AWS files are needed @@ -178,7 +178,7 @@ export async function executeMcpGroupNode( const awsFiles = buildAwsCredentialFiles(credentials); if (awsFiles) { const tenantId = (context as any).tenantId ?? 'default-tenant'; - volume = new IsolatedContainerVolume(tenantId, context.runId); + volume = createIsolatedVolume(tenantId, context.runId); await volume.initialize({ credentials: awsFiles.credentials, config: awsFiles.config, diff --git a/worker/src/components/security/__tests__/nuclei-test-workflow.json b/worker/src/components/security/__tests__/nuclei-test-workflow.json index b8ab742dc..624d3edd9 100644 --- a/worker/src/components/security/__tests__/nuclei-test-workflow.json +++ b/worker/src/components/security/__tests__/nuclei-test-workflow.json @@ -60,9 +60,7 @@ "data": { "label": "Nuclei Scanner", "config": { - "targets": [ - "{{trigger.targetUrl}}" - ], + "targets": ["{{trigger.targetUrl}}"], "templateIds": [ "CVE-2024-1234", "http-missing-security-headers", @@ -116,4 +114,4 @@ "y": 0, "zoom": 1 } -} \ No newline at end of file +} diff --git a/worker/src/components/security/amass.ts b/worker/src/components/security/amass.ts index 25bc9212c..216f5668f 100644 --- a/worker/src/components/security/amass.ts +++ b/worker/src/components/security/amass.ts @@ -17,7 +17,7 @@ import { type ExecutionContext, type ExecutionPayload, } from '@shipsec/component-sdk'; -import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../utils/isolated-volume'; const AMASS_IMAGE = 'ghcr.io/shipsecai/amass:latest'; const AMASS_TIMEOUT_SECONDS = (() => { @@ -595,7 +595,7 @@ const definition = (defineComponent as any)({ const tenantId = (context as any).tenantId ?? 'default-tenant'; // Create isolated volume for this execution - const volume = new IsolatedContainerVolume(tenantId, context.runId); + const volume = createIsolatedVolume(tenantId, context.runId); const baseRunner = definition.runner; if (baseRunner.kind !== 'docker') { diff --git a/worker/src/components/security/dnsx.ts b/worker/src/components/security/dnsx.ts index 55eb41d97..e274b7abf 100644 --- a/worker/src/components/security/dnsx.ts +++ b/worker/src/components/security/dnsx.ts @@ -15,7 +15,7 @@ import { analyticsResultSchema, type AnalyticsResult, } from '@shipsec/component-sdk'; -import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../utils/isolated-volume'; const recordTypeEnum = z.enum([ 'A', @@ -621,7 +621,7 @@ const definition = defineComponent({ const tenantId = (context as any).tenantId ?? 'default-tenant'; // Create isolated volume for this execution - const volume = new IsolatedContainerVolume(tenantId, context.runId); + const volume = createIsolatedVolume(tenantId, context.runId); const baseRunner = definition.runner; if (baseRunner.kind !== 'docker') { diff --git a/worker/src/components/security/httpx.ts b/worker/src/components/security/httpx.ts index ee95318e4..17130a338 100644 --- a/worker/src/components/security/httpx.ts +++ b/worker/src/components/security/httpx.ts @@ -14,7 +14,7 @@ import { analyticsResultSchema, type AnalyticsResult, } from '@shipsec/component-sdk'; -import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../utils/isolated-volume'; const inputSchema = inputs({ targets: port( @@ -309,7 +309,7 @@ const definition = defineComponent({ }); const tenantId = (context as any).tenantId ?? 'default-tenant'; - const volume = new IsolatedContainerVolume(tenantId, context.runId); + const volume = createIsolatedVolume(tenantId, context.runId); try { const targets = Array.from( diff --git a/worker/src/components/security/nuclei.ts b/worker/src/components/security/nuclei.ts index c9d378d2f..56d2db3e7 100644 --- a/worker/src/components/security/nuclei.ts +++ b/worker/src/components/security/nuclei.ts @@ -16,7 +16,7 @@ import { analyticsResultSchema, type AnalyticsResult, } from '@shipsec/component-sdk'; -import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../utils/isolated-volume'; import * as yaml from 'js-yaml'; const inputSchema = inputs({ @@ -330,7 +330,7 @@ const definition = defineComponent({ context.logger.info(`[Nuclei] Starting scan for ${parsedInputs.targets.length} target(s)`); const tenantId = (context as any).tenantId ?? 'default-tenant'; - let volume: IsolatedContainerVolume | null = null; + let volume: ReturnType | null = null; try { const hasCustomArchive = !!parsedInputs.customTemplateArchive; @@ -393,7 +393,7 @@ const definition = defineComponent({ } // ===== TypeScript: Prepare all files for volume ===== - volume = new IsolatedContainerVolume(tenantId, context.runId); + volume = createIsolatedVolume(tenantId, context.runId); const files: Record = {}; // Always add targets file diff --git a/worker/src/components/security/prowler-scan.ts b/worker/src/components/security/prowler-scan.ts index 78077771a..fd9cf81ff 100644 --- a/worker/src/components/security/prowler-scan.ts +++ b/worker/src/components/security/prowler-scan.ts @@ -21,7 +21,7 @@ import { import type { DockerRunnerConfig } from '@shipsec/component-sdk'; import { awsCredentialSchema } from '@shipsec/contracts'; -import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../utils/isolated-volume'; const recommendedFlagOptions = [ { @@ -296,10 +296,41 @@ const recommendedFlagMap = new Map( recommendedFlagOptions.map((option) => [option.id, [...option.args]]), ); -async function listVolumeFiles(volume: IsolatedContainerVolume): Promise { +async function listVolumeFiles(volume: ReturnType): Promise { const volumeName = volume.getVolumeName(); if (!volumeName) return []; + // In K8s mode, volumes are ConfigMap-backed or GCS-backed + if (process.env.EXECUTION_MODE === 'k8s') { + // GCS FUSE volumes: list objects in the GCS prefix via SDK + if (process.env.GCS_VOLUME_BUCKET) { + try { + const { Storage } = await import('@google-cloud/storage'); + const storage = new Storage(); + const bucket = storage.bucket(process.env.GCS_VOLUME_BUCKET); + const [files] = await bucket.getFiles({ prefix: `${volumeName}/` }); + return files.map((f) => f.name.replace(`${volumeName}/`, '')); + } catch { + return []; + } + } + + // ConfigMap-backed volumes: list keys via K8s API + try { + const k8s = await import('@kubernetes/client-node'); + const kc = new k8s.KubeConfig(); + kc.loadFromCluster(); + const coreApi = kc.makeApiClient(k8s.CoreV1Api); + const namespace = process.env.K8S_JOB_NAMESPACE || 'shipsec-workloads'; + const cm = await coreApi.readNamespacedConfigMap({ name: volumeName, namespace }); + const keys = [...Object.keys(cm.data || {}), ...Object.keys(cm.binaryData || {})]; + // Unflatten __ back to / (ConfigMap key encoding from IsolatedK8sVolume) + return keys.map((k) => k.replace(/__/g, '/')); + } catch { + return []; + } + } + const dockerPath = await resolveDockerPath(); return new Promise((resolve, reject) => { const proc = spawn(dockerPath, [ @@ -350,13 +381,16 @@ async function listVolumeFiles(volume: IsolatedContainerVolume): Promise, uid = 1000, gid = 1000, ): Promise { const volumeName = volume.getVolumeName(); if (!volumeName) return; + // ConfigMap volumes in K8s are read-only projections — ownership is N/A + if (process.env.EXECUTION_MODE === 'k8s') return; + const dockerPath = await resolveDockerPath(); return new Promise((resolve, reject) => { const proc = spawn(dockerPath, [ @@ -509,7 +543,7 @@ const definition = defineComponent({ const awsEnv: Record = {}; const tenantId = (context as any).tenantId ?? 'default-tenant'; const awsCredsVolume = parsedInputs.credentials - ? new IsolatedContainerVolume(tenantId, `${context.runId}-prowler-aws`) + ? createIsolatedVolume(tenantId, `${context.runId}-prowler-aws`) : null; if (parsedInputs.credentials) { @@ -590,7 +624,7 @@ const definition = defineComponent({ let rawSegments: string[] = []; let commandForOutput: string[] = cmd; let stderrCombined = ''; - const outputVolume = new IsolatedContainerVolume(tenantId, `${context.runId}-prowler-out`); + const outputVolume = createIsolatedVolume(tenantId, `${context.runId}-prowler-out`); let outputVolumeInitialized = false; let awsVolumeInitialized = false; diff --git a/worker/src/components/security/shuffledns-massdns.ts b/worker/src/components/security/shuffledns-massdns.ts index d3690beab..6e7119724 100644 --- a/worker/src/components/security/shuffledns-massdns.ts +++ b/worker/src/components/security/shuffledns-massdns.ts @@ -16,7 +16,7 @@ import { analyticsResultSchema, type AnalyticsResult, } from '@shipsec/component-sdk'; -import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../utils/isolated-volume'; const DEFAULT_RESOLVERS = ['1.1.1.1', '8.8.8.8'] as const; @@ -288,7 +288,7 @@ const definition = defineComponent({ // Write lists to an isolated volume and mount into the container const tenantId = (context as any).tenantId ?? 'default-tenant'; - const volume = new IsolatedContainerVolume(tenantId, context.runId); + const volume = createIsolatedVolume(tenantId, context.runId); const WORDS = 'words.txt'; const SEEDS = 'seeds.txt'; const RESOLVERS = 'resolvers.txt'; diff --git a/worker/src/components/security/subfinder.ts b/worker/src/components/security/subfinder.ts index e58c26aac..c02b89e8d 100644 --- a/worker/src/components/security/subfinder.ts +++ b/worker/src/components/security/subfinder.ts @@ -15,7 +15,7 @@ import { analyticsResultSchema, type AnalyticsResult, } from '@shipsec/component-sdk'; -import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../utils/isolated-volume'; const SUBFINDER_IMAGE = 'ghcr.io/shipsecai/subfinder:latest'; const SUBFINDER_TIMEOUT_SECONDS = 1800; // 30 minutes @@ -203,7 +203,7 @@ interface BuildSubfinderArgsOptions { const buildSubfinderArgs = (options: BuildSubfinderArgsOptions): string[] => { const args: string[] = []; - // Always use silent mode for clean output + // Use silent mode — stdout is the structured output (one subdomain per line) args.push('-silent'); // Domain list file input @@ -384,7 +384,7 @@ const definition = defineComponent({ const tenantId = (context as any).tenantId ?? 'default-tenant'; // Create isolated volume for this execution - const volume = new IsolatedContainerVolume(tenantId, context.runId); + const volume = createIsolatedVolume(tenantId, context.runId); const baseRunner = definition.runner; if (baseRunner.kind !== 'docker') { @@ -431,7 +431,9 @@ const definition = defineComponent({ network: baseRunner.network, timeoutSeconds: baseRunner.timeoutSeconds ?? SUBFINDER_TIMEOUT_SECONDS, env: { ...(baseRunner.env ?? {}) }, - // Pass subfinder CLI args directly (image default entrypoint is subfinder) + // Preserve the shell wrapper from baseRunner (sh -c 'subfinder "$@"' --) + // The K8s runner detects the "$@" pattern and strips the shell for distroless images + entrypoint: baseRunner.entrypoint, command: [...(baseRunner.command ?? []), ...subfinderArgs], volumes: [volume.getVolumeConfig(CONTAINER_INPUT_DIR, true)], }; diff --git a/worker/src/components/security/supabase-scanner.ts b/worker/src/components/security/supabase-scanner.ts index 5b1d3c96d..463503429 100644 --- a/worker/src/components/security/supabase-scanner.ts +++ b/worker/src/components/security/supabase-scanner.ts @@ -15,7 +15,7 @@ import { type AnalyticsResult, type DockerRunnerConfig, } from '@shipsec/component-sdk'; -import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../utils/isolated-volume'; // Extract Supabase project ref from a standard URL like https://.supabase.co function inferProjectRef(supabaseUrl: string): string | null { @@ -255,7 +255,7 @@ const definition = defineComponent({ }; const tenantId = (context as any).tenantId ?? 'default-tenant'; - const volume = new IsolatedContainerVolume(tenantId, context.runId); + const volume = createIsolatedVolume(tenantId, context.runId); const mountPath = '/data'; const configFilename = 'scanner_config.yaml'; const outputFilename = 'report.json'; diff --git a/worker/src/components/security/trufflehog.ts b/worker/src/components/security/trufflehog.ts index 06a768bbe..4b7ab308b 100644 --- a/worker/src/components/security/trufflehog.ts +++ b/worker/src/components/security/trufflehog.ts @@ -16,7 +16,7 @@ import { analyticsResultSchema, type AnalyticsResult, } from '@shipsec/component-sdk'; -import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../utils/isolated-volume'; const scanTypeSchema = z.enum(['git', 'github', 'gitlab', 's3', 'gcs', 'filesystem', 'docker']); @@ -392,7 +392,7 @@ const definition = defineComponent({ }); // Handle filesystem scanning with isolated volumes - let volume: IsolatedContainerVolume | undefined; + let volume: ReturnType | undefined; let effectiveInput = runnerPayload; const baseRunner = definition.runner; @@ -415,7 +415,7 @@ const definition = defineComponent({ } const tenantId = (context as any).tenantId ?? 'default-tenant'; - volume = new IsolatedContainerVolume(tenantId, context.runId); + volume = createIsolatedVolume(tenantId, context.runId); // Initialize volume with files const volumeName = await volume.initialize(runnerPayload.filesystemContent); diff --git a/worker/src/components/test/simple-http-mcp.ts b/worker/src/components/test/simple-http-mcp.ts index e8a72df6c..84cab7bd1 100644 --- a/worker/src/components/test/simple-http-mcp.ts +++ b/worker/src/components/test/simple-http-mcp.ts @@ -13,7 +13,7 @@ import { runComponentWithRunner, } from '@shipsec/component-sdk'; import { z } from 'zod'; -import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../utils/isolated-volume'; const inputSchema = inputs({}); @@ -70,7 +70,7 @@ const definition = defineComponent({ async execute({ inputs: _inputs, params }, context) { const { port } = params; const { tenantId = 'default' } = context as any; - const volume = new IsolatedContainerVolume(tenantId, context.runId); + const volume = createIsolatedVolume(tenantId, context.runId); try { // Create MCP server script diff --git a/worker/src/temporal/activities/mcp-discovery.activity.ts b/worker/src/temporal/activities/mcp-discovery.activity.ts index cb0aa1bf0..498339e15 100644 --- a/worker/src/temporal/activities/mcp-discovery.activity.ts +++ b/worker/src/temporal/activities/mcp-discovery.activity.ts @@ -464,6 +464,8 @@ async function cleanupContainer(containerId: string | undefined): Promise if (!containerId) { return; } + // In K8s mode there is no Docker daemon — skip container cleanup + if (process.env.EXECUTION_MODE === 'k8s') return; // Validate container ID to prevent command injection if (!/^[a-zA-Z0-9_.-][a-zA-Z0-9_.-]*$/.test(containerId)) { console.warn(`[MCP Discovery] Skipping cleanup with unsafe container id: ${containerId}`); diff --git a/worker/src/temporal/workers/dev.worker.ts b/worker/src/temporal/workers/dev.worker.ts index 6e2495cc6..d29dbb195 100644 --- a/worker/src/temporal/workers/dev.worker.ts +++ b/worker/src/temporal/workers/dev.worker.ts @@ -232,6 +232,14 @@ async function main() { console.log(`✅ Service adapters initialized`); + // Register K8s runner override if EXECUTION_MODE=k8s + if (process.env.EXECUTION_MODE === 'k8s') { + const { runComponentInK8sJob } = await import('../../utils/k8s-runner'); + const { setDockerRunnerOverride } = await import('@shipsec/component-sdk'); + setDockerRunnerOverride(runComponentInK8sJob); + console.log('[Worker] K8s execution mode enabled — docker runner overridden with K8s Jobs'); + } + console.log(`🏗️ Creating Temporal worker...`); console.log( ` - Activities: ${Object.keys({ diff --git a/worker/src/testing/test-gcs-volume.ts b/worker/src/testing/test-gcs-volume.ts new file mode 100644 index 000000000..717886cc3 --- /dev/null +++ b/worker/src/testing/test-gcs-volume.ts @@ -0,0 +1,140 @@ +#!/usr/bin/env bun +/** + * End-to-end integration test for GCS FUSE volume sharing. + * + * Validates the full flow: + * 1. IsolatedGcsVolume.initialize() uploads files to GCS + * 2. K8s job mounts them via GCS FUSE CSI at /inputs + * 3. Job reads the file, writes output to /shipsec-output/result.json + * 4. Worker reads the JSON output from pod logs + * 5. volume.cleanup() removes GCS objects + * + * Run inside the worker pod: + * kubectl exec -n shipsec-workers -- bun run /app/worker/src/testing/test-gcs-volume.ts + */ + +import { IsolatedGcsVolume } from '../utils/gcs-volume'; +import { runComponentInK8sJob } from '../utils/k8s-runner'; +import type { ExecutionContext } from '@shipsec/component-sdk'; +import type { DockerRunnerConfig } from '@shipsec/component-sdk'; + +const PASS = '\x1b[32m✓\x1b[0m'; +const FAIL = '\x1b[31m✗\x1b[0m'; + +function makeContext(): ExecutionContext { + return { + runId: `test-gcs-${Date.now()}`, + componentRef: 'test.gcs.volume', + logger: { + info: (msg: string) => console.log(` [info] ${msg}`), + warn: (msg: string) => console.warn(` [warn] ${msg}`), + error: (msg: string) => console.error(` [error] ${msg}`), + debug: (msg: string) => console.log(` [debug] ${msg}`), + }, + emitProgress: (msg: string) => console.log(` [progress] ${msg}`), + secrets: undefined, + storage: undefined, + artifacts: undefined, + trace: undefined, + logCollector: undefined, + terminalCollector: undefined, + metadata: { runId: `test-gcs-${Date.now()}`, componentRef: 'test.gcs.volume' }, + http: { fetch: fetch as any, toCurl: () => '' }, + } as any; +} + +async function testVolumeWriteRead() { + console.log('\n── Test 1: GCS volume write → K8s job read ──'); + + const volume = new IsolatedGcsVolume('testtenant', `run${Date.now()}`); + const testContent = `hello-from-gcs-${Date.now()}`; + + // 1. Upload file to GCS + const prefix = await volume.initialize({ 'input.txt': testContent }); + console.log(` ${PASS} Uploaded input.txt to GCS prefix: ${prefix}`); + + const ctx = makeContext(); + + // 2. Runner: alpine reads /inputs/input.txt and writes JSON output + const runner: DockerRunnerConfig = { + kind: 'docker', + image: 'alpine:3.20', + entrypoint: 'sh', + command: [ + '-c', + `content=$(cat /inputs/input.txt); printf '{"content":"%s"}' "$content" > /shipsec-output/result.json`, + ], + timeoutSeconds: 60, + volumes: [volume.getVolumeConfig('/inputs', true)], + }; + + try { + const result = await runComponentInK8sJob(runner, {}, ctx); + console.log(` ${PASS} K8s job completed, result:`, result); + + if (result?.content === testContent) { + console.log(` ${PASS} Content matches! "${result.content}"`); + } else { + console.error( + ` ${FAIL} Content mismatch: expected "${testContent}", got "${result?.content}"`, + ); + process.exit(1); + } + } finally { + await volume.cleanup(); + console.log(` ${PASS} GCS volume cleaned up`); + } +} + +async function testVolumeCleanup() { + console.log('\n── Test 2: GCS volume cleanup removes objects ──'); + + const { Storage } = await import('@google-cloud/storage'); + const storage = new Storage(); + const bucket = storage.bucket(process.env.GCS_VOLUME_BUCKET!); + + const volume = new IsolatedGcsVolume('testcleanup', `run${Date.now()}`); + await volume.initialize({ 'deleteme.txt': 'temporary' }); + const prefix = volume.getVolumeName()!; + + // Verify file exists + const [before] = await bucket.getFiles({ prefix }); + if (before.length === 0) { + console.error(` ${FAIL} File not found in GCS before cleanup`); + process.exit(1); + } + console.log(` ${PASS} File exists in GCS (${before.length} object(s))`); + + await volume.cleanup(); + + // Verify file deleted + const [after] = await bucket.getFiles({ prefix }); + if (after.length === 0) { + console.log(` ${PASS} GCS objects cleaned up successfully`); + } else { + console.error(` ${FAIL} ${after.length} objects still remain after cleanup`); + process.exit(1); + } +} + +async function main() { + console.log('🧪 GCS FUSE Volume Integration Tests'); + console.log(` EXECUTION_MODE=${process.env.EXECUTION_MODE}`); + console.log(` GCS_VOLUME_BUCKET=${process.env.GCS_VOLUME_BUCKET}`); + console.log(` K8S_JOB_NAMESPACE=${process.env.K8S_JOB_NAMESPACE}`); + + if (!process.env.GCS_VOLUME_BUCKET) { + console.error(`${FAIL} GCS_VOLUME_BUCKET not set`); + process.exit(1); + } + + await testVolumeCleanup(); + await testVolumeWriteRead(); + + console.log('\n\x1b[32m✓ All tests passed\x1b[0m\n'); +} + +main().catch((err) => { + console.error(`\n${FAIL} Test failed:`, err); + process.exit(1); +}); diff --git a/worker/src/utils/gcs-volume.ts b/worker/src/utils/gcs-volume.ts new file mode 100644 index 000000000..fc3e5aac8 --- /dev/null +++ b/worker/src/utils/gcs-volume.ts @@ -0,0 +1,214 @@ +/** + * IsolatedGcsVolume — GCS FUSE CSI volume replacement for IsolatedK8sVolume. + * + * Uses a GCS bucket mounted via the GCS FUSE CSI driver instead of ConfigMaps. + * Same interface as IsolatedK8sVolume / IsolatedContainerVolume so components + * can swap transparently via the createIsolatedVolume() factory. + * + * Advantages over ConfigMap-backed volumes: + * - No 1 MiB size limit (handles large outputs like Prowler) + * - Native read-write (no log-based writeback hack) + * - ReadWriteMany (parallel pods can share data) + * - Worker reads output directly from GCS via SDK + */ +import { Storage } from '@google-cloud/storage'; +import { ValidationError, ConfigurationError, ContainerError } from '@shipsec/component-sdk'; + +let _storage: Storage | null = null; + +function getStorage(): Storage { + if (!_storage) { + // Auto-discovers Workload Identity credentials in GKE + _storage = new Storage(); + } + return _storage; +} + +function getBucketName(): string { + const bucket = process.env.GCS_VOLUME_BUCKET; + if (!bucket) { + throw new ConfigurationError('GCS_VOLUME_BUCKET environment variable is not set'); + } + return bucket; +} + +function sanitizeName(raw: string): string { + return raw + .toLowerCase() + .replace(/[^a-z0-9-]/g, '-') + .replace(/-+/g, '-') + .replace(/^-|-$/g, '') + .slice(0, 53); +} + +export class IsolatedGcsVolume { + private prefix?: string; + private isInitialized = false; + private bucketName: string; + + constructor( + private tenantId: string, + private runId: string, + ) { + if (!/^[a-zA-Z0-9_-]+$/.test(tenantId)) { + throw new ValidationError( + 'Invalid tenant ID: must contain only alphanumeric characters, hyphens, and underscores', + { + fieldErrors: { + tenantId: ['must contain only alphanumeric characters, hyphens, and underscores'], + }, + }, + ); + } + if (!/^[a-zA-Z0-9_-]+$/.test(runId)) { + throw new ValidationError( + 'Invalid run ID: must contain only alphanumeric characters, hyphens, and underscores', + { + fieldErrors: { + runId: ['must contain only alphanumeric characters, hyphens, and underscores'], + }, + }, + ); + } + this.bucketName = getBucketName(); + } + + /** + * Upload files to GCS under a unique prefix and return the prefix. + * GCS key structure: {tenantId}/{runId}/{timestamp}/{filename} + */ + async initialize(files: Record): Promise { + if (this.isInitialized) { + throw new ConfigurationError('Volume already initialized', { + details: { prefix: this.prefix, tenantId: this.tenantId, runId: this.runId }, + }); + } + + const timestamp = Date.now(); + const tenantShort = sanitizeName(this.tenantId); + const runShort = sanitizeName(this.runId); + this.prefix = `${tenantShort}/${runShort}/${timestamp}`; + + try { + const storage = getStorage(); + const bucket = storage.bucket(this.bucketName); + + const uploads = Object.entries(files).map(async ([filename, content]) => { + this.validateFilename(filename); + const key = `${this.prefix}/${filename}`; + const file = bucket.file(key); + const data = typeof content === 'string' ? Buffer.from(content, 'utf-8') : content; + await file.save(data); + }); + + await Promise.all(uploads); + + this.isInitialized = true; + return this.prefix; + } catch (error) { + if (this.prefix) { + await this.cleanup().catch(() => {}); + } + throw new ContainerError( + `Failed to initialize GCS volume: ${error instanceof Error ? error.message : String(error)}`, + { + cause: error instanceof Error ? error : undefined, + details: { tenantId: this.tenantId, runId: this.runId }, + }, + ); + } + } + + private validateFilename(filename: string): void { + if (filename.includes('..') || filename.startsWith('/')) { + throw new ValidationError(`Invalid filename (path traversal): ${filename}`, { + fieldErrors: { filename: ['path traversal not allowed'] }, + }); + } + const safePattern = /^[a-zA-Z0-9._/-]+$/; + if (!safePattern.test(filename)) { + throw new ValidationError(`Invalid filename (contains unsafe characters): ${filename}`, { + fieldErrors: { filename: ['contains unsafe characters'] }, + }); + } + } + + /** + * Download files from GCS by name. + */ + async readFiles(filenames: string[]): Promise> { + if (!this.prefix) { + throw new ConfigurationError('Volume not initialized'); + } + + const storage = getStorage(); + const bucket = storage.bucket(this.bucketName); + const results: Record = {}; + + for (const filename of filenames) { + try { + const key = `${this.prefix}/${filename}`; + const file = bucket.file(key); + const [contents] = await file.download(); + results[filename] = contents.toString('utf-8'); + } catch (error) { + console.warn( + `Could not read file ${filename} from GCS: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + + return results; + } + + /** + * Returns volume config for the runner. + * The K8s runner recognizes the "gcsfuse:" prefix and creates a CSI volume. + * Format: "gcsfuse:{bucketName}:{prefix}" + */ + getVolumeConfig(containerPath = '/inputs', readOnly = true) { + if (!this.prefix) { + throw new ConfigurationError('Volume not initialized'); + } + return { + source: `gcsfuse:${this.bucketName}:${this.prefix}`, + target: containerPath, + readOnly, + }; + } + + /** + * Returns a bind mount string (for interface compatibility). + */ + getBindMount(containerPath = '/inputs', readOnly = true): string { + if (!this.prefix) { + throw new ConfigurationError('Volume not initialized'); + } + const mode = readOnly ? 'ro' : 'rw'; + return `gcsfuse:${this.bucketName}:${this.prefix}:${containerPath}:${mode}`; + } + + /** + * Delete all objects under the GCS prefix. + */ + async cleanup(): Promise { + if (!this.prefix) return; + + try { + const storage = getStorage(); + const bucket = storage.bucket(this.bucketName); + await bucket.deleteFiles({ prefix: `${this.prefix}/` }); + } catch (error) { + console.error( + `Failed to cleanup GCS volume ${this.prefix}: ${error instanceof Error ? error.message : String(error)}`, + ); + } finally { + this.isInitialized = false; + this.prefix = undefined; + } + } + + getVolumeName(): string | undefined { + return this.prefix; + } +} diff --git a/worker/src/utils/index.ts b/worker/src/utils/index.ts index 4afa27813..a85814125 100644 --- a/worker/src/utils/index.ts +++ b/worker/src/utils/index.ts @@ -2,4 +2,10 @@ * Utility exports for worker components */ -export { IsolatedContainerVolume, cleanupOrphanedVolumes } from './isolated-volume'; +export { + IsolatedContainerVolume, + cleanupOrphanedVolumes, + createIsolatedVolume, +} from './isolated-volume'; +export { IsolatedK8sVolume } from './k8s-volume'; +export { IsolatedGcsVolume } from './gcs-volume'; diff --git a/worker/src/utils/isolated-volume.ts b/worker/src/utils/isolated-volume.ts index ea6304709..f266b6d53 100644 --- a/worker/src/utils/isolated-volume.ts +++ b/worker/src/utils/isolated-volume.ts @@ -2,6 +2,8 @@ import { spawn } from 'child_process'; import { promisify } from 'util'; import { exec as execCallback } from 'child_process'; import { ValidationError, ConfigurationError, ContainerError } from '@shipsec/component-sdk'; +import { IsolatedK8sVolume } from './k8s-volume'; +import { IsolatedGcsVolume } from './gcs-volume'; const exec = promisify(execCallback); @@ -505,6 +507,9 @@ export class IsolatedContainerVolume { * ``` */ export async function cleanupOrphanedVolumes(olderThanHours = 24): Promise { + // In K8s mode volumes are ConfigMaps — no Docker daemon to query + if (process.env.EXECUTION_MODE === 'k8s') return 0; + try { const { stdout } = await exec( 'docker volume ls --filter "label=studio.managed=true" --format "{{.Name}}|||{{.CreatedAt}}"', @@ -542,3 +547,23 @@ export async function cleanupOrphanedVolumes(olderThanHours = 24): Promise; // mountPath → configMapName + hasGcsFuse: boolean; +} + +// Lazy-init shared K8s clients +let _kc: k8s.KubeConfig | null = null; +let _batchApi: k8s.BatchV1Api | null = null; +let _coreApi: k8s.CoreV1Api | null = null; + +function getKubeConfig(): k8s.KubeConfig { + if (!_kc) { + _kc = new k8s.KubeConfig(); + _kc.loadFromCluster(); // uses in-cluster SA token + } + return _kc; +} + +function getBatchApi(): k8s.BatchV1Api { + if (!_batchApi) _batchApi = getKubeConfig().makeApiClient(k8s.BatchV1Api); + return _batchApi; +} + +function getCoreApi(): k8s.CoreV1Api { + if (!_coreApi) _coreApi = getKubeConfig().makeApiClient(k8s.CoreV1Api); + return _coreApi; +} + +function getJobNamespace(): string { + return process.env.K8S_JOB_NAMESPACE || 'shipsec-workloads'; +} + +function sanitizeName(raw: string): string { + // K8s names: lowercase, alphanumeric + hyphens, max 63 chars + return raw + .toLowerCase() + .replace(/[^a-z0-9-]/g, '-') + .replace(/-+/g, '-') + .replace(/^-|-$/g, '') + .slice(0, 53); // leave room for random suffix +} + +function generateJobName(context: ExecutionContext, image: string): string { + const imgShort = sanitizeName(image.split('/').pop()?.split(':')[0] || 'job'); + const runShort = sanitizeName(context.runId).slice(0, 8); + const rand = Math.random().toString(36).slice(2, 8); + return `ss-${imgShort}-${runShort}-${rand}`; +} + +/** + * Shell snippet that captures files from writable volume mounts. + * Reads $SHIPSEC_WRITABLE_MOUNTS (space-separated paths) and emits + * each file as base64 between markers so the worker can parse them + * from pod logs and write them back to their backing ConfigMaps. + */ +const VOLUME_CAPTURE_SCRIPT = [ + `echo '${VOLUME_DELIMITER}'`, + 'for __mp in $SHIPSEC_WRITABLE_MOUNTS; do', + ' find "$__mp" -type f 2>/dev/null | while IFS= read -r __f; do', + ' __rel="${__f#$__mp/}"', + ' echo "___FILE_START___:$__mp:$__rel"', + ' base64 "$__f" 2>/dev/null || true', + ' echo "___FILE_END___"', + ' done', + 'done', +].join('\n'); + +/** + * Build the command wrapper that emits the output file to stdout. + * + * For images with a shell: wraps original command so that after it exits, + * the output file is printed to stdout with a delimiter prefix. + * If writable volumes exist, also captures their contents as base64. + * + * For images without a shell (distroless): returns original command as-is, + * relying on stdout-based output fallback. + */ +function wrapCommandForOutput(runner: DockerRunnerConfig): { command: string[]; args: string[] } { + const { entrypoint, command } = runner; + + // Volume capture suffix — only emits data when SHIPSEC_WRITABLE_MOUNTS is set + const volCapture = `; if [ -n "$SHIPSEC_WRITABLE_MOUNTS" ]; then ${VOLUME_CAPTURE_SCRIPT}; fi`; + + const isShellEntrypoint = + entrypoint === 'sh' || + entrypoint === 'bash' || + entrypoint === '/bin/sh' || + entrypoint === '/bin/bash'; + + if (isShellEntrypoint && command.length >= 2 && command[0] === '-c') { + // Shell wrapper pattern: entrypoint=sh, command=['-c', 'binary "$@"', '--', ...dynamicArgs] + const shellScript = command[1]; + const dynamicArgsMatch = shellScript.match(/^(\S+)\s+"\$@"$/); + + if (dynamicArgsMatch) { + // Dynamic args pattern for distroless images (e.g., 'subfinder "$@"') + // These images don't have sh — use their default ENTRYPOINT directly. + // The dynamic args follow after '--' in the command array. + const dashDashIdx = command.indexOf('--'); + const dynamicArgs = dashDashIdx >= 0 ? command.slice(dashDashIdx + 1) : []; + // Return empty command to use image's ENTRYPOINT, pass dynamic args directly + return { command: [], args: dynamicArgs }; + } + + // Regular shell script — wrap with output capture + const userScript = command.slice(1).join(' '); + const wrapped = `${userScript}; __exit=$?; echo '${OUTPUT_DELIMITER}'; cat ${CONTAINER_OUTPUT_PATH}/${OUTPUT_FILENAME} 2>/dev/null || echo '{}'${volCapture}; exit $__exit`; + return { command: [entrypoint!], args: ['-c', wrapped] }; + } + + if (isShellEntrypoint) { + return { command: [entrypoint!], args: command }; + } + + // For non-shell entrypoints (e.g., 'httpx', 'nuclei', binary entrypoints): + // Use the entrypoint directly — the image may be distroless (no /bin/sh). + // Output is captured from stdout via parseOutputFromLogs fallback. + if (entrypoint) { + return { command: [entrypoint], args: command }; + } + + if (command.length > 0) { + return { command: [command[0]], args: command.slice(1) }; + } + + return { command: [], args: [] }; +} + +/** + * Create a ConfigMap containing the serialized input data. + */ +async function createInputConfigMap( + name: string, + namespace: string, + inputData: unknown, +): Promise { + const core = getCoreApi(); + const body: k8s.V1ConfigMap = { + metadata: { + name, + namespace, + labels: { + 'app.kubernetes.io/managed-by': 'shipsec-worker', + 'shipsec.ai/purpose': 'job-input', + }, + }, + data: { + 'input.json': JSON.stringify(inputData), + }, + }; + await core.createNamespacedConfigMap({ namespace, body }); +} + +/** + * Build the K8s Job spec from a DockerRunnerConfig. + */ +function buildJobSpec( + jobName: string, + namespace: string, + configMapName: string, + runner: DockerRunnerConfig, + context: ExecutionContext, +): BuildJobResult { + const { command, args } = wrapCommandForOutput(runner); + const timeoutSeconds = runner.timeoutSeconds || 300; + + // Track writable ConfigMap volumes for post-execution data capture + const writableVolumeMappings = new Map(); + + // Build env vars + const envVars: k8s.V1EnvVar[] = [ + { name: 'SHIPSEC_INPUT_PATH', value: '/shipsec-input/input.json' }, + { name: 'SHIPSEC_OUTPUT_PATH', value: `${CONTAINER_OUTPUT_PATH}/${OUTPUT_FILENAME}` }, + ]; + if (runner.env) { + for (const [key, value] of Object.entries(runner.env)) { + // Override HOME=/root for distroless images — /root is not writable + if (key === 'HOME' && value === '/root') { + envVars.push({ name: key, value: '/tmp' }); + } else { + envVars.push({ name: key, value }); + } + } + } + + // Build volume mounts + const volumeMounts: k8s.V1VolumeMount[] = [ + { name: 'input', mountPath: '/shipsec-input', readOnly: true }, + { name: 'output', mountPath: CONTAINER_OUTPUT_PATH }, + ]; + + const volumes: k8s.V1Volume[] = [ + { + name: 'input', + configMap: { name: configMapName }, + }, + { + name: 'output', + emptyDir: {}, + }, + ]; + + // Handle additional volumes (from IsolatedK8sVolume or IsolatedGcsVolume) + let hasGcsFuse = false; + if (runner.volumes) { + for (let i = 0; i < runner.volumes.length; i++) { + const vol = runner.volumes[i]; + if (!vol || !vol.source || !vol.target) continue; + + const volName = `extra-vol-${i}`; + + if (vol.source.startsWith('gcsfuse:')) { + // GCS FUSE CSI volume from IsolatedGcsVolume + // Parse "gcsfuse:{bucket}:{prefix}" + const [, bucketName, ...prefixParts] = vol.source.split(':'); + const onlyDir = prefixParts.join(':'); + hasGcsFuse = true; + volumes.push({ + name: volName, + csi: { + driver: 'gcsfuse.csi.storage.gke.io', + readOnly: vol.readOnly ?? false, + volumeAttributes: { + bucketName, + mountOptions: `implicit-dirs,only-dir=${onlyDir}`, + }, + }, + }); + // NO writableVolumeMappings tracking needed — GCS handles write natively + } else if (vol.source.startsWith('configmap:') && (vol.readOnly ?? true)) { + // ConfigMap-backed volume from IsolatedK8sVolume (read-only) + const cmName = vol.source.replace('configmap:', ''); + volumes.push({ + name: volName, + configMap: { name: cmName }, + }); + } else if (vol.source.startsWith('configmap:') && !(vol.readOnly ?? true)) { + const cmName = vol.source.replace('configmap:', ''); + // Use emptyDir for the actual mount (ConfigMaps are read-only in K8s) + volumes.push({ + name: volName, + emptyDir: {}, + }); + // Track for post-execution data capture + writableVolumeMappings.set(vol.target, cmName); + } else { + // Treat as emptyDir (can't use host paths in K8s Jobs) + volumes.push({ + name: volName, + emptyDir: {}, + }); + } + + volumeMounts.push({ + name: volName, + mountPath: vol.target, + readOnly: vol.readOnly ?? false, + }); + } + } + + // Add env var for writable mount paths so the shell wrapper can capture files + if (writableVolumeMappings.size > 0) { + envVars.push({ + name: 'SHIPSEC_WRITABLE_MOUNTS', + value: Array.from(writableVolumeMappings.keys()).join(' '), + }); + } + + const job: k8s.V1Job = { + metadata: { + name: jobName, + namespace, + labels: { + 'app.kubernetes.io/managed-by': 'shipsec-worker', + 'shipsec.ai/run-id': sanitizeName(context.runId), + 'shipsec.ai/component-ref': sanitizeName(context.componentRef), + }, + }, + spec: { + backoffLimit: 0, // no retries — Temporal handles retry logic + activeDeadlineSeconds: timeoutSeconds, + ttlSecondsAfterFinished: 120, // auto-cleanup after 2 min + template: { + metadata: { + labels: { + 'app.kubernetes.io/managed-by': 'shipsec-worker', + 'shipsec.ai/run-id': sanitizeName(context.runId), + }, + ...(hasGcsFuse ? { annotations: { 'gke-gcsfuse/volumes': 'true' } } : {}), + }, + spec: { + restartPolicy: 'Never', + ...(process.env.K8S_JOB_SERVICE_ACCOUNT + ? { serviceAccountName: process.env.K8S_JOB_SERVICE_ACCOUNT } + : {}), + ...(hasGcsFuse ? { terminationGracePeriodSeconds: 60 } : {}), + ...(process.env.K8S_IMAGE_PULL_SECRET + ? { imagePullSecrets: [{ name: process.env.K8S_IMAGE_PULL_SECRET }] } + : {}), + containers: [ + { + name: 'component', + image: runner.image, + imagePullPolicy: + (process.env.K8S_JOB_IMAGE_PULL_POLICY as 'Always' | 'IfNotPresent' | 'Never') || + 'IfNotPresent', + command: command.length > 0 ? command : undefined, + args: args.length > 0 ? args : undefined, + env: envVars, + tty: true, + volumeMounts, + resources: { + requests: { cpu: '100m', memory: '128Mi' }, + limits: { cpu: '1000m', memory: '2Gi' }, + }, + }, + ], + volumes, + }, + }, + }, + }; + + return { job, writableVolumeMappings, hasGcsFuse }; +} + +/** + * Wait for a Job to complete (or fail/timeout). + * Returns the pod name for log retrieval. + */ +async function waitForJobCompletion( + jobName: string, + namespace: string, + timeoutMs: number, + context: ExecutionContext, +): Promise<{ podName: string; succeeded: boolean }> { + const batch = getBatchApi(); + const core = getCoreApi(); + const deadline = Date.now() + timeoutMs; + + // Find the pod created by this Job + let podName = ''; + while (!podName && Date.now() < deadline) { + const pods = await core.listNamespacedPod({ + namespace, + labelSelector: `job-name=${jobName}`, + }); + if (pods.items.length > 0) { + podName = pods.items[0].metadata?.name || ''; + } + if (!podName) { + await new Promise((r) => setTimeout(r, 1000)); + } + } + + if (!podName) { + throw new TimeoutError(`Timed out waiting for Job pod to appear: ${jobName}`, timeoutMs); + } + + context.logger.info(`[K8sRunner] Job ${jobName} → pod ${podName}`); + + // Stream logs in real-time while waiting + const logPromise = streamPodLogs(podName, namespace, context).catch((err) => { + context.logger.warn(`[K8sRunner] Log streaming error: ${err.message}`); + }); + + // Poll Job status until done + while (Date.now() < deadline) { + const job = await batch.readNamespacedJob({ name: jobName, namespace }); + const status = job.status; + + if (status?.succeeded && status.succeeded > 0) { + await logPromise; + return { podName, succeeded: true }; + } + if (status?.failed && status.failed > 0) { + await logPromise; + return { podName, succeeded: false }; + } + + await new Promise((r) => setTimeout(r, 2000)); + } + + throw new TimeoutError(`Job ${jobName} timed out after ${timeoutMs / 1000}s`, timeoutMs, { + details: { jobName, podName }, + }); +} + +/** + * Stream pod logs to the context logger and terminal collector. + * Uses the K8s Log API with a writable stream to capture output in real-time. + */ +async function streamPodLogs( + podName: string, + namespace: string, + context: ExecutionContext, +): Promise { + const core = getCoreApi(); + const kc = getKubeConfig(); + const log = new k8s.Log(kc); + + // Wait for container to be ready (running or already terminated) + const deadline = Date.now() + 60_000; + let containerTerminated = false; + while (Date.now() < deadline) { + const pod = await core.readNamespacedPod({ name: podName, namespace }); + const cs = pod.status?.containerStatuses?.find((c) => c.name === 'component'); + if (cs?.state?.terminated) { + containerTerminated = true; + break; + } + if (cs?.state?.running) { + break; + } + await new Promise((r) => setTimeout(r, 500)); + } + + let chunkIndex = 0; + let lastTimestamp = Date.now(); + const emitToCollectors = (text: string) => { + if (context.terminalCollector) { + chunkIndex += 1; + const now = Date.now(); + const deltaMs = chunkIndex === 1 ? 0 : Math.max(0, now - lastTimestamp); + lastTimestamp = now; + context.terminalCollector({ + runId: context.runId, + nodeRef: context.componentRef, + stream: 'pty', + chunkIndex, + payload: Buffer.from(text).toString('base64'), + recordedAt: new Date(now).toISOString(), + deltaMs, + origin: 'k8s-job', + runnerKind: 'k8s', + }); + } + if (context.logCollector) { + context.logCollector({ + runId: context.runId, + nodeRef: context.componentRef, + stream: 'stdout', + level: 'info', + message: text, + timestamp: new Date().toISOString(), + }); + } + }; + + // If container already terminated, read final logs instead of following + if (containerTerminated) { + try { + const logResponse = await core.readNamespacedPodLog({ + name: podName, + namespace, + container: 'component', + }); + const logText = typeof logResponse === 'string' ? logResponse : String(logResponse); + if (logText) { + emitToCollectors(logText); + } + } catch (err) { + context.logger.warn( + `[K8sRunner] Failed to read terminated pod logs: ${(err as Error).message}`, + ); + } + return; + } + + // Container is running — attach to PTY for real-time streaming + const { Writable } = await import('stream'); + + const stdoutSink = new Writable({ + write(chunk: Buffer, _encoding, callback) { + emitToCollectors(chunk.toString()); + callback(); + }, + }); + + try { + context.logger.info(`[K8sRunner] Attaching to pod ${podName} with TTY for live PTY stream`); + const attach = new k8s.Attach(kc); + const ws = await attach.attach( + namespace, + podName, + 'component', + stdoutSink, + stdoutSink, + null, + true, + ); + + // Wait for the WebSocket to close (container exits → WS closes) + await new Promise((resolve, reject) => { + ws.onclose = () => { + context.logger.info(`[K8sRunner] Attach WebSocket closed for pod ${podName}`); + resolve(); + }; + ws.onerror = (event) => { + context.logger.warn(`[K8sRunner] Attach WebSocket error: ${String(event)}`); + reject(new Error('Attach WebSocket error')); + }; + }); + } catch (err) { + context.logger.warn( + `[K8sRunner] Attach failed, falling back to log stream: ${(err as Error).message}`, + ); + // Fallback to log API if attach fails + const { PassThrough } = await import('stream'); + const logStream = new PassThrough(); + logStream.on('data', (chunk: Buffer) => { + emitToCollectors(chunk.toString()); + }); + try { + await log.log(namespace, podName, 'component', logStream, { + follow: true, + pretty: false, + timestamps: false, + }); + } catch (logErr) { + context.logger.warn(`[K8sRunner] Log streaming also failed: ${(logErr as Error).message}`); + } + } +} + +/** + * Read final pod logs after completion. + */ +async function readPodLogs(podName: string, namespace: string): Promise { + const core = getCoreApi(); + const logResponse = await core.readNamespacedPodLog({ + name: podName, + namespace, + container: 'component', + }); + // The response can be a string directly + return typeof logResponse === 'string' ? logResponse : String(logResponse); +} + +/** + * Parse the volume data section from pod logs. + * Returns a nested map: mountPath -> (relativeFilePath -> base64Content). + */ +function extractVolumeDataFromLogs(logs: string): Map> { + const FILE_START = '___FILE_START___:'; + const FILE_END = '___FILE_END___'; + + const result = new Map>(); + + const volIdx = logs.lastIndexOf(VOLUME_DELIMITER); + if (volIdx === -1) return result; + + const volSection = logs.slice(volIdx + VOLUME_DELIMITER.length); + const lines = volSection.split('\n'); + + let currentMount = ''; + let currentFile = ''; + let currentData: string[] = []; + let inFile = false; + + for (const line of lines) { + if (line.startsWith(FILE_START)) { + // Parse mount path and relative path + const rest = line.slice(FILE_START.length); + const firstColon = rest.indexOf(':'); + if (firstColon === -1) continue; + currentMount = rest.slice(0, firstColon); + currentFile = rest.slice(firstColon + 1); + currentData = []; + inFile = true; + } else if (line.trim() === FILE_END && inFile) { + // Save the file + if (!result.has(currentMount)) { + result.set(currentMount, new Map()); + } + result.get(currentMount)!.set(currentFile, currentData.join('\n')); + inFile = false; + } else if (inFile) { + currentData.push(line); + } + } + + return result; +} + +/** + * Write captured volume data back to their backing ConfigMaps. + * This allows volume.readFiles() to access output data after the pod terminates. + */ +async function writeBackVolumeData( + volumeData: Map>, + writableVolumeMappings: Map, + namespace: string, + context: ExecutionContext, +): Promise { + const core = getCoreApi(); + + for (const [mountPath, files] of volumeData) { + const cmName = writableVolumeMappings.get(mountPath); + if (!cmName) continue; + + const binaryData: Record = {}; + + for (const [relPath, base64Content] of files) { + // Flatten path separators same as IsolatedK8sVolume + const key = relPath.replace(/\//g, '__'); + // Store as binaryData (base64) to handle any file type + binaryData[key] = base64Content; + } + + try { + // Read existing ConfigMap and merge + const existing = await core.readNamespacedConfigMap({ name: cmName, namespace }); + const body: k8s.V1ConfigMap = { + ...existing, + data: { ...(existing.data || {}) }, + binaryData: { ...(existing.binaryData || {}), ...binaryData }, + }; + await core.replaceNamespacedConfigMap({ name: cmName, namespace, body }); + context.logger.info(`[K8sRunner] Wrote back ${files.size} files to ConfigMap ${cmName}`); + } catch (err) { + context.logger.warn( + `[K8sRunner] Failed to write back volume data to ${cmName}: ${(err as Error).message}`, + ); + } + } +} + +/** + * Parse component output from pod logs. + * Looks for the OUTPUT_DELIMITER marker — everything after it is the JSON output. + * Falls back to parsing the full stdout as JSON. + */ +function parseOutputFromLogs(logs: string, context: ExecutionContext): O { + // Strip volume data section if present (comes after output) + let cleanLogs = logs; + const volIdx = cleanLogs.lastIndexOf(VOLUME_DELIMITER); + if (volIdx !== -1) { + cleanLogs = cleanLogs.slice(0, volIdx); + } + + // Look for the output delimiter + const delimiterIdx = cleanLogs.lastIndexOf(OUTPUT_DELIMITER); + if (delimiterIdx !== -1) { + const outputStr = cleanLogs.slice(delimiterIdx + OUTPUT_DELIMITER.length).trim(); + if (outputStr) { + try { + return JSON.parse(outputStr) as O; + } catch { + // Not JSON — return delimited content as raw string + // (e.g., plain text domain lists from -o file output) + return outputStr as unknown as O; + } + } + } + + // Fallback: try parsing the last line as JSON + const lines = cleanLogs.trim().split('\n'); + for (let i = lines.length - 1; i >= 0; i--) { + const line = lines[i].trim(); + if (line.startsWith('{') || line.startsWith('[')) { + try { + return JSON.parse(line) as O; + } catch { + continue; + } + } + } + + // Fallback: return raw logs as string output + context.logger.warn('[K8sRunner] No structured output found, returning raw stdout'); + return cleanLogs.trim() as unknown as O; +} + +/** + * Wait for the GCS FUSE sidecar container to terminate after the main + * container exits. This ensures all writes are flushed to GCS before + * the worker reads output via the GCS SDK. + */ +async function waitForGcsFuseFlush( + podName: string, + namespace: string, + context: ExecutionContext, +): Promise { + const core = getCoreApi(); + const deadline = Date.now() + 60_000; // max 60s wait + + context.logger.info(`[K8sRunner] Waiting for GCS FUSE sidecar flush on pod ${podName}`); + + while (Date.now() < deadline) { + const pod = await core.readNamespacedPod({ name: podName, namespace }); + // GCS FUSE sidecar may appear in containerStatuses (K8s ≥1.28 native sidecar) + // or initContainerStatuses (older injection approach) + const allStatuses = [ + ...(pod.status?.containerStatuses ?? []), + ...(pod.status?.initContainerStatuses ?? []), + ]; + const sidecar = allStatuses.find((c) => c.name === 'gke-gcsfuse-sidecar'); + + if (!sidecar) { + // No sidecar found — GCS FUSE may not have been injected, skip wait + context.logger.info(`[K8sRunner] No GCS FUSE sidecar found, skipping flush wait`); + return; + } + + if (sidecar.state?.terminated) { + context.logger.info(`[K8sRunner] GCS FUSE sidecar terminated, flush complete`); + return; + } + + await new Promise((r) => setTimeout(r, 2000)); + } + + context.logger.warn(`[K8sRunner] GCS FUSE sidecar did not terminate within 60s, proceeding`); +} + +/** + * Clean up resources created for a Job execution. + */ +async function cleanup( + jobName: string, + configMapName: string, + namespace: string, + context: ExecutionContext, +): Promise { + const batch = getBatchApi(); + const core = getCoreApi(); + + try { + await batch.deleteNamespacedJob({ + name: jobName, + namespace, + body: { propagationPolicy: 'Background' }, + }); + } catch (err) { + context.logger.warn(`[K8sRunner] Failed to delete Job ${jobName}: ${(err as Error).message}`); + } + + try { + await core.deleteNamespacedConfigMap({ name: configMapName, namespace }); + } catch (err) { + context.logger.warn( + `[K8sRunner] Failed to delete ConfigMap ${configMapName}: ${(err as Error).message}`, + ); + } +} + +/** + * Execute a component as a Kubernetes Job. + * + * Drop-in replacement for runComponentInDocker — same signature, + * registered via setDockerRunnerOverride() at worker startup. + */ +export async function runComponentInK8sJob( + runner: DockerRunnerConfig, + params: I, + context: ExecutionContext, +): Promise { + const namespace = getJobNamespace(); + const jobName = generateJobName(context, runner.image); + const configMapName = `${jobName}-input`; + const timeoutMs = (runner.timeoutSeconds || 300) * 1000; + + context.logger.info( + `[K8sRunner] Creating Job ${jobName} in ${namespace} (image: ${runner.image})`, + ); + context.emitProgress(`Launching K8s Job: ${runner.image}`); + + try { + // 1. Create input ConfigMap + await createInputConfigMap(configMapName, namespace, params); + context.logger.info(`[K8sRunner] Created ConfigMap ${configMapName}`); + + // 2. Create Job + const { + job: jobSpec, + writableVolumeMappings, + hasGcsFuse: hasGcsFuseVolume, + } = buildJobSpec(jobName, namespace, configMapName, runner, context); + await getBatchApi().createNamespacedJob({ namespace, body: jobSpec }); + context.logger.info(`[K8sRunner] Created Job ${jobName}`); + + // 3. Wait for completion + const { podName, succeeded } = await waitForJobCompletion( + jobName, + namespace, + timeoutMs, + context, + ); + + // 4. Read final logs + const logs = await readPodLogs(podName, namespace); + + if (!succeeded) { + context.logger.error(`[K8sRunner] Job ${jobName} failed`); + throw new ContainerError(`K8s Job failed: ${jobName}`, { + details: { jobName, podName, logs: logs.slice(-500) }, + }); + } + + context.logger.info(`[K8sRunner] Job ${jobName} completed successfully`); + context.emitProgress('K8s Job completed'); + + // 4.5a. Wait for GCS FUSE sidecar to flush writes before reading output + if (hasGcsFuseVolume) { + await waitForGcsFuseFlush(podName, namespace, context); + } + + // 4.5b. Write back writable volume data to ConfigMaps + // Must happen BEFORE cleanup so volume.readFiles() can access updated ConfigMaps + if (writableVolumeMappings.size > 0) { + const volumeData = extractVolumeDataFromLogs(logs); + if (volumeData.size > 0) { + await writeBackVolumeData(volumeData, writableVolumeMappings, namespace, context); + } + } + + // 5. Parse output + return parseOutputFromLogs(logs, context); + } finally { + // 6. Cleanup + await cleanup(jobName, configMapName, namespace, context); + } +} diff --git a/worker/src/utils/k8s-volume.ts b/worker/src/utils/k8s-volume.ts new file mode 100644 index 000000000..248a7e3df --- /dev/null +++ b/worker/src/utils/k8s-volume.ts @@ -0,0 +1,235 @@ +/** + * IsolatedK8sVolume — K8s-native replacement for IsolatedContainerVolume. + * + * Uses ConfigMaps instead of Docker named volumes. Same interface as + * IsolatedContainerVolume so components can swap transparently. + * + * Limits: + * - ConfigMap total size: 1 MiB (sufficient for target lists, configs, templates) + * - For binary data or large payloads, consider using a PVC-based approach + */ +import * as k8s from '@kubernetes/client-node'; +import { ValidationError, ConfigurationError, ContainerError } from '@shipsec/component-sdk'; + +let _kc: k8s.KubeConfig | null = null; +let _coreApi: k8s.CoreV1Api | null = null; + +function getKubeConfig(): k8s.KubeConfig { + if (!_kc) { + _kc = new k8s.KubeConfig(); + _kc.loadFromCluster(); + } + return _kc; +} + +function getCoreApi(): k8s.CoreV1Api { + if (!_coreApi) _coreApi = getKubeConfig().makeApiClient(k8s.CoreV1Api); + return _coreApi; +} + +function getNamespace(): string { + return process.env.K8S_JOB_NAMESPACE || 'shipsec-workloads'; +} + +function sanitizeName(raw: string): string { + return raw + .toLowerCase() + .replace(/[^a-z0-9-]/g, '-') + .replace(/-+/g, '-') + .replace(/^-|-$/g, '') + .slice(0, 53); +} + +export class IsolatedK8sVolume { + private configMapName?: string; + private isInitialized = false; + private namespace: string; + + constructor( + private tenantId: string, + private runId: string, + ) { + if (!/^[a-zA-Z0-9_-]+$/.test(tenantId)) { + throw new ValidationError( + 'Invalid tenant ID: must contain only alphanumeric characters, hyphens, and underscores', + { + fieldErrors: { + tenantId: ['must contain only alphanumeric characters, hyphens, and underscores'], + }, + }, + ); + } + if (!/^[a-zA-Z0-9_-]+$/.test(runId)) { + throw new ValidationError( + 'Invalid run ID: must contain only alphanumeric characters, hyphens, and underscores', + { + fieldErrors: { + runId: ['must contain only alphanumeric characters, hyphens, and underscores'], + }, + }, + ); + } + this.namespace = getNamespace(); + } + + /** + * Creates a ConfigMap containing the provided files. + * Text files go in `data`, binary files go in `binaryData`. + */ + async initialize(files: Record): Promise { + if (this.isInitialized) { + throw new ConfigurationError('Volume already initialized', { + details: { configMapName: this.configMapName, tenantId: this.tenantId, runId: this.runId }, + }); + } + + const timestamp = Date.now(); + const tenantShort = sanitizeName(this.tenantId); + const runShort = sanitizeName(this.runId); + this.configMapName = `vol-${tenantShort}-${runShort}-${timestamp}`.slice(0, 63); + + try { + const data: Record = {}; + const binaryData: Record = {}; + + for (const [filename, content] of Object.entries(files)) { + this.validateFilename(filename); + + // ConfigMap keys can't have slashes — flatten paths + const key = filename.replace(/\//g, '__'); + + if (typeof content === 'string') { + data[key] = content; + } else { + // Buffer → base64 for binaryData + binaryData[key] = content.toString('base64'); + } + } + + const body: k8s.V1ConfigMap = { + metadata: { + name: this.configMapName, + namespace: this.namespace, + labels: { + 'app.kubernetes.io/managed-by': 'shipsec-worker', + 'shipsec.ai/purpose': 'isolated-volume', + 'shipsec.ai/tenant': tenantShort, + 'shipsec.ai/run': runShort, + }, + }, + data: Object.keys(data).length > 0 ? data : undefined, + binaryData: Object.keys(binaryData).length > 0 ? binaryData : undefined, + }; + + await getCoreApi().createNamespacedConfigMap({ + namespace: this.namespace, + body, + }); + + this.isInitialized = true; + return this.configMapName; + } catch (error) { + if (this.configMapName) { + await this.cleanup().catch(() => {}); + } + throw new ContainerError( + `Failed to initialize K8s volume: ${error instanceof Error ? error.message : String(error)}`, + { + cause: error instanceof Error ? error : undefined, + details: { tenantId: this.tenantId, runId: this.runId }, + }, + ); + } + } + + private validateFilename(filename: string): void { + if (filename.includes('..') || filename.startsWith('/')) { + throw new ValidationError(`Invalid filename (path traversal): ${filename}`, { + fieldErrors: { filename: ['path traversal not allowed'] }, + }); + } + const safePattern = /^[a-zA-Z0-9._/-]+$/; + if (!safePattern.test(filename)) { + throw new ValidationError(`Invalid filename (contains unsafe characters): ${filename}`, { + fieldErrors: { filename: ['contains unsafe characters'] }, + }); + } + } + + /** + * Read files from the ConfigMap. + */ + async readFiles(filenames: string[]): Promise> { + if (!this.configMapName) { + throw new ConfigurationError('Volume not initialized'); + } + + const cm = await getCoreApi().readNamespacedConfigMap({ + name: this.configMapName, + namespace: this.namespace, + }); + + const results: Record = {}; + for (const filename of filenames) { + const key = filename.replace(/\//g, '__'); + if (cm.data?.[key]) { + results[filename] = cm.data[key]; + } else if (cm.binaryData?.[key]) { + results[filename] = Buffer.from(cm.binaryData[key], 'base64').toString('utf-8'); + } + } + return results; + } + + /** + * Returns a bind mount string compatible with the K8s runner. + * Format: "configmap:::" + */ + getBindMount(containerPath = '/inputs', readOnly = true): string { + if (!this.configMapName) { + throw new ConfigurationError('Volume not initialized'); + } + const mode = readOnly ? 'ro' : 'rw'; + return `configmap:${this.configMapName}:${containerPath}:${mode}`; + } + + /** + * Returns volume config for the runner. The K8s runner recognizes the + * "configmap:" prefix in source and mounts the ConfigMap accordingly. + */ + getVolumeConfig(containerPath = '/inputs', readOnly = true) { + if (!this.configMapName) { + throw new ConfigurationError('Volume not initialized'); + } + return { + source: `configmap:${this.configMapName}`, + target: containerPath, + readOnly, + }; + } + + /** + * Delete the ConfigMap. + */ + async cleanup(): Promise { + if (!this.configMapName) return; + + try { + await getCoreApi().deleteNamespacedConfigMap({ + name: this.configMapName, + namespace: this.namespace, + }); + } catch (error) { + console.error( + `Failed to cleanup K8s volume ${this.configMapName}: ${error instanceof Error ? error.message : String(error)}`, + ); + } finally { + this.isInitialized = false; + this.configMapName = undefined; + } + } + + getVolumeName(): string | undefined { + return this.configMapName; + } +}