diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index 5097c77..0a1b70b 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -39,6 +39,22 @@ jobs: with: version: latest + # TODO: Remove these two steps once @posthog/warlock is published to npm. + # They exist because warlock is currently installed as a private git dependency, + # so the CI runner needs credentials to clone it. + - name: Generate token for private dependencies + id: app-token + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ secrets.GH_APP_POSTHOG_WIZARD_CI_BOT_APP_ID }} + private-key: ${{ secrets.GH_APP_POSTHOG_WIZARD_CI_BOT_PRIVATE_KEY }} + repositories: warlock + + - name: Configure git auth for private deps + env: + APP_TOKEN: ${{ steps.app-token.outputs.token }} + run: git config --global url."https://x-access-token:${APP_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Install dependencies run: pnpm install @@ -114,8 +130,8 @@ jobs: env: BUILD_VERSION: ${{ steps.version.outputs.version }} - - name: Scan skills for prompt injection - run: bash scripts/scan-prompt-injection.sh dist/skills + - name: Scan skills with Warlock + run: node scripts/scan-warlock.js dist/skills - name: List build artifacts run: | diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c893800..a376753 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,14 +27,30 @@ jobs: with: version: latest + # TODO: Remove these two steps once @posthog/warlock is published to npm. + # They exist because warlock is currently installed as a private git dependency, + # so the CI runner needs credentials to clone it. + - name: Generate token for private dependencies + id: app-token + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ secrets.GH_APP_POSTHOG_WIZARD_CI_BOT_APP_ID }} + private-key: ${{ secrets.GH_APP_POSTHOG_WIZARD_CI_BOT_PRIVATE_KEY }} + repositories: warlock + + - name: Configure git auth for private deps + env: + APP_TOKEN: ${{ steps.app-token.outputs.token }} + run: git config --global url."https://x-access-token:${APP_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Install dependencies run: pnpm install - name: Build run: pnpm build - - name: Scan skills for prompt injection - run: bash scripts/scan-prompt-injection.sh dist/skills + - name: Scan skills with Warlock + run: node scripts/scan-warlock.js dist/skills - name: Lint env var naming conventions run: bash scripts/lint-env-naming.sh diff --git a/package.json b/package.json index 2551d3d..450dbae 100644 --- a/package.json +++ b/package.json @@ -10,13 +10,17 @@ "test:plugins:watch": "vitest scripts/plugins/tests", "test:skills": "vitest run scripts/lib/tests", "test:skills:watch": "vitest scripts/lib/tests", - "test": "vitest run scripts/plugins/tests scripts/lib/tests" + "test": "vitest run scripts/plugins/tests scripts/lib/tests", + "security-scan": "node scripts/scan-warlock.js", + "security-scan:skills": "node scripts/scan-warlock.js dist/skills" }, "devDependencies": { "archiver": "^7.0.1", "vitest": "^2.0.0" }, "dependencies": { + "@anthropic-ai/sdk": "^0.95.0", + "@posthog/warlock": "git+https://github.com/PostHog/warlock.git", "gray-matter": "^4.0.3", "js-yaml": "^4.1.1" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b50ac0e..7ca4f84 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,12 @@ importers: .: dependencies: + '@anthropic-ai/sdk': + specifier: ^0.95.0 + version: 0.95.0 + '@posthog/warlock': + specifier: git+https://github.com/PostHog/warlock.git + version: git+https://github.com/PostHog/warlock.git#c47d997438a8aafc1bb42f2fc18c15a4ed0b4fd6 gray-matter: specifier: ^4.0.3 version: 4.0.3 @@ -24,6 +30,19 @@ importers: packages: + '@anthropic-ai/sdk@0.95.0': + resolution: {integrity: sha512-7It2B76OFJH9jC/a0TicXFMq0ZZM25ei+i/mK7JnsE1Ibmo0Yfkqm+DXOHeU/ZxxKwLLGPP6qaAvKmQmgV6XhA==} + hasBin: true + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + peerDependenciesMeta: + zod: + optional: true + + '@babel/runtime@7.29.2': + resolution: {integrity: sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==} + engines: {node: '>=6.9.0'} + '@esbuild/aix-ppc64@0.21.5': resolution: {integrity: sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==} engines: {node: '>=12'} @@ -173,6 +192,11 @@ packages: resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} + '@posthog/warlock@git+https://github.com/PostHog/warlock.git#c47d997438a8aafc1bb42f2fc18c15a4ed0b4fd6': + resolution: {commit: c47d997438a8aafc1bb42f2fc18c15a4ed0b4fd6, repo: https://github.com/PostHog/warlock.git, type: git} + version: 0.0.0 + engines: {node: ^20.20.0 || >=22.22.0} + '@rollup/rollup-android-arm-eabi@4.56.0': resolution: {integrity: sha512-LNKIPA5k8PF1+jAFomGe3qN3bbIgJe/IlpDBwuVjrDKrJhVWywgnJvflMt/zkbVNLFtF1+94SljYQS6e99klnw==} cpu: [arm] @@ -207,66 +231,79 @@ packages: resolution: {integrity: sha512-E8jKK87uOvLrrLN28jnAAAChNq5LeCd2mGgZF+fGF5D507WlG/Noct3lP/QzQ6MrqJ5BCKNwI9ipADB6jyiq2A==} cpu: [arm] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.56.0': resolution: {integrity: sha512-jQosa5FMYF5Z6prEpTCCmzCXz6eKr/tCBssSmQGEeozA9tkRUty/5Vx06ibaOP9RCrW1Pvb8yp3gvZhHwTDsJw==} cpu: [arm] os: [linux] + libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.56.0': resolution: {integrity: sha512-uQVoKkrC1KGEV6udrdVahASIsaF8h7iLG0U0W+Xn14ucFwi6uS539PsAr24IEF9/FoDtzMeeJXJIBo5RkbNWvQ==} cpu: [arm64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.56.0': resolution: {integrity: sha512-vLZ1yJKLxhQLFKTs42RwTwa6zkGln+bnXc8ueFGMYmBTLfNu58sl5/eXyxRa2RarTkJbXl8TKPgfS6V5ijNqEA==} cpu: [arm64] os: [linux] + libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.56.0': resolution: {integrity: sha512-FWfHOCub564kSE3xJQLLIC/hbKqHSVxy8vY75/YHHzWvbJL7aYJkdgwD/xGfUlL5UV2SB7otapLrcCj2xnF1dg==} cpu: [loong64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.56.0': resolution: {integrity: sha512-z1EkujxIh7nbrKL1lmIpqFTc/sr0u8Uk0zK/qIEFldbt6EDKWFk/pxFq3gYj4Bjn3aa9eEhYRlL3H8ZbPT1xvA==} cpu: [loong64] os: [linux] + libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.56.0': resolution: {integrity: sha512-iNFTluqgdoQC7AIE8Q34R3AuPrJGJirj5wMUErxj22deOcY7XwZRaqYmB6ZKFHoVGqRcRd0mqO+845jAibKCkw==} cpu: [ppc64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.56.0': resolution: {integrity: sha512-MtMeFVlD2LIKjp2sE2xM2slq3Zxf9zwVuw0jemsxvh1QOpHSsSzfNOTH9uYW9i1MXFxUSMmLpeVeUzoNOKBaWg==} cpu: [ppc64] os: [linux] + libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.56.0': resolution: {integrity: sha512-in+v6wiHdzzVhYKXIk5U74dEZHdKN9KH0Q4ANHOTvyXPG41bajYRsy7a8TPKbYPl34hU7PP7hMVHRvv/5aCSew==} cpu: [riscv64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.56.0': resolution: {integrity: sha512-yni2raKHB8m9NQpI9fPVwN754mn6dHQSbDTwxdr9SE0ks38DTjLMMBjrwvB5+mXrX+C0npX0CVeCUcvvvD8CNQ==} cpu: [riscv64] os: [linux] + libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.56.0': resolution: {integrity: sha512-zhLLJx9nQPu7wezbxt2ut+CI4YlXi68ndEve16tPc/iwoylWS9B3FxpLS2PkmfYgDQtosah07Mj9E0khc3Y+vQ==} cpu: [s390x] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.56.0': resolution: {integrity: sha512-MVC6UDp16ZSH7x4rtuJPAEoE1RwS8N4oK9DLHy3FTEdFoUTCFVzMfJl/BVJ330C+hx8FfprA5Wqx4FhZXkj2Kw==} cpu: [x64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-musl@4.56.0': resolution: {integrity: sha512-ZhGH1eA4Qv0lxaV00azCIS1ChedK0V32952Md3FtnxSqZTBTd6tgil4nZT5cU8B+SIw3PFYkvyR4FKo2oyZIHA==} cpu: [x64] os: [linux] + libc: [musl] '@rollup/rollup-openbsd-x64@4.56.0': resolution: {integrity: sha512-O16XcmyDeFI9879pEcmtWvD/2nyxR9mF7Gs44lf1vGGx8Vg2DRNx11aVXBEqOQhWb92WN4z7fW/q4+2NYzCbBA==} @@ -298,9 +335,15 @@ packages: cpu: [x64] os: [win32] + '@stablelib/base64@1.0.1': + resolution: {integrity: sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==} + '@types/estree@1.0.8': resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} + '@virustotal/yara-x@1.15.0': + resolution: {integrity: sha512-tR+Ue5ci9bURbD7/qjXY0VLVXDUP+NK/uvvgjX6UvSXN+3msfMtpAMaiCdsltFEciU+tXaZeGbHu3N07bvfjsw==} + '@vitest/expect@2.1.9': resolution: {integrity: sha512-UJCIkTBenHeKT1TTlKMJWy1laZewsRIzYighyYiJKZreqtdxSos/S1t+ktRMQWu2CKqaarrkeszJx1cgC5tGZw==} @@ -502,6 +545,9 @@ packages: fast-fifo@1.3.2: resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} + fast-sha256@1.3.0: + resolution: {integrity: sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==} + foreground-child@3.3.1: resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} engines: {node: '>=14'} @@ -557,6 +603,10 @@ packages: resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==} hasBin: true + json-schema-to-ts@3.1.1: + resolution: {integrity: sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==} + engines: {node: '>=16'} + kind-of@6.0.3: resolution: {integrity: sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==} engines: {node: '>=0.10.0'} @@ -683,6 +733,9 @@ packages: stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + standardwebhooks@1.0.0: + resolution: {integrity: sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg==} + std-env@3.10.0: resolution: {integrity: sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==} @@ -739,6 +792,9 @@ packages: resolution: {integrity: sha512-n1cw8k1k0x4pgA2+9XrOkFydTerNcJ1zWCO5Nn9scWHTD+5tp8dghT2x1uduQePZTZgd3Tupf+x9BxJjeJi77Q==} engines: {node: '>=14.0.0'} + ts-algebra@2.0.0: + resolution: {integrity: sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==} + util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} @@ -827,6 +883,13 @@ packages: snapshots: + '@anthropic-ai/sdk@0.95.0': + dependencies: + json-schema-to-ts: 3.1.1 + standardwebhooks: 1.0.0 + + '@babel/runtime@7.29.2': {} + '@esbuild/aix-ppc64@0.21.5': optional: true @@ -910,6 +973,10 @@ snapshots: '@pkgjs/parseargs@0.11.0': optional: true + '@posthog/warlock@git+https://github.com/PostHog/warlock.git#c47d997438a8aafc1bb42f2fc18c15a4ed0b4fd6': + dependencies: + '@virustotal/yara-x': 1.15.0 + '@rollup/rollup-android-arm-eabi@4.56.0': optional: true @@ -985,8 +1052,12 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.56.0': optional: true + '@stablelib/base64@1.0.1': {} + '@types/estree@1.0.8': {} + '@virustotal/yara-x@1.15.0': {} + '@vitest/expect@2.1.9': dependencies: '@vitest/spy': 2.1.9 @@ -1198,6 +1269,8 @@ snapshots: fast-fifo@1.3.2: {} + fast-sha256@1.3.0: {} + foreground-child@3.3.1: dependencies: cross-spawn: 7.0.6 @@ -1253,6 +1326,11 @@ snapshots: dependencies: argparse: 2.0.1 + json-schema-to-ts@3.1.1: + dependencies: + '@babel/runtime': 7.29.2 + ts-algebra: 2.0.0 + kind-of@6.0.3: {} lazystream@1.0.1: @@ -1388,6 +1466,11 @@ snapshots: stackback@0.0.2: {} + standardwebhooks@1.0.0: + dependencies: + '@stablelib/base64': 1.0.1 + fast-sha256: 1.3.0 + std-env@3.10.0: {} streamx@2.23.0: @@ -1454,6 +1537,8 @@ snapshots: tinyspy@3.0.2: {} + ts-algebra@2.0.0: {} + util-deprecate@1.0.2: {} vite-node@2.1.9: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml new file mode 100644 index 0000000..3d89da1 --- /dev/null +++ b/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +onlyBuiltDependencies: + - "@posthog/warlock" diff --git a/scripts/scan-prompt-injection.sh b/scripts/scan-prompt-injection.sh deleted file mode 100755 index 3172a06..0000000 --- a/scripts/scan-prompt-injection.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env bash -# -# Scans skill ZIP archives for prompt injection patterns and malicious content. -# Usage: bash scripts/scan-prompt-injection.sh [skills-directory] -# -# Exits 0 if clean, 1 if any matches found. - -set -euo pipefail - -SKILLS_DIR="${1:-dist/skills}" -TMPDIR_BASE=$(mktemp -d) -MATCHES_FILE=$(mktemp) - -trap 'rm -rf "$TMPDIR_BASE" "$MATCHES_FILE"' EXIT - -if [ ! -d "$SKILLS_DIR" ]; then - echo "::error::Skills directory not found: $SKILLS_DIR" - exit 1 -fi - -ZIP_COUNT=0 -for _ in "$SKILLS_DIR"/*.zip; do - ZIP_COUNT=$((ZIP_COUNT + 1)) -done - -if [ "$ZIP_COUNT" -eq 0 ]; then - echo "::warning::No ZIP files found in $SKILLS_DIR" - exit 0 -fi - -echo "Scanning $ZIP_COUNT skill archives for prompt injection patterns..." -echo "" - -# Fixed-string patterns (case insensitive): "category|pattern" -FIXED_PATTERNS=( - "direct-instruction-override|ignore previous instructions" - "direct-instruction-override|forget your instructions" - "direct-instruction-override|override your rules" - "role-manipulation|act as a different" - "role-manipulation|new instructions:" - "wizard-specific-manipulation|do not install posthog" - "wizard-specific-manipulation|uninstall posthog" - "wizard-specific-manipulation|delete the posthog" - "tool-abuse|run the following command" - "tool-abuse|execute this shell command" -) - -# Regex patterns (case insensitive, extended grep): "category|label|regex" -# These need word boundaries or context to avoid false positives on normal prose. -REGEX_PATTERNS=( - "direct-instruction-override|disregard all|disregard all (previous|prior|above|system)" - "role-manipulation|you are now|you are now (a|an|my) " - "wizard-specific-manipulation|skip posthog|skip posthog[^_ a-z]" - "wizard-specific-manipulation|remove posthog|remove posthog([^_ a-z]|$)" - "obfuscated-injection|Base64 block (100+ chars) in comment|(//|#|/\\*)[[:space:]]*[A-Za-z0-9+/=]{100,}" -) - -report_match() { - local zip_name="$1" rel_path="$2" line_num="$3" category="$4" pattern="$5" content="$6" - echo "::error file=${rel_path},line=${line_num}::PROMPT INJECTION DETECTED" - echo " Archive: $zip_name" - echo " File: $rel_path" - echo " Line: $line_num" - echo " Category: $category" - echo " Pattern: $pattern" - echo " Content: ${content:0:200}" - echo "" - echo "1" >> "$MATCHES_FILE" -} - -for zip_file in "$SKILLS_DIR"/*.zip; do - [ -e "$zip_file" ] || continue - zip_name=$(basename "$zip_file") - extract_dir="$TMPDIR_BASE/${zip_name%.zip}" - mkdir -p "$extract_dir" - - unzip -q -o "$zip_file" -d "$extract_dir" 2>/dev/null || { - echo "::warning::Failed to extract $zip_name, skipping" - continue - } - - while IFS= read -r -d '' file; do - rel_path="${file#"$extract_dir"/}" - - # Check fixed-string patterns (case insensitive) - for entry in "${FIXED_PATTERNS[@]}"; do - category="${entry%%|*}" - pattern="${entry#*|}" - - grep -inF "$pattern" "$file" 2>/dev/null | head -5 | while IFS=: read -r line_num line_content; do - report_match "$zip_name" "$rel_path" "$line_num" "$category" "\"$pattern\"" "$line_content" - done || true - done - - # Check regex patterns (case insensitive) - for entry in "${REGEX_PATTERNS[@]}"; do - category="${entry%%|*}" - rest="${entry#*|}" - label="${rest%%|*}" - regex="${rest#*|}" - - grep -inE "$regex" "$file" 2>/dev/null | head -5 | while IFS=: read -r line_num line_content; do - report_match "$zip_name" "$rel_path" "$line_num" "$category" "\"$label\"" "$line_content" - done || true - done - - done < <(find "$extract_dir" -type f \( \ - -name '*.md' -o -name '*.txt' -o -name '*.yaml' -o -name '*.yml' \ - -o -name '*.json' -o -name '*.js' -o -name '*.ts' -o -name '*.py' \ - -o -name '*.rb' -o -name '*.sh' \ - \) -print0) -done - -echo "---" -MATCH_COUNT=$(wc -l < "$MATCHES_FILE" | tr -d ' ') -if [ "$MATCH_COUNT" -gt 0 ]; then - echo "FAILED: $MATCH_COUNT prompt injection pattern(s) detected in skill archives." - echo "Fix the flagged content before releasing." - exit 1 -else - echo "PASSED: No prompt injection patterns found in $ZIP_COUNT skill archives." - exit 0 -fi diff --git a/scripts/scan-warlock.js b/scripts/scan-warlock.js new file mode 100644 index 0000000..1171d40 --- /dev/null +++ b/scripts/scan-warlock.js @@ -0,0 +1,334 @@ +#!/usr/bin/env node + +/** + * Scan skills for security threats using Warlock. + * + * Two modes: + * node scripts/scan-warlock.js dist/skills # Scan built skill ZIPs (build/CI) + * node scripts/scan-warlock.js path/to/file.md # Scan specific file(s) (local) + * + * Exits 0 if clean, 1 if threats found. + */ + +const fs = require("node:fs"); +const path = require("node:path"); +const { execFileSync } = require("node:child_process"); +const os = require("node:os"); +const Anthropic = require("@anthropic-ai/sdk"); + +const TEXT_EXTENSIONS = new Set([ + ".md", + ".txt", + ".yaml", + ".yml", + ".json", + ".js", + ".ts", + ".py", + ".rb", + ".sh", +]); + +const isCI = Boolean(process.env.CI); + +// -- LLM triage setup -- +// Uses the PostHog LLM gateway to triage warlock +// matches. The LLM decides whether each match is a real threat or a +// false positive +// +// Gateway URL pattern matches the wizard: +// US: https://gateway.us.posthog.com/wizard +// EU: https://gateway.eu.posthog.com/wizard +// Local: http://localhost:3308/wizard + +function getGatewayUrl() { + const host = process.env.POSTHOG_HOST || "https://us.posthog.com"; + let hostname = ""; + + try { + hostname = new URL(host).hostname.toLowerCase(); + } catch { + try { + hostname = new URL(`https://${host}`).hostname.toLowerCase(); + } catch { + hostname = ""; + } + } + + if (hostname === "localhost" || hostname === "127.0.0.1" || hostname === "::1") { + return "http://localhost:3308/wizard"; + } + if (hostname === "eu.posthog.com" || hostname === "eu.i.posthog.com") { + return "https://gateway.eu.posthog.com/wizard"; + } + return "https://gateway.us.posthog.com/wizard"; +} + +function createLLMProvider() { + const apiKey = process.env.POSTHOG_API_KEY; + if (!apiKey) return null; + + const client = new Anthropic({ + baseURL: getGatewayUrl(), + apiKey, + }); + + return async (prompt) => { + const res = await client.messages.create({ + model: "claude-haiku-4-5-20251001", + max_tokens: 16384, + messages: [{ role: "user", content: prompt }], + }); + return res.content[0].text; + }; +} + +// -- Colors (disabled in CI where annotations do the work) -- + +const color = isCI + ? { + red: (s) => s, + yellow: (s) => s, + green: (s) => s, + bold: (s) => s, + dim: (s) => s, + } + : { + red: (s) => `\x1b[31m${s}\x1b[0m`, + yellow: (s) => `\x1b[33m${s}\x1b[0m`, + green: (s) => `\x1b[32m${s}\x1b[0m`, + bold: (s) => `\x1b[1m${s}\x1b[0m`, + dim: (s) => `\x1b[2m${s}\x1b[0m`, + }; + +// -- Helpers -- + +/** Recursively collect text files from a directory. */ +function collectTextFiles(dir) { + const files = []; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + files.push(...collectTextFiles(full)); + } else if (TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) { + files.push(full); + } + } + return files; +} + +/** Extract a ZIP and return the temp directory it was extracted to. */ +function extractZip(zipPath, tmpDir) { + const name = path.basename(zipPath, ".zip"); + const dest = path.join(tmpDir, name); + fs.mkdirSync(dest, { recursive: true }); + try { + execFileSync("unzip", ["-q", "-o", zipPath, "-d", dest], { stdio: "pipe" }); + return dest; + } catch { + console.warn( + isCI + ? `::warning::Failed to extract ${path.basename(zipPath)}` + : ` Warning: failed to extract ${path.basename(zipPath)}, skipping`, + ); + return null; + } +} + +function reportMatch(filePath, match, isFalsePositive = false) { + const { rule, metadata } = match; + const sev = metadata.severity || "unknown"; + const cat = metadata.category || "unknown"; + const triageReason = match.triage?.reason; + + if (isCI) { + const level = isFalsePositive ? "warning" : "error"; + const tag = isFalsePositive ? "FALSE POSITIVE" : "THREAT DETECTED"; + console.log( + `::${level} file=${filePath}::${tag} [${sev}] ${rule}: ${metadata.description || ""}`, + ); + } + + const label = isFalsePositive + ? ` ${color.yellow(color.bold("FALSE POSITIVE"))} [${color.yellow(sev)}] ${cat}` + : ` ${color.red(color.bold("THREAT DETECTED"))} [${color.red(sev)}] ${cat}`; + console.log(label); + console.log(` ${color.dim("Rule:")} ${rule}`); + console.log(` ${color.dim("File:")} ${filePath}`); + if (metadata.description) + console.log(` ${color.dim("Why:")} ${metadata.description}`); + if (triageReason) console.log(` ${color.dim("Triage:")} ${triageReason}`); + if (metadata.remediation) + console.log(` ${color.dim("Fix:")} ${metadata.remediation}`); + if (metadata.action) + console.log(` ${color.dim("Action:")} ${metadata.action}`); + console.log(""); +} + +// -- Main -- + +async function main() { + const { scan, triageMatches } = require("@posthog/warlock"); + const llmProvider = createLLMProvider(); + + if (llmProvider) { + console.log("LLM triage enabled (using PostHog gateway).\n"); + } else { + console.log( + "LLM triage disabled (no POSTHOG_API_KEY set). Matches will be treated as threats.\n", + ); + } + + const args = process.argv.slice(2); + if (args.length === 0) { + console.log("Usage:"); + console.log( + " node scripts/scan-warlock.js dist/skills # Scan built skill ZIPs", + ); + console.log( + " node scripts/scan-warlock.js path/to/file.md # Scan specific file(s)", + ); + process.exit(1); + } + + // Determine what to scan + let filesToScan = []; // { label: string, filePath: string } + let tmpDir = null; + + const firstArg = args[0]; + const isSingleDir = + args.length === 1 && + fs.existsSync(firstArg) && + fs.statSync(firstArg).isDirectory(); + + if (isSingleDir) { + const dir = firstArg; + const zips = fs.readdirSync(dir).filter((f) => f.endsWith(".zip")); + + if (zips.length > 0) { + // ZIP mode: extract and scan each archive + console.log(`Scanning ${zips.length} skill archive(s) with Warlock...\n`); + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "warlock-scan-")); + for (const zip of zips) { + const extracted = extractZip(path.join(dir, zip), tmpDir); + if (!extracted) continue; + for (const f of collectTextFiles(extracted)) { + const relPath = path.relative(extracted, f); + filesToScan.push({ label: `${zip} > ${relPath}`, filePath: f }); + } + } + } else { + // Directory without ZIPs: scan text files directly + const files = collectTextFiles(dir); + console.log( + `Scanning ${files.length} file(s) in ${dir} with Warlock...\n`, + ); + for (const f of files) { + filesToScan.push({ + label: path.relative(process.cwd(), f), + filePath: f, + }); + } + } + } else { + // Individual files mode + for (const arg of args) { + if (fs.existsSync(arg) && fs.statSync(arg).isFile()) { + filesToScan.push({ + label: path.relative(process.cwd(), arg), + filePath: arg, + }); + } else { + console.error(`File not found: ${arg}`); + } + } + console.log(`Scanning ${filesToScan.length} file(s) with Warlock...\n`); + } + + // Step 1: Run all YARA scans and collect matches + const allMatches = []; // { label, match, content } + + for (const { label, filePath } of filesToScan) { + const content = fs.readFileSync(filePath, "utf8"); + const result = await scan(content); + + if (result.matched) { + for (const match of result.matches) { + allMatches.push({ label, match, content }); + } + } + } + + console.log( + `YARA scan complete: ${allMatches.length} match(es) across ${filesToScan.length} file(s).\n`, + ); + + // Step 2: Triage ALL matches in one LLM call + let threats = 0; + let warnings = 0; + + if (allMatches.length > 0) { + const rawMatches = allMatches.map((m) => m.match); + + // Build a combined content summary for the LLM (one snippet per unique file) + const uniqueFiles = new Map(); + for (const { label, content } of allMatches) { + if (!uniqueFiles.has(label)) { + uniqueFiles.set(label, content); + } + } + const combinedContent = [...uniqueFiles.entries()] + .map(([label, content]) => `--- ${label} ---\n${content.slice(0, 2000)}`) + .join("\n\n"); + + const triaged = llmProvider + ? await triageMatches(combinedContent, rawMatches, llmProvider) + : rawMatches.map((m) => ({ + ...m, + triage: { + verdict: "true_positive", + reason: "No LLM triage available.", + }, + })); + + for (let i = 0; i < triaged.length; i++) { + const { label } = allMatches[i]; + const match = triaged[i]; + const isFP = match.triage.verdict === "false_positive"; + reportMatch(label, match, isFP); + if (isFP) { + warnings++; + } else { + threats++; + } + } + } + + // Cleanup + if (tmpDir) fs.rmSync(tmpDir, { recursive: true, force: true }); + + // Summary + console.log("---"); + if (warnings > 0) { + console.log( + color.yellow( + `${warnings} false positive(s) identified by LLM triage (not blocking).`, + ), + ); + } + if (threats > 0) { + console.log(color.red(`FAILED: ${threats} threat(s) detected.`)); + console.log("Fix the flagged content before releasing."); + process.exit(1); + } else { + console.log( + color.green(`PASSED: No threats found in ${filesToScan.length} file(s).`), + ); + process.exit(0); + } +} + +main().catch((err) => { + console.error("Scan failed:", err); + process.exit(2); +});