feat: add experiment validation workflow

.github/workflows/experiment_validation.yml

@@ -0,0 +1,133 @@
+name: Experiment Validation
+
+"on":
+  issues:
+    types: [opened, edited, reopened, labeled]
+  workflow_dispatch:
+    inputs:
+      issue_number:
+        description: "Monthly optimization task issue number"
+        required: true
+
+jobs:
+  validate:
+    if: contains(github.event.issue.labels.*.name, 'monthly-optimization-task') || inputs.issue_number != ''
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      issues: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Setup Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: |
+          set -euo pipefail
+          REQ_FILE="requirements-lock.txt"
+          if [ ! -f "$REQ_FILE" ]; then REQ_FILE="requirements.txt"; fi
+          python -m pip install --upgrade pip
+          python -m pip install -r "$REQ_FILE"
+
+      - name: Load issue context
+        id: issue_context
+        run: |
+          python3 - <<'PY'
+          import json
+          import os
+          import urllib.request
+          from pathlib import Path
+
+          repo = os.environ["GITHUB_REPOSITORY"]
+          issue_number = os.environ["ISSUE_NUMBER"]
+          token = os.environ["GITHUB_TOKEN"]
+          api_url = f"https://api.github.com/repos/{repo}/issues/{issue_number}"
+          request = urllib.request.Request(
+              api_url,
+              headers={
+                  "Accept": "application/vnd.github+json",
+                  "Authorization": f"Bearer {token}",
+                  "X-GitHub-Api-Version": "2022-11-28",
+                  "User-Agent": "experiment-validation",
+              },
+          )
+          with urllib.request.urlopen(request) as response:
+              issue = json.load(response)
+
+          issue_context = {
+              "number": issue["number"],
+              "title": issue["title"],
+              "body": issue.get("body", ""),
+          }
+          output_dir = Path("data/output/experiment_validation")
+          output_dir.mkdir(parents=True, exist_ok=True)
+          (output_dir / "issue_context.json").write_text(
+              json.dumps(issue_context, ensure_ascii=False, indent=2) + "\n",
+              encoding="utf-8",
+          )
+          PY
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ISSUE_NUMBER: ${{ inputs.issue_number || github.event.issue.number }}
+
+      - name: Prepare experiment validation payload
+        id: experiment_payload
+        run: |
+          python3 scripts/prepare_experiment_validation.py \
+            --issue-context-file data/output/experiment_validation/issue_context.json \
+            --output-dir data/output/experiment_validation >> "$GITHUB_OUTPUT"
+
+      - name: Append task summary
+        run: cat data/output/experiment_validation/task_summary.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Append skip reason
+        if: steps.experiment_payload.outputs.should_run != 'true'
+        run: |
+          if [ -f data/output/experiment_validation/skip_reason.txt ]; then
+            cat data/output/experiment_validation/skip_reason.txt >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Run monthly shadow build
+        if: steps.experiment_payload.outputs.should_run == 'true' && steps.experiment_payload.outputs.run_shadow_build == 'true'
+        run: |
+          python3 scripts/run_monthly_shadow_build.py --skip-publish-dry-run
+
+      - name: Run walk-forward validation
+        if: steps.experiment_payload.outputs.should_run == 'true' && steps.experiment_payload.outputs.run_walkforward_validation == 'true'
+        run: |
+          python3 scripts/run_walkforward_validation.py
+
+      - name: Render validation summary
+        if: steps.experiment_payload.outputs.should_run == 'true'
+        run: |
+          python3 scripts/render_experiment_validation_summary.py \
+            --payload-file data/output/experiment_validation/payload.json \
+            --shadow-summary-file data/output/monthly_shadow_build_summary.json \
+            --output-file data/output/experiment_validation/validation_summary.md
+
+      - name: Append validation summary
+        if: steps.experiment_payload.outputs.should_run == 'true'
+        run: cat data/output/experiment_validation/validation_summary.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Post validation comment
+        if: steps.experiment_payload.outputs.should_run == 'true'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python3 scripts/post_experiment_validation_comment.py \
+            --repo "${GITHUB_REPOSITORY}" \
+            --issue-number "${{ inputs.issue_number || github.event.issue.number }}" \
+            --review-file data/output/experiment_validation/validation_summary.md \
+            --run-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+
+      - name: Upload validation artifact
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: experiment-validation-${{ inputs.issue_number || github.event.issue.number }}
+          path: data/output/experiment_validation/

.github/workflows/monthly_optimization_planner.yml

@@ -18,7 +18,7 @@ jobs:
   planner:
     runs-on: ubuntu-latest
     permissions:
-      actions: read
+      actions: write
       contents: read
       issues: write
 
@@ -158,6 +158,37 @@ jobs:
               print(line)
           PY
 
+      - name: Resolve upstream experiment validation target
+        id: upstream_experiment_target
+        run: |
+          python3 - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+
+          fanout = json.loads(
+              Path("data/output/monthly_optimization/fanout/crypto_leader_rotation.json").read_text(encoding="utf-8")
+          )
+          plan = json.loads(
+              Path("data/output/monthly_optimization/optimization_plan.json").read_text(encoding="utf-8")
+          )
+          actions = plan.get("repo_action_summary", {}).get("CryptoLeaderRotation", {}).get("actions", [])
+          should_dispatch = bool(fanout.get("issue_number")) and fanout.get("status") in {"created", "updated"} and any(
+              action.get("experiment_only") for action in actions
+          )
+          with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output:
+              print(f"should_dispatch={'true' if should_dispatch else 'false'}", file=output)
+              print(f"issue_number={fanout.get('issue_number') or ''}", file=output)
+          PY
+
+      - name: Dispatch CryptoLeaderRotation experiment validation
+        if: steps.upstream_experiment_target.outputs.should_dispatch == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh workflow run experiment_validation.yml \
+            -f issue_number="${{ steps.upstream_experiment_target.outputs.issue_number }}"
+
       - name: Upload planner artifact
         uses: actions/upload-artifact@v7
         with:

scripts/post_experiment_validation_comment.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import urllib.error
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+
+COMMENT_MARKER = "<!-- monthly-experiment-validation -->"
+DEFAULT_API_URL = "https://api.github.com"
+
+
+
+def build_comment_body(review_markdown: str, run_url: str | None = None) -> str:
+    body = f"{COMMENT_MARKER}\n## Monthly Experiment Validation\n\n{review_markdown.strip()}"
+    if run_url:
+        body += f"\n\n---\n_Generated by experiment validation workflow: {run_url}_"
+    return body
+
+
+
+def github_request(method: str, url: str, token: str, payload: dict[str, Any] | None = None) -> Any:
+    data = None
+    headers = {
+        "Accept": "application/vnd.github+json",
+        "Authorization": f"Bearer {token}",
+        "X-GitHub-Api-Version": "2022-11-28",
+        "User-Agent": "crypto-leader-rotation-experiment-validation",
+    }
+    if payload is not None:
+        data = json.dumps(payload).encode("utf-8")
+        headers["Content-Type"] = "application/json"
+
+    request = urllib.request.Request(url, data=data, headers=headers, method=method)
+    with urllib.request.urlopen(request) as response:
+        charset = response.headers.get_content_charset("utf-8")
+        raw = response.read().decode(charset)
+        return json.loads(raw) if raw else None
+
+
+
+def upsert_issue_comment(*, api_url: str, repo: str, issue_number: int, token: str, body: str) -> None:
+    comments_url = f"{api_url}/repos/{repo}/issues/{issue_number}/comments"
+    comments = github_request("GET", comments_url, token)
+    existing = next((comment for comment in comments if COMMENT_MARKER in comment.get("body", "")), None)
+    if existing:
+        github_request(
+            "PATCH",
+            f"{api_url}/repos/{repo}/issues/comments/{existing['id']}",
+            token,
+            {"body": body},
+        )
+        print(f"Updated issue comment {existing['id']}")
+        return
+
+    github_request("POST", comments_url, token, {"body": body})
+    print(f"Created issue comment for issue #{issue_number}")
+
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Upsert the monthly experiment validation comment on the task issue.")
+    parser.add_argument("--repo", required=True)
+    parser.add_argument("--issue-number", required=True, type=int)
+    parser.add_argument("--review-file", required=True, type=Path)
+    parser.add_argument("--api-url", default=DEFAULT_API_URL)
+    parser.add_argument("--run-url", default="")
+    return parser.parse_args()
+
+
+
+def main() -> int:
+    args = parse_args()
+    token = os.environ.get("GITHUB_TOKEN")
+    if not token:
+        print("GITHUB_TOKEN is required", file=sys.stderr)
+        return 1
+
+    body = build_comment_body(args.review_file.read_text(encoding="utf-8"), args.run_url or None)
+    try:
+        upsert_issue_comment(
+            api_url=args.api_url.rstrip("/"),
+            repo=args.repo,
+            issue_number=args.issue_number,
+            token=token,
+            body=body,
+        )
+    except urllib.error.HTTPError as exc:
+        detail = exc.read().decode("utf-8", errors="replace")
+        print(f"GitHub API request failed: {exc.code} {detail}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())