From 14d90be5096db481dd924023b3f7d6c0ff428a14 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Thu, 2 Apr 2026 05:29:53 +0800 Subject: [PATCH] feat: add experiment validation workflow --- .github/workflows/experiment_validation.yml | 133 ++++++++++++++++++ .../monthly_optimization_planner.yml | 33 ++++- scripts/post_experiment_validation_comment.py | 100 +++++++++++++ scripts/prepare_experiment_validation.py | 122 ++++++++++++++++ .../render_experiment_validation_summary.py | 97 +++++++++++++ ...t_experiment_validation_workflow_config.py | 28 ++++ ...ly_optimization_planner_workflow_config.py | 5 +- ...test_post_experiment_validation_comment.py | 22 +++ tests/test_prepare_experiment_validation.py | 53 +++++++ ...st_render_experiment_validation_summary.py | 43 ++++++ 10 files changed, 634 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/experiment_validation.yml create mode 100644 scripts/post_experiment_validation_comment.py create mode 100644 scripts/prepare_experiment_validation.py create mode 100644 scripts/render_experiment_validation_summary.py create mode 100644 tests/test_experiment_validation_workflow_config.py create mode 100644 tests/test_post_experiment_validation_comment.py create mode 100644 tests/test_prepare_experiment_validation.py create mode 100644 tests/test_render_experiment_validation_summary.py diff --git a/.github/workflows/experiment_validation.yml b/.github/workflows/experiment_validation.yml new file mode 100644 index 0000000..9eed929 --- /dev/null +++ b/.github/workflows/experiment_validation.yml @@ -0,0 +1,133 @@ +name: Experiment Validation + +"on": + issues: + types: [opened, edited, reopened, labeled] + workflow_dispatch: + inputs: + issue_number: + description: "Monthly optimization task issue number" + required: true + +jobs: + validate: + if: contains(github.event.issue.labels.*.name, 'monthly-optimization-task') || inputs.issue_number != '' + runs-on: ubuntu-latest + permissions: + contents: read + issues: write + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + set -euo pipefail + REQ_FILE="requirements-lock.txt" + if [ ! -f "$REQ_FILE" ]; then REQ_FILE="requirements.txt"; fi + python -m pip install --upgrade pip + python -m pip install -r "$REQ_FILE" + + - name: Load issue context + id: issue_context + run: | + python3 - <<'PY' + import json + import os + import urllib.request + from pathlib import Path + + repo = os.environ["GITHUB_REPOSITORY"] + issue_number = os.environ["ISSUE_NUMBER"] + token = os.environ["GITHUB_TOKEN"] + api_url = f"https://api.github.com/repos/{repo}/issues/{issue_number}" + request = urllib.request.Request( + api_url, + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "experiment-validation", + }, + ) + with urllib.request.urlopen(request) as response: + issue = json.load(response) + + issue_context = { + "number": issue["number"], + "title": issue["title"], + "body": issue.get("body", ""), + } + output_dir = Path("data/output/experiment_validation") + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "issue_context.json").write_text( + json.dumps(issue_context, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + PY + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ISSUE_NUMBER: ${{ inputs.issue_number || github.event.issue.number }} + + - name: Prepare experiment validation payload + id: experiment_payload + run: | + python3 scripts/prepare_experiment_validation.py \ + --issue-context-file data/output/experiment_validation/issue_context.json \ + --output-dir data/output/experiment_validation >> "$GITHUB_OUTPUT" + + - name: Append task summary + run: cat data/output/experiment_validation/task_summary.md >> "$GITHUB_STEP_SUMMARY" + + - name: Append skip reason + if: steps.experiment_payload.outputs.should_run != 'true' + run: | + if [ -f data/output/experiment_validation/skip_reason.txt ]; then + cat data/output/experiment_validation/skip_reason.txt >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Run monthly shadow build + if: steps.experiment_payload.outputs.should_run == 'true' && steps.experiment_payload.outputs.run_shadow_build == 'true' + run: | + python3 scripts/run_monthly_shadow_build.py --skip-publish-dry-run + + - name: Run walk-forward validation + if: steps.experiment_payload.outputs.should_run == 'true' && steps.experiment_payload.outputs.run_walkforward_validation == 'true' + run: | + python3 scripts/run_walkforward_validation.py + + - name: Render validation summary + if: steps.experiment_payload.outputs.should_run == 'true' + run: | + python3 scripts/render_experiment_validation_summary.py \ + --payload-file data/output/experiment_validation/payload.json \ + --shadow-summary-file data/output/monthly_shadow_build_summary.json \ + --output-file data/output/experiment_validation/validation_summary.md + + - name: Append validation summary + if: steps.experiment_payload.outputs.should_run == 'true' + run: cat data/output/experiment_validation/validation_summary.md >> "$GITHUB_STEP_SUMMARY" + + - name: Post validation comment + if: steps.experiment_payload.outputs.should_run == 'true' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python3 scripts/post_experiment_validation_comment.py \ + --repo "${GITHUB_REPOSITORY}" \ + --issue-number "${{ inputs.issue_number || github.event.issue.number }}" \ + --review-file data/output/experiment_validation/validation_summary.md \ + --run-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + + - name: Upload validation artifact + if: always() + uses: actions/upload-artifact@v7 + with: + name: experiment-validation-${{ inputs.issue_number || github.event.issue.number }} + path: data/output/experiment_validation/ diff --git a/.github/workflows/monthly_optimization_planner.yml b/.github/workflows/monthly_optimization_planner.yml index f7005c3..e03a0f7 100644 --- a/.github/workflows/monthly_optimization_planner.yml +++ b/.github/workflows/monthly_optimization_planner.yml @@ -18,7 +18,7 @@ jobs: planner: runs-on: ubuntu-latest permissions: - actions: read + actions: write contents: read issues: write @@ -158,6 +158,37 @@ jobs: print(line) PY + - name: Resolve upstream experiment validation target + id: upstream_experiment_target + run: | + python3 - <<'PY' + import json + import os + from pathlib import Path + + fanout = json.loads( + Path("data/output/monthly_optimization/fanout/crypto_leader_rotation.json").read_text(encoding="utf-8") + ) + plan = json.loads( + Path("data/output/monthly_optimization/optimization_plan.json").read_text(encoding="utf-8") + ) + actions = plan.get("repo_action_summary", {}).get("CryptoLeaderRotation", {}).get("actions", []) + should_dispatch = bool(fanout.get("issue_number")) and fanout.get("status") in {"created", "updated"} and any( + action.get("experiment_only") for action in actions + ) + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + print(f"should_dispatch={'true' if should_dispatch else 'false'}", file=output) + print(f"issue_number={fanout.get('issue_number') or ''}", file=output) + PY + + - name: Dispatch CryptoLeaderRotation experiment validation + if: steps.upstream_experiment_target.outputs.should_dispatch == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh workflow run experiment_validation.yml \ + -f issue_number="${{ steps.upstream_experiment_target.outputs.issue_number }}" + - name: Upload planner artifact uses: actions/upload-artifact@v7 with: diff --git a/scripts/post_experiment_validation_comment.py b/scripts/post_experiment_validation_comment.py new file mode 100644 index 0000000..cbb38b1 --- /dev/null +++ b/scripts/post_experiment_validation_comment.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +import argparse +import json +import os +import sys +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any + + +COMMENT_MARKER = "" +DEFAULT_API_URL = "https://api.github.com" + + + +def build_comment_body(review_markdown: str, run_url: str | None = None) -> str: + body = f"{COMMENT_MARKER}\n## Monthly Experiment Validation\n\n{review_markdown.strip()}" + if run_url: + body += f"\n\n---\n_Generated by experiment validation workflow: {run_url}_" + return body + + + +def github_request(method: str, url: str, token: str, payload: dict[str, Any] | None = None) -> Any: + data = None + headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "crypto-leader-rotation-experiment-validation", + } + if payload is not None: + data = json.dumps(payload).encode("utf-8") + headers["Content-Type"] = "application/json" + + request = urllib.request.Request(url, data=data, headers=headers, method=method) + with urllib.request.urlopen(request) as response: + charset = response.headers.get_content_charset("utf-8") + raw = response.read().decode(charset) + return json.loads(raw) if raw else None + + + +def upsert_issue_comment(*, api_url: str, repo: str, issue_number: int, token: str, body: str) -> None: + comments_url = f"{api_url}/repos/{repo}/issues/{issue_number}/comments" + comments = github_request("GET", comments_url, token) + existing = next((comment for comment in comments if COMMENT_MARKER in comment.get("body", "")), None) + if existing: + github_request( + "PATCH", + f"{api_url}/repos/{repo}/issues/comments/{existing['id']}", + token, + {"body": body}, + ) + print(f"Updated issue comment {existing['id']}") + return + + github_request("POST", comments_url, token, {"body": body}) + print(f"Created issue comment for issue #{issue_number}") + + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Upsert the monthly experiment validation comment on the task issue.") + parser.add_argument("--repo", required=True) + parser.add_argument("--issue-number", required=True, type=int) + parser.add_argument("--review-file", required=True, type=Path) + parser.add_argument("--api-url", default=DEFAULT_API_URL) + parser.add_argument("--run-url", default="") + return parser.parse_args() + + + +def main() -> int: + args = parse_args() + token = os.environ.get("GITHUB_TOKEN") + if not token: + print("GITHUB_TOKEN is required", file=sys.stderr) + return 1 + + body = build_comment_body(args.review_file.read_text(encoding="utf-8"), args.run_url or None) + try: + upsert_issue_comment( + api_url=args.api_url.rstrip("/"), + repo=args.repo, + issue_number=args.issue_number, + token=token, + body=body, + ) + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace") + print(f"GitHub API request failed: {exc.code} {detail}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/prepare_experiment_validation.py b/scripts/prepare_experiment_validation.py new file mode 100644 index 0000000..3d9ff4d --- /dev/null +++ b/scripts/prepare_experiment_validation.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +try: + from scripts.prepare_auto_optimization_pr import parse_actions +except ModuleNotFoundError: # pragma: no cover - script execution fallback + from prepare_auto_optimization_pr import parse_actions + + +SHADOW_BUILD_MARKERS = ( + "shadow build", + "challenger", + "official_baseline", + "track_summary", +) +WALKFORWARD_MARKERS = ( + "walk-forward", + "walkforward", + "leader capture", + "backtest", + "validation window", +) + + +def _combined_text(action: dict[str, object]) -> str: + return f"{action.get('title', '')} {action.get('summary', '')}".lower() + + +def build_payload(issue_context: dict[str, object]) -> dict[str, object]: + issue_number = int(issue_context["number"]) + issue_title = str(issue_context["title"]).strip() + parsed_actions = parse_actions(str(issue_context.get("body", ""))) + experiment_actions = [action for action in parsed_actions if "experiment-only" in action.get("flags", [])] + run_shadow_build = any(any(marker in _combined_text(action) for marker in SHADOW_BUILD_MARKERS) for action in experiment_actions) + run_walkforward_validation = any( + any(marker in _combined_text(action) for marker in WALKFORWARD_MARKERS) for action in experiment_actions + ) + + should_run = bool(experiment_actions) and (run_shadow_build or run_walkforward_validation) + skip_reason = "" + if not experiment_actions: + skip_reason = "No experiment-only tasks were found in this monthly optimization issue." + elif not should_run: + skip_reason = "No supported upstream experiment validation target was found in the selected tasks." + + return { + "issue_number": issue_number, + "issue_title": issue_title, + "should_run": should_run, + "experiment_task_count": len(experiment_actions), + "run_shadow_build": run_shadow_build, + "run_walkforward_validation": run_walkforward_validation, + "experiment_actions": experiment_actions, + "skip_reason": skip_reason, + } + + + +def render_task_summary(payload: dict[str, object]) -> str: + lines = [ + "# Experiment Validation Candidate Tasks", + "", + f"- Issue: #{payload['issue_number']} {payload['issue_title']}", + f"- Experiment-only tasks: `{payload['experiment_task_count']}`", + f"- Shadow build selected: `{str(payload['run_shadow_build']).lower()}`", + f"- Walk-forward validation selected: `{str(payload['run_walkforward_validation']).lower()}`", + ] + actions = payload["experiment_actions"] + if not actions: + lines.extend(["", payload["skip_reason"]]) + return "\n".join(lines).strip() + "\n" + + lines.extend(["", "## Selected Tasks"]) + for action in actions: + flag_suffix = f" [{', '.join(action['flags'])}]" if action.get("flags") else "" + lines.extend( + [ + f"- `{action['risk_level']}` {action['title']}{flag_suffix}", + f" - Summary: {action.get('summary', 'No summary provided.')}", + f" - Source: {action.get('source_label', 'Unknown source')} ({action.get('source_url', 'n/a')})", + ] + ) + + if payload["skip_reason"]: + lines.extend(["", payload["skip_reason"]]) + return "\n".join(lines).strip() + "\n" + + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Prepare metadata for experiment-only monthly optimization validation.") + parser.add_argument("--issue-context-file", required=True, type=Path) + parser.add_argument("--output-dir", required=True, type=Path) + return parser.parse_args() + + + +def main() -> int: + args = parse_args() + issue_context = json.loads(args.issue_context_file.read_text(encoding="utf-8")) + payload = build_payload(issue_context) + args.output_dir.mkdir(parents=True, exist_ok=True) + payload_file = args.output_dir / "payload.json" + task_summary_file = args.output_dir / "task_summary.md" + if payload["skip_reason"]: + (args.output_dir / "skip_reason.txt").write_text(str(payload["skip_reason"]) + "\n", encoding="utf-8") + payload_file.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + task_summary_file.write_text(render_task_summary(payload), encoding="utf-8") + print(f"should_run={'true' if payload['should_run'] else 'false'}") + print(f"issue_number={payload['issue_number']}") + print(f"run_shadow_build={'true' if payload['run_shadow_build'] else 'false'}") + print(f"run_walkforward_validation={'true' if payload['run_walkforward_validation'] else 'false'}") + print(f"payload_file={payload_file}") + print(f"task_summary_file={task_summary_file}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/render_experiment_validation_summary.py b/scripts/render_experiment_validation_summary.py new file mode 100644 index 0000000..63efbee --- /dev/null +++ b/scripts/render_experiment_validation_summary.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + + + +def load_optional_json(path: Path | None) -> dict[str, Any] | None: + if path is None or not path.exists(): + return None + return json.loads(path.read_text(encoding="utf-8")) + + + +def _track_line(track: dict[str, Any]) -> str: + parts = [f"`{track.get('track_id', 'unknown')}`"] + if track.get("profile_name"): + parts.append(str(track["profile_name"])) + if track.get("pool_size") is not None: + parts.append(f"pool_size={track['pool_size']}") + if track.get("release_index_path"): + parts.append(f"index={track['release_index_path']}") + return "- " + " · ".join(parts) + + + +def build_summary_markdown(payload: dict[str, Any], shadow_summary: dict[str, Any] | None) -> str: + lines = [ + "## Monthly Experiment Validation", + "", + f"- Issue: #{payload['issue_number']} {payload['issue_title']}", + f"- Experiment-only tasks: `{payload['experiment_task_count']}`", + f"- Validation executed: `{'yes' if payload['should_run'] else 'no'}`", + ] + if payload.get("skip_reason"): + lines.append(f"- Skip reason: {payload['skip_reason']}") + + actions = payload.get("experiment_actions", []) + if actions: + lines.extend(["", "### Selected Tasks"]) + for action in actions: + flags = f" [{', '.join(action['flags'])}]" if action.get("flags") else "" + lines.extend( + [ + f"- `{action['risk_level']}` {action['title']}{flags}", + f" - Summary: {action.get('summary', 'No summary provided.')}", + ] + ) + + if shadow_summary is not None: + official = shadow_summary.get("official_baseline", {}) + shadow_tracks = shadow_summary.get("shadow_candidate_tracks", {}).get("tracks", []) + lines.extend( + [ + "", + "### Validation Results", + f"- As of date: `{shadow_summary.get('as_of_date', 'unknown')}`", + f"- Official baseline version: `{official.get('version', 'unknown')}`", + f"- Official baseline mode: `{official.get('mode', 'unknown')}`", + f"- Official baseline pool size: `{official.get('pool_size', 'unknown')}`", + f"- Shadow candidate tracks generated: `{len(shadow_tracks)}`", + ] + ) + if official.get("publish_manifest_path"): + lines.append(f"- Publish manifest path: `{official['publish_manifest_path']}`") + if shadow_tracks: + lines.extend(["", "### Shadow Tracks"]) + lines.extend(_track_line(track) for track in shadow_tracks) + elif payload.get("should_run"): + lines.extend(["", "### Validation Results", "- No shadow-build summary artifact was found."]) + + return "\n".join(lines).strip() + "\n" + + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Render experiment validation markdown for the monthly optimization task issue.") + parser.add_argument("--payload-file", required=True, type=Path) + parser.add_argument("--output-file", required=True, type=Path) + parser.add_argument("--shadow-summary-file", type=Path) + return parser.parse_args() + + + +def main() -> int: + args = parse_args() + payload = json.loads(args.payload_file.read_text(encoding="utf-8")) + shadow_summary = load_optional_json(args.shadow_summary_file) + args.output_file.write_text(build_summary_markdown(payload, shadow_summary), encoding="utf-8") + print(f"summary_file={args.output_file}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_experiment_validation_workflow_config.py b/tests/test_experiment_validation_workflow_config.py new file mode 100644 index 0000000..4e4a814 --- /dev/null +++ b/tests/test_experiment_validation_workflow_config.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import unittest +from pathlib import Path + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +WORKFLOW_PATH = PROJECT_ROOT / ".github" / "workflows" / "experiment_validation.yml" + + +class ExperimentValidationWorkflowConfigTests(unittest.TestCase): + def test_workflow_runs_shadow_build_and_posts_comment(self) -> None: + workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + + self.assertIn("issues:", workflow) + self.assertIn("workflow_dispatch:", workflow) + self.assertIn("issue_number:", workflow) + self.assertIn("prepare_experiment_validation.py", workflow) + self.assertIn("run_monthly_shadow_build.py", workflow) + self.assertIn("run_walkforward_validation.py", workflow) + self.assertIn("render_experiment_validation_summary.py", workflow) + self.assertIn("post_experiment_validation_comment.py", workflow) + self.assertIn("actions/upload-artifact@v7", workflow) + self.assertIn("monthly-optimization-task", workflow) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_monthly_optimization_planner_workflow_config.py b/tests/test_monthly_optimization_planner_workflow_config.py index 08845fd..87b44de 100644 --- a/tests/test_monthly_optimization_planner_workflow_config.py +++ b/tests/test_monthly_optimization_planner_workflow_config.py @@ -16,7 +16,7 @@ def test_planner_workflow_downloads_artifacts_posts_issue_and_fans_out_tasks(sel self.assertIn("upstream_run_id:", workflow) self.assertIn("downstream_run_id:", workflow) self.assertIn("downstream_repo:", workflow) - self.assertIn("actions: read", workflow) + self.assertIn("actions: write", workflow) self.assertIn("CROSS_REPO_GITHUB_TOKEN", workflow) self.assertIn("gh run download", workflow) self.assertIn("Resolve downloaded artifact paths", workflow) @@ -29,6 +29,9 @@ def test_planner_workflow_downloads_artifacts_posts_issue_and_fans_out_tasks(sel self.assertIn("Fan out CryptoLeaderRotation task issue", workflow) self.assertIn("Fan out CryptoStrategies task issue", workflow) self.assertIn("Fan out BinancePlatform task issue", workflow) + self.assertIn("Resolve upstream experiment validation target", workflow) + self.assertIn("Dispatch CryptoLeaderRotation experiment validation", workflow) + self.assertIn("gh workflow run experiment_validation.yml", workflow) self.assertIn("--allow-permission-skip", workflow) self.assertIn("Append fanout summary", workflow) self.assertIn("upstream_review_payload.json", workflow) diff --git a/tests/test_post_experiment_validation_comment.py b/tests/test_post_experiment_validation_comment.py new file mode 100644 index 0000000..e014810 --- /dev/null +++ b/tests/test_post_experiment_validation_comment.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +import unittest + +from scripts.post_experiment_validation_comment import COMMENT_MARKER, build_comment_body + + +class PostExperimentValidationCommentTests(unittest.TestCase): + def test_build_comment_body_includes_marker_and_run_link(self) -> None: + body = build_comment_body( + "Validation content", + "https://github.com/example/repo/actions/runs/123", + ) + + self.assertIn(COMMENT_MARKER, body) + self.assertIn("## Monthly Experiment Validation", body) + self.assertIn("Validation content", body) + self.assertIn("actions/runs/123", body) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prepare_experiment_validation.py b/tests/test_prepare_experiment_validation.py new file mode 100644 index 0000000..d59e1cd --- /dev/null +++ b/tests/test_prepare_experiment_validation.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import unittest + +from scripts.prepare_experiment_validation import build_payload + + +class PrepareExperimentValidationTests(unittest.TestCase): + def test_build_payload_selects_shadow_build_experiment(self) -> None: + issue_context = { + "number": 22, + "title": "Monthly Optimization Tasks · CryptoLeaderRotation", + "body": """# Monthly Optimization Tasks · CryptoLeaderRotation + +## Actions +- [ ] `low` Run monthly shadow build and archive challenger summaries [auto-pr-safe, experiment-only] + - Summary: Generate official_baseline and challenger_topk_60 coverage each month. + - Source: [QuantStrategyLab/CryptoLeaderRotation #11](https://example.com/11) +- [ ] `low` Document and verify tie-breaking for equal scores [auto-pr-safe] + - Summary: Confirm deterministic secondary sorting. + - Source: [QuantStrategyLab/CryptoLeaderRotation #11](https://example.com/11) +""", + } + + payload = build_payload(issue_context) + + self.assertTrue(payload["should_run"]) + self.assertEqual(payload["experiment_task_count"], 1) + self.assertTrue(payload["run_shadow_build"]) + self.assertFalse(payload["run_walkforward_validation"]) + + def test_build_payload_skips_when_no_experiment_tasks_exist(self) -> None: + issue_context = { + "number": 30, + "title": "Monthly Optimization Tasks · CryptoLeaderRotation", + "body": """# Monthly Optimization Tasks · CryptoLeaderRotation + +## Actions +- [ ] `low` Document and verify tie-breaking for equal scores [auto-pr-safe] + - Summary: Confirm deterministic secondary sorting. + - Source: [QuantStrategyLab/CryptoLeaderRotation #11](https://example.com/11) +""", + } + + payload = build_payload(issue_context) + + self.assertFalse(payload["should_run"]) + self.assertEqual(payload["experiment_task_count"], 0) + self.assertIn("No experiment-only tasks", payload["skip_reason"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_render_experiment_validation_summary.py b/tests/test_render_experiment_validation_summary.py new file mode 100644 index 0000000..81b1e79 --- /dev/null +++ b/tests/test_render_experiment_validation_summary.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import unittest + +from scripts.render_experiment_validation_summary import build_summary_markdown + + +class RenderExperimentValidationSummaryTests(unittest.TestCase): + def test_build_summary_includes_shadow_track_details(self) -> None: + payload = { + "issue_number": 22, + "issue_title": "Monthly Optimization Tasks · CryptoLeaderRotation", + "should_run": True, + "experiment_task_count": 1, + "experiment_actions": [ + { + "risk_level": "low", + "title": "Run monthly shadow build and archive challenger summaries", + "flags": ["auto-pr-safe", "experiment-only"], + "summary": "Generate official_baseline and challenger_topk_60 coverage each month.", + } + ], + "skip_reason": "", + } + shadow_summary = { + "as_of_date": "2026-04-01", + "official_baseline": {"version": "2026-04-01-core_major", "mode": "core_major", "pool_size": 5}, + "shadow_candidate_tracks": { + "tracks": [ + {"track_id": "challenger_topk_60", "profile_name": "challenger_topk_60", "pool_size": 5} + ] + }, + } + + summary = build_summary_markdown(payload, shadow_summary) + + self.assertIn("Monthly Experiment Validation", summary) + self.assertIn("Official baseline version", summary) + self.assertIn("challenger_topk_60", summary) + + +if __name__ == "__main__": + unittest.main()