diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0d20b64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.pyc diff --git a/.helper_bash_functions b/.helper_bash_functions index 6f6bf8f..7c92938 100644 --- a/.helper_bash_functions +++ b/.helper_bash_functions @@ -44,18 +44,37 @@ for d in sys.argv[1:]: print(f" ... ({len(lines) - 20} more lines)") PYEOF } +_find_verbose_test_packages() { + local requested_packages="$1" + local verbose_packages="" + while IFS=$'\t' read -r name path _; do + if echo " ${requested_packages} " | grep -q " ${name} "; then + if [ -f "${path}/.verbose_tests" ]; then + verbose_packages="${verbose_packages} ${name}" + fi + fi + done < <(colcon list 2>/dev/null) + echo "${verbose_packages## }" +} colcon_test_these_packages() { THIS_DIR=$(pwd) cd ${CATKIN_WS_PATH} colcon_build_no_deps $1 - source install/setup.bash && \ - colcon test --packages-select $1 + source install/setup.bash + local verbose_packages + verbose_packages=$(_find_verbose_test_packages "$1") + if [ -n "${verbose_packages}" ]; then + echo -e "${Yellow}Packages with verbose tests: ${verbose_packages}${Color_Off}" + colcon test --packages-select $1 --packages-skip ${verbose_packages} + colcon test --packages-select ${verbose_packages} --event-handlers console_direct+ + else + colcon test --packages-select $1 + fi for pkg in $1; do if ! colcon test-result --test-result-base "build/$pkg/test_results"; then _show_test_failures "build/$pkg/test_results" fi done cd $THIS_DIR - } colcon_test_this_package() { colcon_test_these_packages "$@"; } # Alias for backwards compatability find_cmake_project_names_from_dir() { if [[ -z $1 ]]; then DIR_TO_SEARCH="."; else DIR_TO_SEARCH=$1; fi; wget -qO- https://raw.githubusercontent.com/Extend-Robotics/er_build_tools/refs/heads/main/bin/find_cmake_project_names.py | python3 - $DIR_TO_SEARCH; } @@ -136,11 +155,35 @@ ps_aux() { ps aux | cgrep $1 | grep -v grep ; } kill_any_process() { ps_aux_command $1; conf="$(confirm "kill these processes? [Y/n]")"; if [[ $conf == "y" ]]; then echo "killing..."; sudo kill -9 $(ps_aux $1 | awk {'print $2}'); sleep 1; echo "remaining: "; ps_aux_command $1 else echo "not killing"; fi ; } docker_exec () { if [[ $(docker container ls -q | wc -l) -eq 1 ]]; then docker exec -it $(docker container ls -q) bash; else echo "wrong number of containers running"; fi; } awk_line_length() { if [[ -z $2 ]]; then MAX_LINE_LENGTH=200; else MAX_LINE_LENGTH=$2; fi; cat $1 | awk 'length($0) < '"$MAX_LINE_LENGTH"''; } -update_helper_bash_functions() { if [ ! -f ~/.helper_bash_functions ]; then - echo -e "${Red}ERROR: Tried to replace this file but couldn't find it, something has gone wrong!${Color_Off}\n" - return - fi - wget -O ~/.helper_bash_functions ${THIS_SCRIPT_URL} +MERGE_VARS_URL="https://raw.githubusercontent.com/Extend-Robotics/er_build_tools/refs/heads/${THIS_SCRIPT_BRANCH}/bin/merge_helper_vars.py" + +update_helper_bash_functions() { + if [ ! -f ~/.helper_bash_functions ]; then + echo -e "${Red}ERROR: Tried to replace this file but couldn't find it, something has gone wrong!${Color_Off}\n" + return 1 + fi + + local tmp_new + tmp_new=$(mktemp) + if ! wget -q -O "${tmp_new}" "${THIS_SCRIPT_URL}"; then + echo -e "${Red}Error: Failed to download updated helper_bash_functions${Color_Off}" + rm -f "${tmp_new}" + return 1 + fi + + local merge_script + merge_script=$(curl -fSL "${MERGE_VARS_URL}") || { + echo -e "${Red}Error: Failed to fetch merge_helper_vars.py${Color_Off}" + rm -f "${tmp_new}" + return 1 + } + + python3 <(echo "${merge_script}") ~/.helper_bash_functions "${tmp_new}" + cp "${tmp_new}" ~/.helper_bash_functions + rm -f "${tmp_new}" + + echo "" + echo -e "${Green}helper_bash_functions updated. Run 'source ~/.helper_bash_functions' to reload.${Color_Off}" } # python linters @@ -203,6 +246,64 @@ remove_ci_container() { echo -e "${Green}Container '${container_name}' removed${Color_Off}" } +# CI tool (Python CLI) - interactive CI reproduction + Claude-powered fix +CI_TOOL_CACHE_DIR="${HOME}/.ci_tool" +CI_TOOL_RAW_URL="https://raw.githubusercontent.com/Extend-Robotics/er_build_tools/refs/heads/${THIS_SCRIPT_BRANCH}/bin/ci_tool" +CI_TOOL_FILES="__init__.py __main__.py cli.py preflight.py containers.py ci_reproduce.py claude_setup.py ci_fix.py claude_session.py display_progress.py requirements.txt" +CI_TOOL_DATA_FILES="ci_context/CLAUDE.md" + +install_ci_tool() { + echo -e "${Yellow}Installing ci_tool from er_build_tools (branch: ${THIS_SCRIPT_BRANCH})...${Color_Off}" + mkdir -p "${CI_TOOL_CACHE_DIR}/ci_tool" + local failed="false" + for file in ${CI_TOOL_FILES}; do + echo " Fetching ${file}..." + if ! curl -fSL "${CI_TOOL_RAW_URL}/${file}" -o "${CI_TOOL_CACHE_DIR}/ci_tool/${file}"; then + echo -e "${Red} Failed to fetch ${file}${Color_Off}" + failed="true" + fi + done + for file in ${CI_TOOL_DATA_FILES}; do + echo " Fetching ${file}..." + mkdir -p "${CI_TOOL_CACHE_DIR}/ci_tool/$(dirname "${file}")" + if ! curl -fSL "${CI_TOOL_RAW_URL}/${file}" -o "${CI_TOOL_CACHE_DIR}/ci_tool/${file}"; then + echo -e "${Red} Failed to fetch ${file}${Color_Off}" + failed="true" + fi + done + if [ "${failed}" = "true" ]; then + echo -e "${Red}Error: Failed to install ci_tool. Check network and branch '${THIS_SCRIPT_BRANCH}'.${Color_Off}" + return 1 + fi + echo -e "${Green}ci_tool installed to ${CI_TOOL_CACHE_DIR}${Color_Off}" +} + +update_ci_tool() { install_ci_tool; } + +ci_tool() { + mkdir -p "${CI_TOOL_CACHE_DIR}/ci_tool" + local fetch_failed="false" + for file in ${CI_TOOL_FILES}; do + if ! curl -fsSL "${CI_TOOL_RAW_URL}/${file}" -o "${CI_TOOL_CACHE_DIR}/ci_tool/${file}" 2>/dev/null; then + fetch_failed="true" + fi + done + for file in ${CI_TOOL_DATA_FILES}; do + mkdir -p "${CI_TOOL_CACHE_DIR}/ci_tool/$(dirname "${file}")" 2>/dev/null + if ! curl -fsSL "${CI_TOOL_RAW_URL}/${file}" -o "${CI_TOOL_CACHE_DIR}/ci_tool/${file}" 2>/dev/null; then + fetch_failed="true" + fi + done + if [ "${fetch_failed}" = "true" ]; then + if [ ! -f "${CI_TOOL_CACHE_DIR}/ci_tool/__main__.py" ]; then + echo -e "${Red}Error: Failed to fetch ci_tool and no cached version available${Color_Off}" + return 1 + fi + echo -e "${Yellow}Warning: Failed to fetch latest ci_tool, using cached version${Color_Off}" + fi + python3 "${CI_TOOL_CACHE_DIR}/ci_tool" "$@" +} +ci_fix() { ci_tool fix "$@"; } er_python_linters_here() { local ret=0 diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..977f42a --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,71 @@ +# er_build_tools + +Public build tools and CI utilities for Extend Robotics ROS1 repositories. The main component is `ci_tool`, an interactive CLI that reproduces CI failures locally in Docker and uses Claude Code to fix them. + +## Project Structure + +``` +bin/ + ci_tool/ # Main Python package + __main__.py # Entry point (auto-installs missing deps) + cli.py # Menu dispatch router + ci_fix.py # Core workflow: reproduce -> Claude analysis -> fix -> shell + ci_reproduce.py # Docker container setup for CI reproduction + claude_setup.py # Install Claude + copy credentials/config into container + claude_session.py # Interactive Claude session launcher + containers.py # Docker lifecycle (create, exec, cp, remove) + preflight.py # Auth/setup validation (fail-fast) + display_progress.py # Stream-json output processor for Claude + ci_context/ + CLAUDE.md # CI-specific instructions for Claude inside containers + setup.sh # User-facing setup script + reproduce_ci.sh # Public wrapper for CI reproduction +.helper_bash_functions # Sourced by users; provides colcon/rosdep helpers + ci_tool alias +pylintrc # Pylint config (strict: fail-under=10.0, max-line-length=140) +``` + +## Code Style + +- Python 3.6+, `from __future__ import annotations` in all modules +- snake_case everywhere; PascalCase for classes only +- 4-space indentation, max 140 char line length +- Pylint must pass at 10.0 (`pylintrc` at repo root) +- Use `# pylint: disable=...` pragmas only when essential, with justification +- Interactive prompts via `InquirerPy`; terminal UI via `rich` + +## Conventions + +- **Fail fast**: `PreflightError` for expected failures, `RuntimeError` for unexpected. No silent defaults, no fallback behaviour. +- **Minimal diffs**: Only change what's requested. No cosmetic cleanups, no "while I'm here" changes. +- **Self-documenting code**: Verbose variable names. Comments only for maths or external doc links. +- **Subprocess calls**: Use `docker_exec()` / `run_command()` from `containers.py`. Pass `check=False` when non-zero is expected; `quiet=True` to suppress echo. +- **State files**: `/ros_ws/.ci_fix_state.json` inside containers (session_id, phase, attempt_count) +- **Learnings persistence**: `~/.ci_tool/learnings/{org}_{repo}.md` on host, `/ros_ws/.ci_learnings.md` in container + +## Environment Variables + +- `GH_TOKEN` or `ER_SETUP_TOKEN` — GitHub token (checked in preflight) +- `CI_TOOL_SCRIPTS_BRANCH` — branch of er_build_tools_internal to fetch scripts from +- `IS_SANDBOX=1` — injected into all docker exec calls for Claude + +## Running + +```bash +# From host +source ~/.helper_bash_functions +ci_tool # interactive menu +ci_fix # shortcut for ci_tool fix + +# Lint +pylint --rcfile=pylintrc bin/ci_tool/ +``` + +## Testing + +No unit tests yet. Test manually by running `ci_tool` workflows end-to-end. + +## Common Pitfalls + +- Container Claude settings must use valid `defaultMode` values: `"acceptEdits"`, `"bypassPermissions"`, `"default"`, `"dontAsk"`, `"plan"`. The old `"dangerouslySkipPermissions"` is invalid. +- `docker_cp_to_container` requires the container to be running. +- Claude inside containers runs with `--dangerously-skip-permissions` flag (separate from the settings.json mode). diff --git a/README.md b/README.md index 5f56e6a..e4c6fe2 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,95 @@ Public build tools and utilities for Extend Robotics repositories. -## reproduce_ci.sh — Reproduce CI Locally +## Quick Setup + +Install helper bash functions, set your GitHub token, and authenticate Claude: + +```bash +bash <(curl -fsSL https://raw.githubusercontent.com/Extend-Robotics/er_build_tools/refs/heads/main/bin/setup.sh) +``` + +This installs `~/.helper_bash_functions` which provides build helpers, git aliases, and `ci_tool`. + +## ci_tool: Fix CI Failures with Claude + +`ci_tool` is an interactive CLI that reproduces CI failures locally in Docker and uses Claude Code to fix them. + +### Prerequisites + +- **Docker** installed and running +- **Claude Code** installed and authenticated: `npm install -g @anthropic-ai/claude-code && claude` +- **GitHub token** with `repo` scope: [create one](https://github.com/settings/tokens) + +### Usage + +```bash +source ~/.helper_bash_functions +ci_tool +``` + +This opens an interactive menu. You can also run subcommands directly — see `ci_tool --help`: + +``` +$ ci_tool --help +ci_tool — Fix CI failures with Claude + +Usage: ci_tool [command] + +Commands: + fix Fix CI failures with Claude + reproduce Reproduce CI environment in Docker + claude Interactive Claude session in container + shell Shell into an existing CI container + retest Re-run tests in a CI container + clean Remove CI containers + +Shortcuts: + ci_fix Alias for 'ci_tool fix' + +Run without arguments for interactive menu. +``` + +### Fix Workflow + +1. Run `ci_fix` +2. Create a new session or reuse an existing container +3. Optionally paste a GitHub Actions URL to target a specific failure +4. ci_tool reproduces the CI environment in Docker +5. Claude analyses the test output and applies fixes +6. You're dropped into a shell to review changes, commit, and push + +### Manual Setup + +If you prefer not to use the setup script: + +1. Download helper functions: + ```bash + curl -fsSL https://raw.githubusercontent.com/Extend-Robotics/er_build_tools/refs/heads/main/.helper_bash_functions \ + -o ~/.helper_bash_functions + ``` +2. Add your GitHub token to the top of `~/.helper_bash_functions`: + ```bash + export GH_TOKEN="ghp_your_token_here" + ``` +3. Source in your shell: + ```bash + echo 'source ~/.helper_bash_functions' >> ~/.bashrc + source ~/.helper_bash_functions + ``` +4. Install and authenticate Claude Code: + ```bash + npm install -g @anthropic-ai/claude-code + claude + ``` + +--- + +## reproduce_ci.sh: Reproduce CI Locally When CI fails, debugging requires pushing commits and waiting for results. This script reproduces the exact CI environment locally in a persistent Docker container, so you can debug interactively. -It creates a Docker container using the same image as CI, clones your repo and its dependencies, builds everything, and optionally runs tests — mirroring the steps in `setup_and_build_ros_ws.yml`. +It creates a Docker container using the same image as CI, clones your repo and its dependencies, builds everything, and optionally runs tests, mirroring the steps in `setup_and_build_ros_ws.yml`. ### Quick Start @@ -98,8 +182,8 @@ docker rm -f er_ci_reproduced_testing_env ### Troubleshooting -**Container already exists** — Remove it first: `docker rm -f er_ci_reproduced_testing_env` +**Container already exists**: Remove it first: `docker rm -f er_ci_reproduced_testing_env` -**404 when fetching scripts** — Check that your `--gh-token` has access to `er_build_tools_internal`, and that the `--scripts-branch` exists. +**404 when fetching scripts**: Check that your `--gh-token` has access to `er_build_tools_internal`, and that the `--scripts-branch` exists. -**`DISPLAY` error with graphical forwarding** — Either set `DISPLAY` (e.g. via X11 forwarding) or pass `--graphical false`. +**`DISPLAY` error with graphical forwarding**: Either set `DISPLAY` (e.g. via X11 forwarding) or pass `--graphical false`. diff --git a/bin/ci_tool/TODO.md b/bin/ci_tool/TODO.md new file mode 100644 index 0000000..a3593d8 --- /dev/null +++ b/bin/ci_tool/TODO.md @@ -0,0 +1,35 @@ +# ci_tool TODO + +## Features + +- [ ] **Analyse CI mode**: Implement the plan in `docs/plans/2026-02-25-ci-analyse-mode-plan.md`. New "Analyse CI" menu item with two sub-modes: "Remote only (fast)" fetches GH Actions logs, filters with regex, diagnoses with Claude haiku on host — no Docker. "Remote + local reproduction" runs both in parallel with a Rich Live split-panel display, then offers to transition into fix mode. New files: `ci_analyse.py`, `ci_analyse_display.py`, `ci_log_filter.py`. +- [x] ~~**Parallel CI analysis during local reproduction**~~: Superseded by the Analyse CI mode above. + +## UX + +- [ ] **Simplify the main menu**: Too many top-level options (reproduce, fix, claude, shell, retest, clean, exit). Several overlap — e.g. "Reproduce CI" is already a step within "Fix CI with Claude", and "Claude session" / "Shell into container" / "Re-run tests" are all post-reproduce actions on an existing container. Consolidate into fewer choices and push the rest into sub-menus or contextual prompts. + +## Bug Fixes + +- [ ] If branch name is empty/blank, default to the repo's default branch instead of requiring input +- [ ] In "Reproduce CI (create container)" mode, extract the branch name from the GitHub Actions URL (like `extract_info_from_ci_url` already does in "Fix CI with Claude" mode) instead of requiring the user to enter it manually + +## Done +- [x] ~~Render markdown in terminal~~ — display_progress.py now buffers text between tool calls and renders via `rich.markdown.Markdown` (tables, headers, code blocks, bold/italic) +- [x] ~~Empty workspace after reproduce~~ — `_docker_exec_workspace_setup()` distinguishes setup failures from test failures; `wstool scrape` fixed in internal repo +- [x] ~~Silent failures~~ — 21 issues audited and fixed across all modules +- [x] ~~resume_claude auth~~ — `IS_SANDBOX=1` passed via `docker exec -e` on all calls (`.bashrc` not sourced by non-interactive shells) +- [x] ~~gh CLI auth warning~~ — removed redundant `gh auth login` (GH_TOKEN env var handles auth) +- [x] ~~Token efficiency~~ — prompts updated to use grep instead of reading full logs +- [x] ~~Persistent learnings~~ — `~/.ci_tool/learnings/{org}_{repo}.md` persists between sessions + +## Testing + +- [ ] Add unit tests for each module, leveraging the clean separation of concerns: + - **ci_reproduce.py**: `_parse_repo_url` (edge cases: trailing slashes, `.git` suffix, invalid URLs, non-GitHub URLs), `_fetch_github_raw_file` (HTTP errors, timeouts, bad tokens), `prompt_for_reproduce_args` / `prompt_for_repo_and_branch` (input validation) + - **ci_fix.py**: `extract_run_id_from_url` (valid/invalid/malformed URLs), `extract_info_from_ci_url` (API errors, missing fields, bad URLs), `gather_session_info` (all input combinations: with/without CI URL, new/resume, empty fields) + - **containers.py**: `sanitize_container_name`, `container_exists`/`container_is_running` (mock docker calls) + - **preflight.py**: each check in isolation (mock docker/gh/claude) + - **cli.py**: `dispatch_subcommand` routing, `_handle_container_collision` (all three choices) +- [ ] Input validation / boundary tests: verify weird input combinations at module boundaries (e.g. gather_session_info output dict is always valid input for reproduce_ci, prompt outputs satisfy reproduce_ci preconditions) +- [ ] Integration-style tests: mock Docker/GitHub and run full flows end-to-end (new session, resume session, reproduce-only) diff --git a/bin/ci_tool/__init__.py b/bin/ci_tool/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bin/ci_tool/__main__.py b/bin/ci_tool/__main__.py new file mode 100644 index 0000000..1123b1a --- /dev/null +++ b/bin/ci_tool/__main__.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +"""Entry point for: python3 -m ci_tool or python3 /path/to/ci_tool""" +import os +import subprocess +import sys + +ci_tool_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.dirname(ci_tool_dir)) + +try: + from ci_tool.cli import main +except ImportError: + requirements_file = os.path.join(ci_tool_dir, "requirements.txt") + subprocess.check_call( + [sys.executable, "-m", "pip", "install", "--user", "--quiet", + "-r", requirements_file] + ) + from ci_tool.cli import main + +if __name__ == "__main__": + main() diff --git a/bin/ci_tool/ci_context/CLAUDE.md b/bin/ci_tool/ci_context/CLAUDE.md new file mode 100644 index 0000000..6eed6bf --- /dev/null +++ b/bin/ci_tool/ci_context/CLAUDE.md @@ -0,0 +1,125 @@ +# CI Context — Extend Robotics ROS1 Workspace + +You are inside a CI reproduction container. The ROS workspace is at `/ros_ws/`. +Source code is under `/ros_ws/src/`. Built packages install to `/ros_ws/install/`. + +## Token Efficiency + +Use Grep to search log files for relevant errors — never read entire log files. +When examining test output, search for FAILURE, FAILED, ERROR, or assertion messages. +Pipe long command output through `tail -200` or `grep` to avoid dumping huge logs. + +Always use the helper functions (`colcon_build`, `colcon_build_no_deps`, `colcon_test_this_package`) instead of raw `colcon` commands — they limit output to the last 50 lines and log full output to `/ros_ws/.colcon_build.log` and `/ros_ws/.colcon_test.log`. If you need more detail, Grep the log files. + +## Environment Setup + +```bash +source /opt/ros/noetic/setup.bash && source /ros_ws/install/setup.bash +source ~/.helper_bash_functions +``` + +## Build Commands + +```bash +source ~/.helper_bash_functions + +# Full workspace build +colcon_build + +# Single package (with deps) +colcon_build + +# Single package (no deps — use after editing only Python in that package) +colcon_build_no_deps + +# Multiple packages +colcon_build " " +``` + +Python imports resolve to installed `.pyc` in `/ros_ws/install/`, not source. Always build after editing Python before running tests. + +## Testing + +```bash +# Run all tests in a package +colcon_test_this_package + +# Run specific rostest +rostest .test +``` + +`colcon_test_this_package` does NOT build dependencies — only builds and tests the named package. If you changed code in other packages, build those first. + +When reporting test results, check per-package XML files in `build//test_results/` — `colcon test-result --verbose` without `--test-result-base` aggregates stale results from the entire `build/` directory. + +## Linting + +```bash +source ~/.helper_bash_functions && cd /ros_ws/src/er_interface/ && er_python_linters_here +``` + +Linters must pass including warnings. Don't use `# pylint: disable` unless absolutely necessary. Always lint before committing. + +After changing `er_robot_description`, run `rosrun er_interface xacro_lint.py` to validate all assembly XACRO permutations. `er_python_linters_here` does NOT run this. + +## Style Guide + +- Code must follow KISS, YAGNI, and SOLID principles. +- Self-documenting code with verbose variable names. Comments only for maths or external doc links. +- Fail fast — no fallback behaviour or silent defaults. If something goes wrong, raise a clear exception. +- Never add try/except, None-return, or fallback behaviour to existing functions that currently raise on error. +- Scope changes to what was requested — no cosmetic cleanups, no "while I'm here" changes. +- Do not rename functions, variables, or files unless renaming is the task. +- Keep diffs minimal. Every changed line must serve the requested purpose. +- Do not mention Claude in commit messages or PRs. + +## Common CI Failure Patterns + +1. **Missing package.xml dependencies**: Code works locally because a dependency is installed system-wide, but CI only installs declared dependencies. Check ``, ``, and `` tags match all imports. + +2. **Import errors**: If a node crashes with `ModuleNotFoundError`, the package is missing from `package.xml`. Trace the import chain to find which dependency is needed. + +3. **Race conditions in launch files**: Use `conditional_delayed_rostool` to wait for topics/params/services before launching dependent nodes. Don't restructure node startup code or add timeouts. + +4. **Stale test results**: `colcon test-result --verbose` aggregates stale results from the entire `build/` directory. Always check per-package XML files in `build//test_results/`. + +5. **XACRO validation failures**: After changing `er_robot_description`, run `rosrun er_interface xacro_lint.py`. The CI `er_xacro_checks.yml` workflow runs this automatically. + +6. **Test tolerance failures**: Check per-joint tolerance overrides in test config YAML files. Some joints (e.g. thumb) have higher variability under IK. + +## Architecture Overview + +ROS Noetic catkin workspace for multi-robot assemblies: + +- **er_robot_description**: URDF/XACRO files for all robots +- **er_robot_config**: Configuration generation (SRDF, controllers, kinematics from Jinja2 templates) +- **er_robot_launch**: Main launch entrypoint for complete robot system +- **er_robot_hand_interface**: Human hand pose projection to robot hands/grippers via IK +- **er_state_validity_checker**: In-process collision checking, joint limits, manipulability via MoveIt +- **er_auto_moveit_config**: MoveIt configuration generation +- **er_utilities_common**: Shared utilities (conditional_delayed_rostool, joint state aggregation) +- **er_moveit_collisions_updater_python**: Automatic MoveIt collision pair exclusion via randomised sampling + +Configuration pipeline: Assembly configs → robot configs → Jinja2 templates → URDF/SRDF/controllers. Generated files output to `/tmp/`. + +## Learnings + +If `/ros_ws/.ci_learnings.md` exists, read it before starting — it contains lessons from +previous CI fix sessions for this repo. + +After fixing CI failures, update `/ros_ws/.ci_learnings.md` with any new insights: +- Root causes that were non-obvious +- Patterns that recur (e.g. "this repo often breaks because of X") +- Debugging techniques that saved time +- False leads to avoid next time + +Keep it concise. This file persists across sessions. + +## Design Principles + +- Explicit over implicit. Use named fields with clear values, not absence-of-key or empty-dict semantics. +- Mode-dispatching methods must branch on mode first. No computation before the branch unless genuinely shared. +- After any code change, run existing tests before declaring done. A test passing before and failing after is a regression. +- Before modifying shared helper functions, read the whole file and check all callers. +- When adding fail-fast errors, trace all call paths. +- Check assumptions empirically before asserting them. Don't dismiss failures without data. diff --git a/bin/ci_tool/ci_fix.py b/bin/ci_tool/ci_fix.py new file mode 100644 index 0000000..6048355 --- /dev/null +++ b/bin/ci_tool/ci_fix.py @@ -0,0 +1,624 @@ +#!/usr/bin/env python3 +"""Fix CI test failures using Claude Code inside a container.""" +# pylint: disable=duplicate-code # shared imports with ci_reproduce.py +from __future__ import annotations + +import json +import os +import subprocess +import sys +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +from InquirerPy import inquirer +from rich.console import Console +from rich.panel import Panel + +from ci_tool.claude_setup import ( + copy_ci_context, + copy_claude_credentials, + copy_display_script, + copy_learnings_from_container, + copy_learnings_to_container, + inject_colcon_wrappers, + inject_rerun_tests_function, + inject_resume_function, + is_claude_installed_in_container, + save_package_list, + seed_claude_state, + setup_claude_in_container, +) +from ci_tool.ci_reproduce import ( + _parse_repo_url, + prompt_for_reproduce_args, + reproduce_ci, +) +from ci_tool.containers import ( + container_exists, + container_is_running, + docker_exec, + docker_exec_interactive, + list_ci_containers, + remove_container, + sanitize_container_name, + start_container, +) +from ci_tool.preflight import run_all_preflight_checks, PreflightError + +console = Console() + +SUMMARY_FORMAT = ( + "When done, print EXACTLY this format:\n\n" + "--- SUMMARY ---\n" + "Problem: \n" + "Fix: \n" + "Assumptions: \n\n" + "--- COMMIT MESSAGE ---\n" + "\n" + "--- END ---" +) + +ROS_SOURCE_PREAMBLE = ( + "You are inside a CI reproduction container at /ros_ws. " + "Source the ROS workspace: " + "`source /opt/ros/noetic/setup.bash && source /ros_ws/install/setup.bash`." + "\n\n" +) + +ANALYSIS_PROMPT_TEMPLATE = ( + ROS_SOURCE_PREAMBLE + + "The CI tests have already been run. Analyse the failures:\n" + "1. Use Grep to search /ros_ws/test_output.log for FAILURE, FAILED, ERROR, " + "and assertion messages. Do NOT read the entire file.\n" + "2. For each failing test, report:\n" + " - Package and test name\n" + " - The error/assertion message\n" + " - Your hypothesis for the root cause\n" + "3. Suggest a fix strategy for each failure\n\n" + "Do NOT make any code changes. Only analyse and report.\n" + "{extra_context}" +) + +CI_COMPARE_EXTRA_CONTEXT_TEMPLATE = ( + "\nAlso investigate the CI run: {ci_run_url}\n" + "- Verify local and CI are on the same commit:\n" + " - Local: check HEAD in the repo under /ros_ws/src/\n" + " - CI: `gh api repos/{owner_repo}/actions/runs/{run_id}" + " --jq '.head_sha'`\n" + " - If they differ, determine whether the missing/extra commits " + "explain the failure\n" + "- Fetch CI logs: `gh run view {run_id} --log-failed 2>&1 | tail -200` " + "(increase if needed, but avoid dumping full logs)\n" + "- Compare CI failures with local test results\n" +) + +FIX_PROMPT_TEMPLATE = ( + "The user has reviewed your analysis. Their feedback:\n" + "{user_feedback}\n\n" + "Now fix the CI failures based on this understanding.\n" + "Rebuild the affected packages and re-run the failing tests to verify.\n" + "Iterate until all tests pass.\n\n" + + SUMMARY_FORMAT +) + +FIX_MODE_CHOICES = [ + {"name": "Fix CI failures (from test_output.log)", "value": "fix_from_log"}, + {"name": "Compare with GitHub Actions CI run", "value": "compare_ci_run"}, + {"name": "Custom prompt", "value": "custom"}, +] + +CLAUDE_STDERR_LOG = "/ros_ws/.claude_stderr.log" + + +def read_container_state(container_name): + """Read the ci_fix state file from a container. Returns dict or None.""" + result = subprocess.run( + ["docker", "exec", container_name, + "cat", "/ros_ws/.ci_fix_state.json"], + capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + return None + try: + return json.loads(result.stdout) + except json.JSONDecodeError: + console.print( + f"[yellow]State file exists but contains invalid JSON: " + f"{result.stdout[:200]}[/yellow]" + ) + return None + + +def extract_run_id_from_url(ci_run_url): + """Extract the numeric run ID from a GitHub Actions URL. + + Handles URLs like: + https://github.com/org/repo/actions/runs/12345678901 + https://github.com/org/repo/actions/runs/12345678901/job/98765 + """ + parts = ci_run_url.rstrip("/").split("/runs/") + if len(parts) < 2: + raise ValueError(f"Cannot extract run ID from URL: {ci_run_url}") + run_id = parts[1].split("/")[0] + if not run_id.isdigit(): + raise ValueError(f"Run ID is not numeric: {run_id}") + return run_id + + +def extract_info_from_ci_url(ci_run_url): + """Extract repo URL, branch, and run ID from a GitHub Actions URL.""" + run_id = extract_run_id_from_url(ci_run_url) + + if "github.com/" not in ci_run_url or "/actions/" not in ci_run_url: + raise ValueError(f"Not a valid GitHub Actions URL: {ci_run_url}") + + owner_repo = ci_run_url.split("github.com/")[1].split("/actions/")[0] + repo_url = f"https://github.com/{owner_repo}" + + token = os.environ.get("GH_TOKEN") or os.environ.get("ER_SETUP_TOKEN") + if not token: + raise ValueError( + "No GitHub token found (GH_TOKEN or ER_SETUP_TOKEN)" + ) + + api_url = ( + f"https://api.github.com/repos/{owner_repo}/actions/runs/{run_id}" + ) + request = Request(api_url, headers={ + "Authorization": f"token {token}", + "Accept": "application/vnd.github.v3+json", + }) + try: + with urlopen(request, timeout=15) as response: + data = json.loads(response.read()) + except HTTPError as error: + raise RuntimeError( + f"Failed to fetch run info for {owner_repo} " + f"run {run_id} (HTTP {error.code})" + ) from error + except URLError as error: + raise RuntimeError( + f"Cannot reach GitHub API: {error.reason}" + ) from error + + return { + "repo_url": repo_url, + "owner_repo": owner_repo, + "branch": data["head_branch"], + "run_id": run_id, + "ci_run_url": ci_run_url, + } + + +def prompt_for_session_name(branch_hint=None): + """Ask user for a session name. Returns full container name (er_ci_). + + Exits if the container already exists. + """ + default = sanitize_container_name(branch_hint) if branch_hint else "" + name = inquirer.text( + message="Session name (container will be er_ci_):", + default=default, + validate=lambda n: len(n.strip()) > 0, + invalid_message="Session name cannot be empty", + ).execute().strip() + + container_name = f"er_ci_{sanitize_container_name(name)}" + console.print(f" Container name: [cyan]{container_name}[/cyan]") + + if container_exists(container_name): + console.print( + f"[red]Container '{container_name}' already exists. " + f"Choose a different name or clean up first.[/red]" + ) + sys.exit(1) + + return container_name + + +def _prompt_resume_session(container_name): + """Prompt user to resume or start fresh in an existing container. + + Returns a resume dict for gather_session_info(). + """ + if not container_is_running(container_name): + start_container(container_name) + + resume_session_id = None + state = read_container_state(container_name) + if state and state.get("session_id"): + session_id = state["session_id"] + phase = state.get("phase", "unknown") + attempt = state.get("attempt_count", 0) + console.print( + f" [dim]Previous session: {phase} " + f"(attempt {attempt}, id: {session_id})[/dim]" + ) + + resume_choice = inquirer.select( + message=( + "Resume previous Claude session or start fresh?" + ), + choices=[ + { + "name": f"Resume session ({phase})", + "value": "resume", + }, + { + "name": "Start fresh fix attempt", + "value": "fresh", + }, + ], + ).execute() + + if resume_choice == "resume": + resume_session_id = session_id + + return { + "mode": "resume", + "container_name": container_name, + "resume_session_id": resume_session_id, + } + + +def gather_session_info(): + """Collect all session information up front via interactive prompts. + + Returns a dict with 'mode' key: + - mode='new': container_name, repo_url, branch, only_needed_deps, + ci_run_info (or None) + - mode='resume': container_name, resume_session_id (or None) + """ + existing = list_ci_containers() + + if existing: + choices = [{"name": "Start new session", "value": "_new"}] + for container in existing: + choices.append({ + "name": ( + f"Resume '{container['name']}' ({container['status']})" + ), + "value": container["name"], + }) + + selection = inquirer.select( + message="Select a session:", + choices=choices, + ).execute() + + if selection != "_new": + return _prompt_resume_session(selection) + + # New session: collect all info up front + ci_run_info = None + ci_run_url = inquirer.text( + message="GitHub Actions run URL (leave blank to skip):", + default="", + ).execute().strip() + + if ci_run_url: + ci_run_info = extract_info_from_ci_url(ci_run_url) + repo_url = ci_run_info["repo_url"] + branch = ci_run_info["branch"] + console.print(f" [green]Repo:[/green] {repo_url}") + console.print(f" [green]Branch:[/green] {branch}") + console.print(f" [green]Run ID:[/green] {ci_run_info['run_id']}") + only_needed_deps = not inquirer.confirm( + message="Build everything (slower, disable --only-needed-deps)?", + default=False, + ).execute() + else: + repo_url, branch, only_needed_deps = prompt_for_reproduce_args() + + container_name = prompt_for_session_name(branch) + + return { + "mode": "new", + "container_name": container_name, + "repo_url": repo_url, + "branch": branch, + "only_needed_deps": only_needed_deps, + "ci_run_info": ci_run_info, + } + + +def select_fix_mode(): + """Let the user choose how Claude should fix CI failures. + + Returns (ci_run_info_or_none, custom_prompt_or_none). + """ + mode = inquirer.select( + message="How should Claude fix CI?", + choices=FIX_MODE_CHOICES, + default="fix_from_log", + ).execute() + + if mode == "fix_from_log": + return None, None + + if mode == "compare_ci_run": + ci_run_url = inquirer.text( + message="GitHub Actions run URL:", + validate=lambda url: "/runs/" in url, + invalid_message=( + "URL must contain /runs/ " + "(e.g. https://github.com/org/repo/actions/runs/12345)" + ), + ).execute() + return extract_info_from_ci_url(ci_run_url), None + + custom_prompt = inquirer.text( + message="Enter your custom prompt for Claude:" + ).execute() + return None, custom_prompt + + +def build_analysis_prompt(ci_run_info): + """Build the analysis prompt, optionally including CI compare context.""" + if ci_run_info: + extra_context = CI_COMPARE_EXTRA_CONTEXT_TEMPLATE.format( + **ci_run_info + ) + else: + extra_context = "" + return ANALYSIS_PROMPT_TEMPLATE.format(extra_context=extra_context) + + +def run_claude_streamed(container_name, prompt): + """Run Claude non-interactively with stream-json output.""" + escaped_prompt = prompt.replace("'", "'\\''") + claude_command = ( + f"set -o pipefail && " + f"cd /ros_ws && IS_SANDBOX=1 claude --dangerously-skip-permissions " + f"-p '{escaped_prompt}' --verbose --output-format stream-json " + f"2>{CLAUDE_STDERR_LOG} | ci_fix_display" + ) + result = docker_exec(container_name, claude_command, tty=True, check=False, quiet=True) + if result.returncode != 0: + console.print( + f"[yellow]Claude exited with code {result.returncode} — " + f"check {CLAUDE_STDERR_LOG} inside the container for details[/yellow]" + ) + + +def run_claude_resumed(container_name, session_id, prompt): + """Resume a Claude session with a new prompt, streaming output.""" + escaped_prompt = prompt.replace("'", "'\\''") + claude_command = ( + f"set -o pipefail && " + f"cd /ros_ws && IS_SANDBOX=1 claude --dangerously-skip-permissions " + f"--resume '{session_id}' -p '{escaped_prompt}' " + f"--verbose --output-format stream-json " + f"2>{CLAUDE_STDERR_LOG} | ci_fix_display" + ) + result = docker_exec(container_name, claude_command, tty=True, check=False, quiet=True) + if result.returncode != 0: + console.print( + f"[yellow]Claude exited with code {result.returncode} — " + f"check {CLAUDE_STDERR_LOG} inside the container for details[/yellow]" + ) + + +def prompt_user_for_feedback(): + """Ask user to review Claude's analysis and provide corrections.""" + feedback = inquirer.text( + message=( + "Review the analysis above. " + "Provide corrections or context (Enter to accept as-is):" + ), + default="", + ).execute().strip() + if not feedback: + return "Analysis looks correct, proceed with fixing." + return feedback + + +def refresh_claude_config(container_name): + """Refresh Claude config in an existing container.""" + console.print( + "[green]Claude already installed — refreshing config...[/green]" + ) + copy_claude_credentials(container_name) + copy_ci_context(container_name) + copy_display_script(container_name) + inject_resume_function(container_name) + inject_rerun_tests_function(container_name) + inject_colcon_wrappers(container_name) + seed_claude_state(container_name) + + +def run_claude_workflow(container_name, ci_run_info): + """Run the Claude analysis -> feedback -> fix workflow.""" + if ci_run_info: + custom_prompt = None + else: + ci_run_info, custom_prompt = select_fix_mode() + + if custom_prompt: + console.print( + "\n[bold cyan]Launching Claude Code (custom prompt)...[/bold cyan]" + ) + run_claude_streamed(container_name, custom_prompt) + else: + # Analysis phase + analysis_prompt = build_analysis_prompt(ci_run_info) + console.print( + "\n[bold cyan]Launching Claude Code " + "— analysis phase...[/bold cyan]" + ) + console.print( + "[dim]Claude will analyse failures before " + "attempting fixes[/dim]\n" + ) + run_claude_streamed(container_name, analysis_prompt) + + # User review + console.print() + try: + user_feedback = prompt_user_for_feedback() + except KeyboardInterrupt: + console.print("\n[yellow]Interrupted — skipping fix phase.[/yellow]") + return + + # Fix phase (resume session) + state = read_container_state(container_name) + session_id = state.get("session_id") if state else None + if session_id: + console.print( + "\n[bold cyan]Resuming Claude " + "— fix phase...[/bold cyan]" + ) + console.print( + "[dim]Claude will now fix the failures[/dim]\n" + ) + fix_prompt = FIX_PROMPT_TEMPLATE.format( + user_feedback=user_feedback + ) + run_claude_resumed(container_name, session_id, fix_prompt) + else: + console.print( + "\n[yellow]No session ID from analysis phase — " + "cannot resume. Dropping to shell.[/yellow]" + ) + + # Show outcome + state = read_container_state(container_name) + if state: + phase = state.get("phase", "unknown") + session_id = state.get("session_id") + attempt = state.get("attempt_count", 1) + console.print( + f"\n[bold]Claude finished — " + f"phase: {phase}, attempt: {attempt}[/bold]" + ) + if session_id: + console.print(f"[dim]Session ID: {session_id}[/dim]") + else: + console.print( + "\n[yellow]Could not read state file from container[/yellow]" + ) + + +def drop_to_shell(container_name): + """Drop user into an interactive container shell.""" + console.print("\n[bold green]Dropping into container shell.[/bold green]") + console.print("[cyan]Useful commands:[/cyan]") + console.print( + " [bold]rerun_tests[/bold] " + "— rebuild and re-run CI tests locally" + ) + console.print( + " [bold]resume_claude[/bold] " + "— resume the Claude session interactively" + ) + console.print(" [bold]git diff[/bold] — review changes") + console.print( + " [bold]git add && git commit[/bold] — commit fixes" + ) + console.print(" [dim]Repo is at /ros_ws/src/[/dim]\n") + docker_exec_interactive(container_name) + + +def _read_container_env(container_name, var_name): + """Read an environment variable from a running container.""" + result = subprocess.run( + ["docker", "exec", container_name, "printenv", var_name], + capture_output=True, text=True, check=False, + ) + return result.stdout.strip() if result.returncode == 0 else "" + + +def _resolve_org_repo(session, container_name): + """Resolve (org, repo_name) from session info or container env vars.""" + repo_url = session.get("repo_url") + if repo_url: + org, repo_name, _ = _parse_repo_url(repo_url) + return org, repo_name + + org = _read_container_env(container_name, "ORG") + repo_name = _read_container_env(container_name, "REPO_NAME") + return org, repo_name + + +def fix_ci(_args): + """Main fix workflow: gather -> preflight -> reproduce -> Claude -> shell. + + _args is accepted for backward compat but ignored (interactive only). + """ + console.print( + Panel("[bold cyan]CI Fix with Claude[/bold cyan]", expand=False) + ) + + # Step 1: Gather all session info up front + session = gather_session_info() + container_name = session["container_name"] + + if session["mode"] == "new": + # Step 2: Preflight checks + try: + gh_token = run_all_preflight_checks( + repo_url=session["repo_url"] + ) + except PreflightError as error: + console.print( + f"\n[bold red]Preflight failed:[/bold red] {error}" + ) + sys.exit(1) + + # Step 3: Reproduce CI in container + if container_exists(container_name): + remove_container(container_name) + reproduce_ci( + repo_url=session["repo_url"], + branch=session["branch"], + container_name=container_name, + gh_token=gh_token, + only_needed_deps=session["only_needed_deps"], + ) + save_package_list(container_name) + + # Step 4: Setup Claude in container + if is_claude_installed_in_container(container_name): + refresh_claude_config(container_name) + else: + setup_claude_in_container(container_name) + + # Step 4b: Copy learnings into container + org, repo_name = _resolve_org_repo(session, container_name) + if org and repo_name: + copy_learnings_to_container(container_name, org, repo_name) + + # Step 5: Run Claude + resume_session_id = session.get("resume_session_id") + try: + if resume_session_id: + console.print( + "\n[bold cyan]Resuming Claude session...[/bold cyan]" + ) + console.print( + "[dim]You are now in an interactive Claude session[/dim]\n" + ) + result = docker_exec( + container_name, + "cd /ros_ws && IS_SANDBOX=1 claude " + "--dangerously-skip-permissions " + f'--resume "{resume_session_id}"', + interactive=True, check=False, + ) + if result.returncode != 0: + console.print( + f"[yellow]Claude exited with code {result.returncode}[/yellow]" + ) + else: + run_claude_workflow( + container_name, session.get("ci_run_info") + ) + except KeyboardInterrupt: + console.print("\n[yellow]Interrupted.[/yellow]") + + # Step 6: Save learnings from container back to host + if org and repo_name: + copy_learnings_from_container(container_name, org, repo_name) + + # Step 7: Drop to shell + drop_to_shell(container_name) diff --git a/bin/ci_tool/ci_reproduce.py b/bin/ci_tool/ci_reproduce.py new file mode 100644 index 0000000..5975c70 --- /dev/null +++ b/bin/ci_tool/ci_reproduce.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 +"""Reproduce CI locally by creating a Docker container with Python Docker orchestration.""" +# pylint: disable=duplicate-code # shared imports with ci_fix.py +from __future__ import annotations + +import os +import subprocess +import tempfile +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +from InquirerPy import inquirer +from rich.console import Console +from rich.panel import Panel + +from ci_tool.containers import ( + DEFAULT_CONTAINER_NAME, + container_exists, + run_command, +) + +console = Console() + +DEFAULT_DOCKER_IMAGE = "rostooling/setup-ros-docker:ubuntu-focal-ros-noetic-desktop-latest" +DEFAULT_SCRIPTS_BRANCH = os.environ.get( + "CI_TOOL_SCRIPTS_BRANCH", "ERD-1633_reproduce_ci_locally", +) + +INTERNAL_REPO = "Extend-Robotics/er_build_tools_internal" +CONTAINER_SETUP_SCRIPT_PATH = "/tmp/ci_workspace_setup.sh" +CONTAINER_RETEST_SCRIPT_PATH = "/tmp/ci_repull_and_retest.sh" + + +def _parse_repo_url(repo_url): + """Extract org and repo name from a GitHub URL. + + Returns (org, repo_name, clean_url) tuple. + """ + if not repo_url.startswith("https://github.com/"): + raise ValueError( + f"URL must start with https://github.com/, got: {repo_url}" + ) + clean_url = repo_url.rstrip("/") + if clean_url.endswith(".git"): + clean_url = clean_url[:-4] + github_prefix = "https://github.com/" + repo_path = clean_url[len(github_prefix):] + parts = repo_path.split("/") + if len(parts) != 2 or not all(parts): + raise ValueError(f"Cannot parse org/repo from URL: {repo_url}") + return parts[0], parts[1], clean_url + + +def _fetch_github_raw_file(repo, file_path, branch, gh_token): + """Fetch a file from a GitHub repo via raw.githubusercontent.com. + + Returns the file content as a string. + Raises RuntimeError if the file cannot be fetched. + """ + url = f"https://raw.githubusercontent.com/{repo}/{branch}/{file_path}" + request = Request(url, headers={"Authorization": f"token {gh_token}"}) + try: + with urlopen(request, timeout=15) as response: + return response.read().decode() + except HTTPError as error: + raise RuntimeError( + f"Failed to fetch {file_path} from {repo} branch '{branch}' " + f"(HTTP {error.code}). Check the branch exists and your token has access." + ) from error + except URLError as error: + raise RuntimeError( + f"Cannot reach GitHub to fetch {file_path}: {error.reason}" + ) from error + + +def _validate_deps_repos_reachable(org, repo_name, branch, gh_token, deps_file="deps.repos"): + """Validate that deps.repos is reachable at the target branch before creating the container.""" + branch_for_raw = branch or "main" + deps_url = ( + f"https://raw.githubusercontent.com/{org}/{repo_name}/" + f"{branch_for_raw}/{deps_file}" + ) + console.print(f" Validating {deps_file} is reachable at: [dim]{deps_url}[/dim]") + request = Request(deps_url, headers={"Authorization": f"token {gh_token}"}) + try: + with urlopen(request, timeout=10) as response: + http_code = response.getcode() + except HTTPError as error: + http_code = error.code + hints = [ + f"Could not reach {deps_file} (HTTP {http_code})", + f"Check that '{org}/{repo_name}' exists and your token has access", + f"Check that '{deps_file}' exists at ref '{branch_for_raw}'", + ] + if branch: + hints.insert(1, f"Branch/commit '{branch}' may not exist in '{org}/{repo_name}'") + raise RuntimeError("\n ".join(hints)) from error + except URLError as error: + raise RuntimeError( + f"Cannot reach GitHub to validate {deps_file}: {error.reason}" + ) from error + + console.print(f" [green]\u2713[/green] Validation passed (HTTP {http_code})") + + +def _fetch_internal_scripts(gh_token, scripts_branch): + """Fetch ci_workspace_setup.sh and ci_repull_and_retest.sh from er_build_tools_internal. + + Returns (setup_script_path, retest_script_path) as temporary file paths on the host. + """ + console.print( + f" Fetching CI scripts from [cyan]{INTERNAL_REPO}[/cyan] " + f"branch [cyan]{scripts_branch}[/cyan]" + ) + + setup_content = _fetch_github_raw_file( + INTERNAL_REPO, "bin/ci_workspace_setup.sh", scripts_branch, gh_token, + ) + retest_content = _fetch_github_raw_file( + INTERNAL_REPO, "bin/ci_repull_and_retest.sh", scripts_branch, gh_token, + ) + + # Scripts are bind-mounted into the container, so they must persist on the + # host for the container's lifetime. Temp dir is not cleaned up here. + script_dir = tempfile.mkdtemp(prefix="ci_reproduce_scripts_") + setup_script_host_path = os.path.join(script_dir, "ci_workspace_setup.sh") + retest_script_host_path = os.path.join(script_dir, "ci_repull_and_retest.sh") + + with open(setup_script_host_path, "w", encoding="utf-8") as script_file: + script_file.write(setup_content) + os.chmod(setup_script_host_path, 0o755) + + with open(retest_script_host_path, "w", encoding="utf-8") as script_file: + script_file.write(retest_content) + os.chmod(retest_script_host_path, 0o755) + + console.print(" [green]\u2713[/green] Scripts fetched and saved to temp directory") + return setup_script_host_path, retest_script_host_path + + +def _build_graphical_docker_args(): + """Build Docker args for X11/NVIDIA graphical forwarding. + + Raises RuntimeError if DISPLAY is not set. + """ + display = os.environ.get("DISPLAY") + if not display: + raise RuntimeError( + "Graphical mode requires DISPLAY to be set (X11 forwarding). " + "Set DISPLAY or pass graphical=False." + ) + + console.print(" Enabling graphical forwarding (X11 + NVIDIA)...") + xhost_result = subprocess.run( + ["xhost", "+local:"], check=False, capture_output=True + ) + if xhost_result.returncode != 0: + console.print(" [yellow]xhost +local: failed — graphical forwarding may not work[/yellow]") + + return [ + "--runtime", "nvidia", + "--gpus", "all", + "--privileged", + "--security-opt", "seccomp=unconfined", + "-v", "/tmp/.X11-unix:/tmp/.X11-unix:rw", + "-e", f"DISPLAY={display}", + "-e", "QT_X11_NO_MITSHM=1", + "-e", "NVIDIA_DRIVER_CAPABILITIES=all", + "-e", "NVIDIA_VISIBLE_DEVICES=all", + ] + + +def _docker_create_and_start( + container_name, + docker_image, + env_vars, + volume_mounts, + graphical_args, +): + """Create and start a Docker container with the given configuration.""" + create_command = ["docker", "create", "--name", container_name] + create_command.extend(["--network=host", "--ipc=host"]) + + for volume_mount in volume_mounts: + create_command.extend(["-v", volume_mount]) + + for env_key, env_value in env_vars.items(): + create_command.extend(["-e", f"{env_key}={env_value}"]) + + create_command.extend(graphical_args) + create_command.extend([docker_image, "sleep", "infinity"]) + + console.print(f"\n Creating container [cyan]'{container_name}'[/cyan]...") + run_command(create_command, quiet=True) + console.print(f" [green]\u2713[/green] Container '{container_name}' created") + + console.print(f" Starting container [cyan]'{container_name}'[/cyan]...") + run_command(["docker", "start", container_name], quiet=True) + console.print(f" [green]\u2713[/green] Container '{container_name}' started") + + +def _container_path_exists(container_name, path): + """Check if a path exists inside a container.""" + result = subprocess.run( + ["docker", "exec", container_name, "test", "-e", path], + check=False, + ) + return result.returncode == 0 + + +def _docker_exec_workspace_setup(container_name): + """Run ci_workspace_setup.sh inside the container. + + Raises RuntimeError if setup fails before the build completes. + Warns (but continues) if tests fail after a successful build. + """ + console.print("\n Running CI workspace setup inside container...") + try: + result = subprocess.run( + ["docker", "exec", container_name, "bash", CONTAINER_SETUP_SCRIPT_PATH], + check=False, + ) + except KeyboardInterrupt: + console.print( + "\n[yellow]Interrupted during workspace setup " + "-- container is still running with partial setup[/yellow]" + ) + raise + + if result.returncode == 0: + return + + # Non-zero exit: distinguish setup failure from test failure by checking + # whether the build actually completed (/ros_ws/install only exists after + # a successful colcon build). + build_completed = _container_path_exists(container_name, "/ros_ws/install") + + if not build_completed: + raise RuntimeError( + f"Workspace setup failed (exit code {result.returncode}). " + f"The build did not complete — check the output above for errors." + ) + + console.print( + f"\n[yellow]Tests exited with code {result.returncode} " + f"(build succeeded, test failures are expected)[/yellow]" + ) + + +def prompt_for_repo_and_branch(): + """Ask user for repository URL and branch name interactively. + + Returns (repo_url, branch) tuple. + """ + repo_url = inquirer.text( + message="Repository URL:", + validate=lambda url: url.startswith("https://github.com/"), + invalid_message="Must be a GitHub URL (https://github.com/...)", + ).execute() + + branch = inquirer.text( + message="Branch name:", + validate=lambda b: len(b.strip()) > 0, + invalid_message="Branch name cannot be empty", + ).execute() + + return repo_url, branch + + +def prompt_for_reproduce_args(): + """Interactively ask user for the required reproduce arguments. + + Returns (repo_url, branch, only_needed_deps) tuple. + """ + repo_url, branch = prompt_for_repo_and_branch() + + only_needed_deps = not inquirer.confirm( + message="Build everything (slower, disable --only-needed-deps)?", + default=False, + ).execute() + + return repo_url, branch, only_needed_deps + + +def reproduce_ci( # pylint: disable=too-many-arguments + repo_url, + branch, + container_name=DEFAULT_CONTAINER_NAME, + gh_token=None, + only_needed_deps=True, + scripts_branch=DEFAULT_SCRIPTS_BRANCH, + graphical=True, +): + """Create a CI reproduction container using direct Docker orchestration. + + Fetches container-side scripts from er_build_tools_internal, validates + deps.repos is reachable, creates and starts a Docker container, then + runs workspace setup inside it. + + Callers are responsible for running preflight checks before calling this. + """ + if gh_token is None: + gh_token = os.environ.get("GH_TOKEN") or os.environ.get("ER_SETUP_TOKEN") or "" + if not gh_token: + raise RuntimeError( + "No GitHub token found. Set GH_TOKEN or ER_SETUP_TOKEN environment variable." + ) + + console.print(Panel("[bold cyan]Reproduce CI[/bold cyan]", expand=False)) + + # Parse repo URL + org, repo_name, clean_repo_url = _parse_repo_url(repo_url) + console.print(f" Organization: [cyan]{org}[/cyan]") + console.print(f" Repository: [cyan]{repo_name}[/cyan]") + + # Validate deps.repos is reachable before doing anything expensive + _validate_deps_repos_reachable(org, repo_name, branch, gh_token) + + # Fetch internal scripts + setup_script_host_path, retest_script_host_path = _fetch_internal_scripts( + gh_token, scripts_branch, + ) + + # Build graphical args (fail fast if DISPLAY not set) + graphical_docker_args = [] + if graphical: + graphical_docker_args = _build_graphical_docker_args() + + # Environment variables for the container-side scripts + container_env_vars = { + "GH_TOKEN": gh_token, + "REPO_URL": clean_repo_url, + "REPO_NAME": repo_name, + "ORG": org, + "DEPS_FILE": "deps.repos", + "BRANCH": branch or "", + "ONLY_NEEDED_DEPS": "true" if only_needed_deps else "false", + "SKIP_TESTS": "false", + "ADDITIONAL_COMMAND": "", + "IS_SANDBOX": "1", + } + + # Volume mounts (scripts mounted read-only into the container) + volume_mounts = [ + f"{setup_script_host_path}:{CONTAINER_SETUP_SCRIPT_PATH}:ro", + f"{retest_script_host_path}:{CONTAINER_RETEST_SCRIPT_PATH}:ro", + ] + + # Create and start container + _docker_create_and_start( + container_name, + DEFAULT_DOCKER_IMAGE, + container_env_vars, + volume_mounts, + graphical_docker_args, + ) + + # Run workspace setup + _docker_exec_workspace_setup(container_name) + + # Safety net: verify the container survived workspace setup + if not container_exists(container_name): + raise RuntimeError( + f"Container '{container_name}' does not exist after workspace setup." + ) + + console.print(f"\n[green]\u2713 Container '{container_name}' is ready[/green]") diff --git a/bin/ci_tool/claude_session.py b/bin/ci_tool/claude_session.py new file mode 100644 index 0000000..baf0bc7 --- /dev/null +++ b/bin/ci_tool/claude_session.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Launch an interactive Claude Code session inside a CI container.""" +from __future__ import annotations + +import os +import sys + +from rich.console import Console + +from ci_tool.claude_setup import ( + copy_ci_context, + copy_claude_credentials, + is_claude_installed_in_container, + setup_claude_in_container, +) +from ci_tool.containers import ( + container_exists, + container_is_running, + list_ci_containers, + require_docker, + start_container, +) + +console = Console() + + +def select_container(args): + """Select a running CI container from args or interactive prompt.""" + if args: + return args[0] + + existing = list_ci_containers() + if not existing: + console.print("[red]No CI containers found. Run 'Reproduce CI' first.[/red]") + sys.exit(1) + + from InquirerPy import inquirer + choices = [] + for container in existing: + choices.append({ + "name": f"{container['name']} ({container['status']})", + "value": container["name"], + }) + + return inquirer.select( + message="Select a container:", + choices=choices, + ).execute() + + +def claude_session(args): + """Launch an interactive Claude session in a CI container.""" + require_docker() + container_name = select_container(args) + + if not container_exists(container_name): + console.print(f"[red]Container '{container_name}' does not exist[/red]") + sys.exit(1) + + if not container_is_running(container_name): + console.print(f"[yellow]Starting container '{container_name}'...[/yellow]") + start_container(container_name) + + if not is_claude_installed_in_container(container_name): + console.print("[cyan]Claude not installed — running full setup...[/cyan]") + setup_claude_in_container(container_name) + else: + copy_claude_credentials(container_name) + copy_ci_context(container_name) + + console.print(f"\n[bold cyan]Starting Claude session in '{container_name}'...[/bold cyan]") + console.print("[dim]Type /exit or Ctrl+C to leave Claude[/dim]\n") + + os.execvp("docker", [ + "docker", "exec", "-it", container_name, + "bash", "-c", + "source ~/.bashrc && cd /ros_ws && claude --dangerously-skip-permissions", + ]) diff --git a/bin/ci_tool/claude_setup.py b/bin/ci_tool/claude_setup.py new file mode 100644 index 0000000..7811d9a --- /dev/null +++ b/bin/ci_tool/claude_setup.py @@ -0,0 +1,541 @@ +#!/usr/bin/env python3 +"""Install Claude Code in a container and copy auth/config from host.""" +from __future__ import annotations + +import json +import os +import subprocess +import tempfile +from pathlib import Path + +from rich.console import Console + +from ci_tool.containers import docker_exec, docker_cp_to_container, run_command + +console = Console() + +LEARNINGS_HOST_DIR = Path.home() / ".ci_tool" / "learnings" +LEARNINGS_CONTAINER_PATH = "/ros_ws/.ci_learnings.md" + +CLAUDE_HOME = Path.home() / ".claude" +CI_CONTEXT_DIR = Path(__file__).parent / "ci_context" + + +def install_node_in_container(container_name): + """Install Node.js 20 LTS in the container.""" + console.print("[cyan]Installing Node.js 20 in container...[/cyan]") + docker_exec(container_name, ( + "curl -fsSL https://deb.nodesource.com/setup_20.x | bash - " + "&& apt-get install -y nodejs" + )) + + +def install_claude_in_container(container_name): + """Install Claude Code via npm in the container.""" + console.print("[cyan]Installing Claude Code in container...[/cyan]") + docker_exec(container_name, "npm install -g @anthropic-ai/claude-code") + + +def install_fzf_in_container(container_name): + """Install fzf in the container (non-critical).""" + console.print("[cyan]Installing fzf in container...[/cyan]") + result = docker_exec(container_name, "apt-get update && apt-get install -y fzf", check=False) + if result.returncode != 0: + console.print("[yellow]fzf installation failed (non-critical) — continuing[/yellow]") + + +def install_python_deps_in_container(container_name): + """Install ci_tool Python dependencies (rich, etc.) in the container.""" + requirements_file = Path(__file__).parent / "requirements.txt" + if not requirements_file.exists(): + return + + console.print("[cyan]Installing Python dependencies in container...[/cyan]") + docker_cp_to_container( + str(requirements_file), container_name, "/tmp/ci_tool_requirements.txt" + ) + docker_exec( + container_name, + "pip install --quiet -r /tmp/ci_tool_requirements.txt", + ) + + +def copy_claude_credentials(container_name): + """Copy Claude credentials into the container.""" + credentials_path = CLAUDE_HOME / ".credentials.json" + if not credentials_path.exists(): + raise RuntimeError(f"Claude credentials not found at {credentials_path}") + + console.print("[cyan]Copying Claude credentials...[/cyan]") + docker_exec(container_name, "mkdir -p /root/.claude") + docker_cp_to_container( + str(credentials_path), container_name, "/root/.claude/.credentials.json" + ) + + +def copy_claude_config(container_name): + """Copy CLAUDE.md and modified settings.json into the container.""" + claude_md_path = CLAUDE_HOME / "CLAUDE.md" + settings_path = CLAUDE_HOME / "settings.json" + + if claude_md_path.exists(): + console.print("[cyan]Copying CLAUDE.md...[/cyan]") + docker_cp_to_container(str(claude_md_path), container_name, "/root/.claude/CLAUDE.md") + + if settings_path.exists(): + console.print("[cyan]Copying settings.json (modified for dangerous mode)...[/cyan]") + with open(settings_path, encoding="utf-8") as settings_file: + settings = json.load(settings_file) + + settings.setdefault("permissions", {})["defaultMode"] = "bypassPermissions" + + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp: + json.dump(settings, tmp, indent=2) + tmp_path = tmp.name + + try: + docker_cp_to_container(tmp_path, container_name, "/root/.claude/settings.json") + finally: + os.unlink(tmp_path) + + +def copy_ci_context(container_name): + """Copy CI-specific CLAUDE.md into the container, replacing the host's global CLAUDE.md.""" + ci_claude_md = CI_CONTEXT_DIR / "CLAUDE.md" + if not ci_claude_md.exists(): + console.print("[yellow]CI context CLAUDE.md not found, skipping[/yellow]") + return + + console.print("[cyan]Copying CI context CLAUDE.md...[/cyan]") + docker_exec(container_name, "mkdir -p /root/.claude") + docker_cp_to_container(str(ci_claude_md), container_name, "/root/.claude/CLAUDE.md") + + +def copy_display_script(container_name): + """Copy the stream-json display processor into the container.""" + display_script = Path(__file__).parent / "display_progress.py" + if not display_script.exists(): + raise RuntimeError(f"display_progress.py not found at {display_script}") + + console.print("[cyan]Copying ci_fix display script...[/cyan]") + docker_cp_to_container( + str(display_script), container_name, "/usr/local/bin/ci_fix_display" + ) + docker_exec(container_name, "chmod +x /usr/local/bin/ci_fix_display") + + +RERUN_TESTS_FUNCTION = r''' +rerun_tests() { + local packages_file="/ros_ws/.ci_packages" + if [ ! -f "$packages_file" ]; then + echo "No package list found at $packages_file" + return 1 + fi + local packages + packages=$(tr '\n' ' ' < "$packages_file") + echo "Rebuilding and testing: ${packages}" + cd /ros_ws + colcon build --packages-select ${packages} --cmake-args -DSETUPTOOLS_DEB_LAYOUT=OFF \ + 2>&1 | tee /ros_ws/.colcon_build.log | tail -n 50 + local build_ret=${PIPESTATUS[0]} + local build_total=$(wc -l < /ros_ws/.colcon_build.log) + [ "$build_total" -gt 50 ] && echo "--- (last 50 of $build_total lines — full log: /ros_ws/.colcon_build.log) ---" + if [ "$build_ret" -ne 0 ]; then + echo "Build failed (exit code $build_ret)" + return $build_ret + fi + source /ros_ws/install/setup.bash + + local verbose_packages="" + while IFS=$'\t' read -r name path _; do + if echo " ${packages} " | grep -q " ${name} "; then + if [ -f "${path}/.verbose_tests" ]; then + verbose_packages="${verbose_packages} ${name}" + fi + fi + done < <(colcon list 2>/dev/null) + verbose_packages="${verbose_packages## }" + + if [ -n "${verbose_packages}" ]; then + echo "Packages with verbose tests: ${verbose_packages}" + colcon test --packages-select ${packages} --packages-skip ${verbose_packages} \ + 2>&1 | tee /ros_ws/.colcon_test.log | tail -n 50 + colcon test --packages-select ${verbose_packages} --event-handlers console_direct+ \ + 2>&1 | tee -a /ros_ws/.colcon_test.log + else + colcon test --packages-select ${packages} \ + 2>&1 | tee /ros_ws/.colcon_test.log | tail -n 50 + fi + local test_total=$(wc -l < /ros_ws/.colcon_test.log) + [ "$test_total" -gt 50 ] && echo "--- (last 50 of $test_total lines — full log: /ros_ws/.colcon_test.log) ---" + for pkg in ${packages}; do + if ! colcon test-result --test-result-base "build/$pkg/test_results"; then + python3 - "build/$pkg/test_results" <<'PYEOF' +import sys, xml.etree.ElementTree as ET +from pathlib import Path +for p in sorted(Path(sys.argv[1]).rglob("*.xml")): + for tc in ET.parse(p).iter("testcase"): + for f in list(tc.iter("failure")) + list(tc.iter("error")): + tag = "FAIL" if f.tag == "failure" else "ERROR" + print(f"\n {tag}: {tc.get('classname', '')}.{tc.get('name', '')}") + if f.text: + lines = f.text.strip().splitlines() + for l in lines[:20]: + print(f" {l}") + if len(lines) > 20: + print(f" ... ({len(lines) - 20} more lines)") +PYEOF + fi + done +} +''' + + +RESUME_CLAUDE_FUNCTION = r''' +resume_claude() { + local state_file="/ros_ws/.ci_fix_state.json" + if [ ! -f "$state_file" ]; then + echo "No ci_fix state found. Run ci_fix first." + return 1 + fi + local session_id + session_id=$(python3 -c "import json,sys; print(json.load(open('$state_file'))['session_id'])") + if [ -z "$session_id" ] || [ "$session_id" = "None" ]; then + echo "No session_id in state file. Starting fresh Claude session." + cd /ros_ws && IS_SANDBOX=1 claude --dangerously-skip-permissions + return + fi + echo "Resuming Claude session ${session_id}..." + cd /ros_ws && IS_SANDBOX=1 claude --dangerously-skip-permissions --resume "$session_id" +} +''' + + +COLCON_WRAPPERS = r''' +colcon_build() { + cd /ros_ws && source /opt/ros/noetic/setup.bash + local cmd="colcon build --cmake-args -DSETUPTOOLS_DEB_LAYOUT=OFF" + [ -n "$1" ] && cmd="$cmd --packages-up-to $1" + echo "$cmd" + eval "$cmd" 2>&1 | tee /ros_ws/.colcon_build.log | tail -n 50 + local ret=${PIPESTATUS[0]} + local total=$(wc -l < /ros_ws/.colcon_build.log) + [ "$total" -gt 50 ] && echo "--- (last 50 of $total lines — full log: /ros_ws/.colcon_build.log) ---" + source /ros_ws/install/setup.bash 2>/dev/null + return $ret +} + +colcon_build_no_deps() { + cd /ros_ws && source /opt/ros/noetic/setup.bash + local cmd="colcon build --cmake-args -DSETUPTOOLS_DEB_LAYOUT=OFF" + [ -n "$1" ] && cmd="$cmd --packages-select $1" + echo "$cmd" + eval "$cmd" 2>&1 | tee /ros_ws/.colcon_build.log | tail -n 50 + local ret=${PIPESTATUS[0]} + local total=$(wc -l < /ros_ws/.colcon_build.log) + [ "$total" -gt 50 ] && echo "--- (last 50 of $total lines — full log: /ros_ws/.colcon_build.log) ---" + source /ros_ws/install/setup.bash 2>/dev/null + return $ret +} + +colcon_build_this() { + colcon_build "$(find_cmake_project_names_from_dir .)" +} + +colcon_test_these_packages() { + cd /ros_ws + colcon_build_no_deps "$1" + local build_ret=$? + [ "$build_ret" -ne 0 ] && return $build_ret + source /ros_ws/install/setup.bash + colcon test --packages-select $1 2>&1 | tee /ros_ws/.colcon_test.log | tail -n 50 + local total=$(wc -l < /ros_ws/.colcon_test.log) + [ "$total" -gt 50 ] && echo "--- (last 50 of $total lines — full log: /ros_ws/.colcon_test.log) ---" + for pkg in $1; do + if ! colcon test-result --test-result-base "build/$pkg/test_results"; then + type _show_test_failures &>/dev/null && _show_test_failures "build/$pkg/test_results" + fi + done +} + +colcon_test_this_package() { + colcon_test_these_packages "$@" +} +''' + + +def inject_resume_function(container_name): + """Add resume_claude bash function to the container's bashrc.""" + console.print("[cyan]Injecting resume_claude function...[/cyan]") + marker = "# ci_fix resume_claude" + check_command = f"grep -q '{marker}' /root/.bashrc" + already_present = docker_exec( + container_name, check_command, check=False, quiet=True, + ) + if already_present.returncode == 0: + return + + docker_exec( + container_name, + f"echo '{marker}' >> /root/.bashrc && cat >> /root/.bashrc << 'RESUME_EOF'\n" + f"{RESUME_CLAUDE_FUNCTION}\nRESUME_EOF", + quiet=True, + ) + + +def inject_colcon_wrappers(container_name): + """Add output-limiting colcon wrapper functions to the container. + + Appends to /root/.helper_bash_functions so wrappers override originals. + Uses a marker comment for idempotent injection. + """ + console.print("[cyan]Injecting colcon output wrappers...[/cyan]") + target = "/root/.helper_bash_functions" + marker = "# ci_fix colcon_wrappers" + + docker_exec(container_name, f"touch {target}", quiet=True) + already_present = docker_exec( + container_name, f"grep -q '{marker}' {target}", + check=False, quiet=True, + ) + if already_present.returncode == 0: + return + + docker_exec( + container_name, + f"echo '{marker}' >> {target} && cat >> {target} << 'WRAPPERS_EOF'\n" + f"{COLCON_WRAPPERS}\nWRAPPERS_EOF", + quiet=True, + ) + + +def save_package_list(container_name): + """Run colcon list in the container and save package names to /ros_ws/.ci_packages.""" + console.print("[cyan]Saving workspace package list...[/cyan]") + result = docker_exec( + container_name, + "cd /ros_ws && colcon list --names-only > /ros_ws/.ci_packages", + check=False, + quiet=True, + ) + if result.returncode != 0: + console.print( + "[yellow]Could not save package list (colcon list failed). " + "The 'rerun_tests' helper will not work.[/yellow]" + ) + + +def inject_rerun_tests_function(container_name): + """Add rerun_tests bash function to the container's bashrc.""" + console.print("[cyan]Injecting rerun_tests function...[/cyan]") + marker = "# ci_fix rerun_tests" + check_command = f"grep -q '{marker}' /root/.bashrc" + already_present = docker_exec( + container_name, check_command, check=False, quiet=True, + ) + if already_present.returncode == 0: + return + + docker_exec( + container_name, + f"echo '{marker}' >> /root/.bashrc && cat >> /root/.bashrc << 'RERUN_EOF'\n" + f"{RERUN_TESTS_FUNCTION}\nRERUN_EOF", + quiet=True, + ) + + +def copy_claude_memory(container_name): + """Copy Claude project memory files into the container.""" + projects_dir = CLAUDE_HOME / "projects" + if not projects_dir.exists(): + return + + console.print("[cyan]Copying Claude memory files...[/cyan]") + for project_dir in projects_dir.iterdir(): + memory_dir = project_dir / "memory" + if not memory_dir.exists(): + continue + memory_files = [f for f in memory_dir.iterdir() if f.is_file()] + if not memory_files: + continue + + container_memory_path = f"/root/.claude/projects/{project_dir.name}/memory" + docker_exec(container_name, f"mkdir -p {container_memory_path}") + for memory_file in memory_files: + docker_cp_to_container( + str(memory_file), + container_name, + f"{container_memory_path}/{memory_file.name}", + ) + + +def _learnings_host_path(org, repo_name): + """Return the host path for a repo's learnings file.""" + return LEARNINGS_HOST_DIR / f"{org}_{repo_name}.md" + + +def copy_learnings_to_container(container_name, org, repo_name): + """Copy repo-specific learnings file into the container (if it exists).""" + host_path = _learnings_host_path(org, repo_name) + if not host_path.exists(): + return + console.print("[cyan]Copying CI learnings into container...[/cyan]") + docker_cp_to_container(str(host_path), container_name, LEARNINGS_CONTAINER_PATH) + + +def copy_learnings_from_container(container_name, org, repo_name): + """Copy learnings file back from container to host (if Claude updated it).""" + result = subprocess.run( + ["docker", "exec", container_name, "test", "-s", LEARNINGS_CONTAINER_PATH], + check=False, + ) + if result.returncode != 0: + return + + host_path = _learnings_host_path(org, repo_name) + host_path.parent.mkdir(parents=True, exist_ok=True) + run_command( + ["docker", "cp", f"{container_name}:{LEARNINGS_CONTAINER_PATH}", str(host_path)], + quiet=True, + ) + console.print(f"[green]Learnings saved to {host_path}[/green]") + + +def copy_helper_bash_functions(container_name): + """Copy ~/.helper_bash_functions and source it in bashrc.""" + helper_path = Path.home() / ".helper_bash_functions" + if not helper_path.exists(): + console.print("[yellow]~/.helper_bash_functions not found, skipping[/yellow]") + return + + console.print("[cyan]Copying helper bash functions...[/cyan]") + docker_cp_to_container(str(helper_path), container_name, "/root/.helper_bash_functions") + docker_exec( + container_name, + "grep -q 'source ~/.helper_bash_functions' /root/.bashrc " + "|| echo 'source ~/.helper_bash_functions' >> /root/.bashrc", + ) + + +def get_host_git_config(key): + """Read a value from the host's git config.""" + result = subprocess.run( + ["git", "config", "--global", key], + capture_output=True, text=True, check=False, + ) + value = result.stdout.strip() + if not value: + raise RuntimeError( + f"git config --global {key} is not set on the host. " + f"Set it with: git config --global {key} 'Your Value'" + ) + return value + + +def configure_git_in_container(container_name): + """Set up git identity, token-based auth, and gh CLI auth in the container.""" + gh_token = os.environ.get("GH_TOKEN") or os.environ.get("ER_SETUP_TOKEN") or "" + + git_user_name = get_host_git_config("user.name") + git_user_email = get_host_git_config("user.email") + + console.print("[cyan]Configuring git in container...[/cyan]") + docker_exec(container_name, f'git config --global user.name "{git_user_name}"') + docker_exec(container_name, f'git config --global user.email "{git_user_email}"') + + if not gh_token: + console.print( + "[yellow]No GH_TOKEN or ER_SETUP_TOKEN found — " + "git auth and gh CLI will not be configured in container[/yellow]" + ) + return + + docker_exec( + container_name, + f'git config --global url."https://{gh_token}@github.com/"' + f'.insteadOf "https://github.com/"', + quiet=True, + ) + install_gh_cli(container_name) + + +def install_gh_cli(container_name): + """Install gh CLI in the container. + + Authentication is handled by the GH_TOKEN env var already set on the container. + """ + console.print("[cyan]Installing gh CLI in container...[/cyan]") + install_result = docker_exec(container_name, ( + "type gh >/dev/null 2>&1 || (" + "echo ' Downloading GPG key...' " + "&& curl -fSL --progress-bar -o /usr/share/keyrings/githubcli-archive-keyring.gpg " + "https://cli.github.com/packages/githubcli-archive-keyring.gpg " + "&& chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg " + "&& echo ' Adding apt repository...' " + '&& echo "deb [arch=$(dpkg --print-architecture) ' + "signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] " + 'https://cli.github.com/packages stable main" ' + "| tee /etc/apt/sources.list.d/github-cli-stable.list > /dev/null " + "&& echo ' Running apt-get update...' " + "&& apt-get update " + "&& echo ' Installing gh...' " + "&& apt-get install -y gh)" + ), check=False) + if install_result.returncode != 0: + console.print( + "[yellow]gh CLI installation failed — " + "Claude will not be able to interact with GitHub from inside the container[/yellow]" + ) + + +def seed_claude_state(container_name): + """Pre-seed /root/.claude.json so the onboarding wizard is skipped.""" + console.print("[cyan]Seeding Claude onboarding state...[/cyan]") + docker_exec( + container_name, + """python3 -c " +import json, os +path = '/root/.claude.json' +data = {} +if os.path.exists(path): + try: + data = json.load(open(path)) + except (json.JSONDecodeError, OSError): + pass +data['hasCompletedOnboarding'] = True +json.dump(data, open(path, 'w')) +" """, + quiet=True, + ) + + +def is_claude_installed_in_container(container_name): + """Check if Claude Code is already installed in the container.""" + result = subprocess.run( + ["docker", "exec", container_name, "bash", "-c", "which claude"], + capture_output=True, text=True, check=False, + ) + return result.returncode == 0 + + +def setup_claude_in_container(container_name): + """Full setup: install Claude Code and copy all config into container.""" + console.print("\n[bold cyan]Setting up Claude in container...[/bold cyan]") + + install_node_in_container(container_name) + install_claude_in_container(container_name) + seed_claude_state(container_name) + install_fzf_in_container(container_name) + install_python_deps_in_container(container_name) + copy_claude_credentials(container_name) + copy_claude_config(container_name) + copy_ci_context(container_name) + copy_display_script(container_name) + inject_resume_function(container_name) + inject_rerun_tests_function(container_name) + copy_claude_memory(container_name) + copy_helper_bash_functions(container_name) + inject_colcon_wrappers(container_name) + configure_git_in_container(container_name) + + console.print("[bold green]Claude Code is installed and configured in the container[/bold green]") diff --git a/bin/ci_tool/cli.py b/bin/ci_tool/cli.py new file mode 100644 index 0000000..6d968dc --- /dev/null +++ b/bin/ci_tool/cli.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Interactive CLI menu for CI tool.""" +# pylint: disable=import-outside-toplevel +from __future__ import annotations + +import sys + +from InquirerPy import inquirer +from rich.console import Console +from rich.panel import Panel + +console = Console() + +MENU_CHOICES = [ + {"name": "Reproduce CI (create container)", "value": "reproduce"}, + {"name": "Fix CI with Claude", "value": "fix"}, + {"name": "Claude session (interactive)", "value": "claude"}, + {"name": "Shell into container", "value": "shell"}, + {"name": "Re-run tests in container", "value": "retest"}, + {"name": "Clean up containers", "value": "clean"}, + {"name": "Exit", "value": "exit"}, +] + +HELP_TEXT = """\ +ci_tool — Fix CI failures with Claude + +Usage: ci_tool [command] + +Commands: + fix Fix CI failures with Claude + reproduce Reproduce CI environment in Docker + claude Interactive Claude session in container + shell Shell into an existing CI container + retest Re-run tests in a CI container + clean Remove CI containers + +Shortcuts: + ci_fix Alias for 'ci_tool fix' + +Run without arguments for interactive menu.""" + + +def print_help(): + """Print usage information.""" + console.print(HELP_TEXT) + + +def main(): + """Entry point - show menu or dispatch subcommand.""" + if len(sys.argv) > 1: + if sys.argv[1] in ("-h", "--help", "help"): + print_help() + return + dispatch_subcommand(sys.argv[1], sys.argv[2:]) + return + + console.print(Panel("[bold cyan]CI Tool[/bold cyan]", expand=False)) + + action = inquirer.select( + message="What would you like to do?", + choices=MENU_CHOICES, + default="fix", + ).execute() + + if action == "exit": + return + + dispatch_subcommand(action, []) + + +def dispatch_subcommand(command, args): + """Route to the appropriate subcommand handler.""" + handlers = { + "reproduce": _handle_reproduce, + "fix": _handle_fix, + "claude": _handle_claude, + "shell": _handle_shell, + "retest": _handle_retest, + "clean": _handle_clean, + } + handler = handlers.get(command) + if handler is None: + console.print(f"[red]Unknown command: {command}[/red]") + console.print(f"Available: {', '.join(handlers.keys())}") + sys.exit(1) + handler(args) + + +def _handle_container_collision(container_name): + """Handle an existing container: recreate, keep, or cancel. + + Returns True if reproduce should proceed, False to skip. + """ + from ci_tool.containers import ( + container_is_running, + remove_container, + start_container, + ) + + action = inquirer.select( + message=f"Container '{container_name}' already exists. What to do?", + choices=[ + {"name": "Remove and recreate", "value": "recreate"}, + {"name": "Keep existing (skip creation)", "value": "keep"}, + {"name": "Cancel", "value": "cancel"}, + ], + ).execute() + + if action == "cancel": + return False + if action == "recreate": + remove_container(container_name) + return True + # action == "keep" + if not container_is_running(container_name): + start_container(container_name) + console.print( + f"[green]Using existing container '{container_name}'[/green]" + ) + return False + + +def _handle_reproduce(_args): + from ci_tool.ci_reproduce import reproduce_ci, prompt_for_reproduce_args + from ci_tool.containers import DEFAULT_CONTAINER_NAME, container_exists + from ci_tool.preflight import validate_docker_available, validate_gh_token, PreflightError + + try: + validate_docker_available() + except PreflightError as error: + console.print(f"\n[bold red]Preflight failed:[/bold red] {error}") + sys.exit(1) + + repo_url, branch, only_needed_deps = prompt_for_reproduce_args() + container_name = DEFAULT_CONTAINER_NAME + + if container_exists(container_name): + if not _handle_container_collision(container_name): + return + + try: + gh_token = validate_gh_token(repo_url=repo_url) + except PreflightError as error: + console.print(f"\n[bold red]Preflight failed:[/bold red] {error}") + sys.exit(1) + + reproduce_ci( + repo_url=repo_url, + branch=branch, + container_name=container_name, + gh_token=gh_token, + only_needed_deps=only_needed_deps, + ) + + +def _handle_fix(args): + from ci_tool.ci_fix import fix_ci + fix_ci(args) + + +def _handle_claude(args): + from ci_tool.claude_session import claude_session + claude_session(args) + + +def _handle_shell(args): + from ci_tool.containers import shell_into_container + shell_into_container(args) + + +def _handle_retest(args): + from ci_tool.containers import retest_in_container + retest_in_container(args) + + +def _handle_clean(args): + from ci_tool.containers import clean_containers + clean_containers(args) diff --git a/bin/ci_tool/containers.py b/bin/ci_tool/containers.py new file mode 100644 index 0000000..7e523f9 --- /dev/null +++ b/bin/ci_tool/containers.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Docker container lifecycle management.""" +from __future__ import annotations + +import os +import re +import shutil +import subprocess +import sys + +from rich.console import Console + +console = Console() + +DEFAULT_CONTAINER_NAME = "er_ci_reproduced_testing_env" + + +def require_docker(): + """Fail fast if docker is not available.""" + if not shutil.which("docker"): + console.print("[red]Error: 'docker' command not found. Is Docker installed?[/red]") + sys.exit(1) + + +def run_command(command, capture_output=False, check=True, quiet=False): + """Run a shell command, raising on failure.""" + if not quiet: + console.print(f"[dim]$ {' '.join(command)}[/dim]") + return subprocess.run(command, capture_output=capture_output, check=check, text=True) + + +def container_exists(container_name=DEFAULT_CONTAINER_NAME): + """Check if a container exists (running or stopped).""" + result = subprocess.run( + ["docker", "ps", "-a", "--filter", f"name=^{container_name}$", "--format", "{{.Names}}"], + capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + raise RuntimeError(f"docker ps failed: {result.stderr.strip()}") + return container_name in result.stdout.strip() + + +def container_is_running(container_name=DEFAULT_CONTAINER_NAME): + """Check if a container is currently running.""" + result = subprocess.run( + ["docker", "ps", "--filter", f"name=^{container_name}$", "--format", "{{.Names}}"], + capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + raise RuntimeError(f"docker ps failed: {result.stderr.strip()}") + return container_name in result.stdout.strip() + + +def start_container(container_name=DEFAULT_CONTAINER_NAME): + """Start a stopped container.""" + run_command(["docker", "start", container_name]) + + +def remove_container(container_name=DEFAULT_CONTAINER_NAME): + """Force remove a container.""" + run_command(["docker", "rm", "-f", container_name]) + console.print(f"[green]Container '{container_name}' removed[/green]") + + +def list_ci_containers(): + """List all CI containers (er_ci_* prefix) with their status.""" + result = subprocess.run( + ["docker", "ps", "-a", "--filter", "name=er_ci_", + "--format", "{{.Names}}\t{{.Status}}"], + capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + raise RuntimeError(f"docker ps failed: {result.stderr.strip()}") + if not result.stdout.strip(): + return [] + containers = [] + for line in result.stdout.strip().split("\n"): + parts = line.split("\t") + containers.append({ + "name": parts[0], + "status": parts[1] if len(parts) > 1 else "unknown", + }) + return containers + + +def sanitize_container_name(name): + """Replace characters invalid for Docker container names with underscores.""" + return re.sub(r'[^a-zA-Z0-9_.-]', '_', name) + + +def docker_exec( # pylint: disable=too-many-arguments + container_name, command, interactive=False, tty=False, check=True, quiet=False, +): + """Run a command inside a container.""" + docker_command = ["docker", "exec", "-e", "IS_SANDBOX=1"] + if interactive: + docker_command.extend(["-it"]) + elif tty: + docker_command.extend(["-t"]) + docker_command.extend([container_name, "bash", "-c", command]) + return run_command(docker_command, check=check, quiet=quiet) + + +def docker_exec_interactive(container_name=DEFAULT_CONTAINER_NAME): + """Drop user into an interactive shell inside the container.""" + console.print(f"\n[bold cyan]Entering container '{container_name}'...[/bold cyan]") + console.print("[dim]Type 'exit' to leave the container[/dim]\n") + os.execvp("docker", ["docker", "exec", "-e", "IS_SANDBOX=1", "-it", container_name, "bash"]) + + +def docker_cp_to_container(host_path, container_name, container_path): + """Copy a file/directory from host into container.""" + run_command(["docker", "cp", host_path, f"{container_name}:{container_path}"]) + + +def shell_into_container(args): + """Shell subcommand handler.""" + require_docker() + container_name = args[0] if args else DEFAULT_CONTAINER_NAME + if not container_exists(container_name): + console.print(f"[red]Container '{container_name}' does not exist[/red]") + sys.exit(1) + if not container_is_running(container_name): + console.print(f"[yellow]Container '{container_name}' is stopped, starting...[/yellow]") + start_container(container_name) + docker_exec_interactive(container_name) + + +def retest_in_container(args): + """Re-run tests subcommand handler.""" + require_docker() + container_name = args[0] if args else DEFAULT_CONTAINER_NAME + if not container_is_running(container_name): + console.print(f"[red]Container '{container_name}' is not running[/red]") + sys.exit(1) + console.print(f"[cyan]Re-running tests in '{container_name}'...[/cyan]") + docker_exec(container_name, "bash /tmp/ci_repull_and_retest.sh") + + +def clean_containers(_args): + """Clean up CI containers.""" + require_docker() + from InquirerPy import inquirer # pylint: disable=import-outside-toplevel + + result = subprocess.run( + ["docker", "ps", "-a", "--filter", "name=er_ci_", "--format", "{{.Names}}\t{{.Status}}"], + capture_output=True, text=True, check=False, + ) + if not result.stdout.strip(): + console.print("[green]No CI containers found[/green]") + return + + console.print("[bold]CI containers:[/bold]") + containers = [] + for line in result.stdout.strip().split("\n"): + parts = line.split("\t") + name = parts[0] + status = parts[1] if len(parts) > 1 else "unknown" + containers.append({"name": f"{name} ({status})", "value": name}) + console.print(f" {name}: {status}") + + selected = inquirer.checkbox( + message="Select containers to remove:", + choices=containers, + ).execute() + + for name in selected: + remove_container(name) diff --git a/bin/ci_tool/display_progress.py b/bin/ci_tool/display_progress.py new file mode 100644 index 0000000..0449371 --- /dev/null +++ b/bin/ci_tool/display_progress.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Display human-readable progress from Claude Code stream-json output. + +Reads newline-delimited JSON from stdin (Claude Code's --output-format stream-json), +shows assistant text + tool activity via rich, captures the session_id, and +writes a state file on exit. + +Claude Code stream-json event types: + {"type":"system","subtype":"init","session_id":"..."} + {"type":"assistant","message":{"content":[{"type":"text","text":"..."}, + {"type":"tool_use","name":"..."}]}, + "session_id":"..."} + {"type":"tool_result","tool_use_id":"...","content":"...","session_id":"..."} + {"type":"result","subtype":"success","result":"...","session_id":"..."} + +Designed to run INSIDE a CI container via docker exec. +Requires: rich (from requirements.txt). +""" +from __future__ import annotations + +import json +import sys +import time +import traceback +from datetime import datetime, timezone + +from rich.console import Console +from rich.markdown import Markdown + +STATE_FILE = "/ros_ws/.ci_fix_state.json" +CLAUDE_STDERR_LOG = "/ros_ws/.claude_stderr.log" +EVENT_DEBUG_LOG = "/ros_ws/.ci_fix_events.jsonl" + +# force_terminal=True is required because docker exec may not allocate a PTY, +# which would cause Rich to suppress all ANSI output. +console = Console(stderr=True, force_terminal=True) + + +def write_state(session_id, phase, attempt_count=1): + """Write the ci_fix state file.""" + state = { + "session_id": session_id, + "phase": phase, + "attempt_count": attempt_count, + "timestamp": datetime.now(timezone.utc).isoformat(), + } + with open(STATE_FILE, "w", encoding="utf-8") as state_file: + json.dump(state, state_file, indent=2) + + +def read_existing_attempt_count(): + """Read attempt_count from existing state file, or return 0.""" + try: + with open(STATE_FILE, encoding="utf-8") as state_file: + return json.load(state_file).get("attempt_count", 0) + except FileNotFoundError: + return 0 + except (json.JSONDecodeError, KeyError) as error: + console.print(f"[yellow]State file corrupt, resetting attempt count: {error}[/yellow]") + return 0 + + +def format_elapsed(start_time): + """Format elapsed time as 'Xm Ys'.""" + elapsed_seconds = int(time.time() - start_time) + minutes = elapsed_seconds // 60 + seconds = elapsed_seconds % 60 + if minutes > 0: + return f"{minutes}m {seconds:02d}s" + return f"{seconds}s" + + +def format_tool_status(tool_counts, start_time): + """Format spinner status text showing tool activity summary.""" + if not tool_counts: + return "[cyan]Working...[/cyan]" + parts = [] + for name, count in tool_counts.items(): + if count > 1: + parts.append(f"{name} x{count}") + else: + parts.append(name) + return f"[cyan]{', '.join(parts)}[/cyan] [dim][{format_elapsed(start_time)}][/dim]" + + +def format_tool_summary(tool_counts): + """Format a final one-line summary of all tools used.""" + parts = [] + for name, count in tool_counts.items(): + if count > 1: + parts.append(f"{name} x{count}") + else: + parts.append(name) + return ", ".join(parts) + + +def flush_text_buffer(text_buffer): + """Render accumulated text as rich markdown and clear the buffer.""" + if not text_buffer: + return + combined = "".join(text_buffer) + text_buffer.clear() + if combined.strip(): + console.print(Markdown(combined)) + + +def handle_assistant_event(message, start_time, text_buffer, tool_counts, status): + """Display content blocks from an assistant message event.""" + for block in message.get("content", []): + block_type = block.get("type", "") + + if block_type == "text": + text = block.get("text", "") + if text: + text_buffer.append(text) + + elif block_type == "tool_use": + flush_text_buffer(text_buffer) + tool_name = block.get("name", "unknown") + tool_counts[tool_name] = tool_counts.get(tool_name, 0) + 1 + status.update(format_tool_status(tool_counts, start_time)) + + +def handle_event(event, start_time, text_buffer, tool_counts, status): + """Handle a single stream-json event. Returns session_id if found, else None.""" + session_id = event.get("session_id") or None + event_type = event.get("type", "") + + if event_type == "assistant": + message = event.get("message", {}) + handle_assistant_event(message, start_time, text_buffer, tool_counts, status) + + elif event_type == "tool_result": + flush_text_buffer(text_buffer) + status.update(format_tool_status(tool_counts, start_time)) + + return session_id + + +def print_session_summary(session_id, start_time, tool_counts): + """Print the final session summary after stream ends.""" + elapsed = format_elapsed(start_time) + if tool_counts: + console.print(f" [dim]Tools: {format_tool_summary(tool_counts)}[/dim]") + if session_id: + console.print( + f"\n[green]Session saved ({session_id}). " + f"Elapsed: {elapsed}. Use 'resume_claude' to continue.[/green]" + ) + return + + console.print(f"\n[yellow]No session ID captured. Elapsed: {elapsed}.[/yellow]") + try: + with open(CLAUDE_STDERR_LOG, encoding="utf-8") as stderr_log: + stderr_content = stderr_log.read().strip() + if stderr_content: + console.print("[yellow]Claude stderr output:[/yellow]") + console.print(stderr_content) + except FileNotFoundError: + console.print(f"[dim]No stderr log at {CLAUDE_STDERR_LOG}[/dim]") + + +def main(): + """Read stream-json from stdin, display progress, write state on exit.""" + session_id = None + attempt_count = read_existing_attempt_count() + 1 + phase = "fixing" + start_time = time.time() + text_buffer = [] + tool_counts = {} + + try: + with console.status("[cyan]Working...[/cyan]", spinner="dots") as status: + with open(EVENT_DEBUG_LOG, "w", encoding="utf-8") as debug_log: + for line in sys.stdin: + line = line.strip() + if not line: + continue + + debug_log.write(line + "\n") + debug_log.flush() + + try: + event = json.loads(line) + except json.JSONDecodeError: + sys.stderr.write(f" {line}\n") + continue + + event_session_id = handle_event(event, start_time, text_buffer, tool_counts, status) + if event_session_id: + session_id = event_session_id + + flush_text_buffer(text_buffer) + phase = "completed" + + except KeyboardInterrupt: + phase = "interrupted" + except (IOError, ValueError, UnicodeDecodeError): + console.print( + f"\n[red]Display processor error:[/red]\n{traceback.format_exc()}" + ) + phase = "stuck" + + write_state(session_id, phase, attempt_count) + print_session_summary(session_id, start_time, tool_counts) + + +if __name__ == "__main__": + main() diff --git a/bin/ci_tool/preflight.py b/bin/ci_tool/preflight.py new file mode 100644 index 0000000..ccf86fa --- /dev/null +++ b/bin/ci_tool/preflight.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Preflight validation - fail fast on auth issues before any docker operations.""" +from __future__ import annotations + +import json +import os +import subprocess +import time +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +from rich.console import Console +from rich.panel import Panel + +console = Console() + +CLAUDE_HOME = Path.home() / ".claude" + + +class PreflightError(RuntimeError): + """Raised when a preflight check fails.""" + + +def _check_pass(message): + console.print(f" [green]\u2713[/green] {message}") + + +def _check_fail(message): + console.print(f" [red]\u2717[/red] {message}") + + +def _check_warn(message): + console.print(f" [yellow]![/yellow] {message}") + + +def _github_api_get(endpoint, gh_token): + """Make an authenticated GET request to the GitHub API.""" + url = f"https://api.github.com{endpoint}" + request = Request(url, headers={ + "Authorization": f"token {gh_token}", + "Accept": "application/vnd.github.v3+json", + }) + with urlopen(request, timeout=10) as response: + return json.loads(response.read().decode()) + + +def validate_gh_token(repo_url=None): + """Validate GitHub token exists, is valid, and has repo access.""" + console.print("\n[bold]Checking GitHub token...[/bold]") + + gh_token = os.environ.get("GH_TOKEN") or os.environ.get("ER_SETUP_TOKEN") or "" + if not gh_token: + _check_fail("GH_TOKEN or ER_SETUP_TOKEN not set") + raise PreflightError( + "No GitHub token found. Set GH_TOKEN or ER_SETUP_TOKEN environment variable." + ) + _check_pass("Token environment variable found") + + try: + user_data = _github_api_get("/user", gh_token) + username = user_data.get("login", "unknown") + _check_pass(f"Token is valid (authenticated as: {username})") + except HTTPError as error: + _check_fail(f"Token validation failed (HTTP {error.code})") + if error.code == 401: + raise PreflightError("GitHub token is invalid or expired.") from error + raise PreflightError(f"GitHub API error: HTTP {error.code}") from error + except URLError as error: + _check_fail(f"Cannot reach GitHub API: {error.reason}") + raise PreflightError(f"Cannot reach GitHub API: {error.reason}") from error + + if repo_url: + repo_url_clean = repo_url.rstrip("/") + if repo_url_clean.endswith(".git"): + repo_url_clean = repo_url_clean[:-4] + repo_path = repo_url_clean.split("github.com/")[-1] + try: + _github_api_get(f"/repos/{repo_path}", gh_token) + _check_pass(f"Token has access to {repo_path}") + except HTTPError as error: + _check_fail(f"Cannot access {repo_path} (HTTP {error.code})") + if error.code == 404: + raise PreflightError( + f"Token does not have access to {repo_path}. " + "Check the token has 'repo' scope and org access." + ) from error + raise PreflightError( + f"GitHub API error for {repo_path}: HTTP {error.code}" + ) from error + + return gh_token + + +def validate_claude_credentials(): + """Validate Claude credentials file exists and is structurally valid.""" + console.print("\n[bold]Checking Claude credentials...[/bold]") + + credentials_path = CLAUDE_HOME / ".credentials.json" + if not credentials_path.exists(): + _check_fail(f"Credentials file not found: {credentials_path}") + raise PreflightError( + f"Claude credentials not found at {credentials_path}. " + "Run 'claude' to authenticate first." + ) + _check_pass("Credentials file exists") + + try: + with open(credentials_path, encoding="utf-8") as credentials_file: + credentials = json.load(credentials_file) + except json.JSONDecodeError as error: + _check_fail("Credentials file is not valid JSON") + raise PreflightError(f"Invalid JSON in {credentials_path}") from error + _check_pass("Credentials file is valid JSON") + + oauth_data = credentials.get("claudeAiOauth", {}) + access_token = oauth_data.get("accessToken", "") + if not access_token: + _check_fail("No accessToken found in credentials") + raise PreflightError( + "Claude credentials missing accessToken. Re-run 'claude' to authenticate." + ) + _check_pass("Access token present") + + expires_at_ms = oauth_data.get("expiresAt", 0) + now_ms = int(time.time() * 1000) + if expires_at_ms and expires_at_ms < now_ms: + refresh_token = oauth_data.get("refreshToken", "") + if refresh_token: + _check_warn("Access token expired, but refresh token exists (Claude may auto-refresh)") + else: + _check_fail("Access token expired and no refresh token") + raise PreflightError( + "Claude access token has expired. Re-run 'claude' to re-authenticate." + ) + else: + remaining_hours = (expires_at_ms - now_ms) / (1000 * 60 * 60) + _check_pass(f"Access token valid ({remaining_hours:.0f}h remaining)") + + +def validate_claude_auth_works(): + """Run a minimal Claude prompt to verify auth actually works.""" + console.print("\n[bold]Testing Claude authentication...[/bold]") + + try: + result = subprocess.run( + ["claude", "-p", "say ok", "--max-turns", "1"], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + if result.returncode == 0 and result.stdout.strip(): + _check_pass("Claude auth verified (test prompt succeeded)") + else: + stderr_preview = result.stderr.strip()[:200] if result.stderr else "no stderr" + _check_fail(f"Claude test prompt failed (exit {result.returncode}): {stderr_preview}") + raise PreflightError( + f"Claude auth test failed. Exit code: {result.returncode}. " + f"stderr: {stderr_preview}" + ) + except FileNotFoundError as error: + _check_fail("'claude' command not found on host") + raise PreflightError( + "'claude' is not installed on the host. Install with: npm install -g @anthropic-ai/claude-code" + ) from error + except subprocess.TimeoutExpired: + _check_warn("Claude test prompt timed out (30s) - proceeding anyway") + + +def validate_docker_available(): + """Check that docker is available and running.""" + console.print("\n[bold]Checking Docker...[/bold]") + + try: + result = subprocess.run( + ["docker", "info"], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + if result.returncode == 0: + _check_pass("Docker is available and running") + else: + _check_fail("Docker is not running or not accessible") + raise PreflightError( + "Docker is not running. Start Docker and try again." + ) + except FileNotFoundError as error: + _check_fail("'docker' command not found") + raise PreflightError("Docker is not installed.") from error + + +def run_all_preflight_checks(repo_url=None): + """Run all preflight checks. Raises PreflightError on first failure.""" + console.print(Panel("[bold]Preflight Checks[/bold]", expand=False)) + + validate_docker_available() + gh_token = validate_gh_token(repo_url=repo_url) + validate_claude_credentials() + validate_claude_auth_works() + + console.print("\n[bold green]All preflight checks passed![/bold green]\n") + return gh_token diff --git a/bin/ci_tool/requirements.txt b/bin/ci_tool/requirements.txt new file mode 100644 index 0000000..cde09fd --- /dev/null +++ b/bin/ci_tool/requirements.txt @@ -0,0 +1,2 @@ +rich +InquirerPy diff --git a/bin/merge_helper_vars.py b/bin/merge_helper_vars.py new file mode 100644 index 0000000..fa4f67d --- /dev/null +++ b/bin/merge_helper_vars.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +"""Merge user-customised variables when updating .helper_bash_functions. + +Reads old and new versions of the file, extracts top-level variable assignments, +and applies these rules: + - Variable only in old file (user-added): preserve it + - Same value in both: keep new (no-op) + - Different value: ask the user which to keep + +Can also set individual variables: + python3 merge_helper_vars.py --set VAR=VALUE + python3 merge_helper_vars.py --set --export VAR=VALUE + +Writes the merged result to the new file path. + +Usage: + python3 merge_helper_vars.py + python3 merge_helper_vars.py --set [--export] VAR=VALUE +""" +import re +import sys + +SKIP_VARS = {"Red", "Green", "Yellow", "Color_Off"} +VAR_PATTERN = re.compile(r'^(export\s+)?([A-Za-z_][A-Za-z0-9_]*)=(.*)') +REFERENCES_OTHER_VAR = re.compile(r'\$\{') +FUNCTION_OR_SECTION = re.compile(r'^[a-zA-Z_]+\(\)|^# [A-Z]') + + +def extract_top_level_vars(filepath): + """Extract VAR=value lines from the top of the file, before functions start. + + Returns (variables, exports) where variables is {name: value} and + exports is the set of variable names that had an 'export' prefix. + """ + variables = {} + exports = set() + with open(filepath, encoding="utf-8") as file_handle: + for line in file_handle: + stripped = line.rstrip('\n') + if FUNCTION_OR_SECTION.match(stripped): + break + match = VAR_PATTERN.match(stripped) + if not match: + continue + is_export = bool(match.group(1)) + var_name = match.group(2) + if var_name in SKIP_VARS: + continue + if REFERENCES_OTHER_VAR.search(match.group(3)): + continue + variables[var_name] = match.group(3) + if is_export: + exports.add(var_name) + return variables, exports + + +def ask_user(var_name, old_value, new_value): + """Ask user which value to keep. Returns the chosen value.""" + print(f"\n\033[0;33m{var_name} has changed:\033[0m") + print(f" Current: {old_value}") + print(f" Updated: {new_value}") + response = input(" Keep current value? [Y/n] ").strip().lower() + if response in ("n", "no"): + print(" \033[0;32mUsing updated value\033[0m") + return new_value + print(" \033[0;32mKeeping current value\033[0m") + return old_value + + +def apply_var_to_file(filepath, var_name, value, export=False): + """Set a variable in the file, replacing if present or inserting after Color_Off.""" + with open(filepath, encoding="utf-8") as file_handle: + lines = file_handle.readlines() + + replaced = False + for i, line in enumerate(lines): + if line.startswith(f"export {var_name}=") or line.startswith(f"{var_name}="): + had_export = line.startswith("export ") + use_export = had_export or export + prefix = "export " if use_export else "" + lines[i] = f"{prefix}{var_name}={value}\n" + replaced = True + break + + if not replaced: + prefix = "export " if export else "" + new_line = f"{prefix}{var_name}={value}\n" + for i, line in enumerate(lines): + if line.startswith("Color_Off="): + lines.insert(i + 1, new_line) + break + + with open(filepath, "w", encoding="utf-8") as file_handle: + file_handle.writelines(lines) + + +def set_variable(args): + """Handle --set mode: set a single variable in a file.""" + use_export = False + remaining = list(args) + + if "--export" in remaining: + use_export = True + remaining.remove("--export") + + if len(remaining) != 2: + print(f"Usage: {sys.argv[0]} --set [--export] VAR=VALUE ") + sys.exit(1) + + assignment, filepath = remaining + match = re.match(r'^([A-Za-z_][A-Za-z0-9_]*)=(.*)', assignment) + if not match: + print(f"Error: Invalid assignment '{assignment}'. Expected VAR=VALUE.") + sys.exit(1) + + var_name = match.group(1) + value = match.group(2) + apply_var_to_file(filepath, var_name, value, export=use_export) + prefix = "export " if use_export else "" + print(f"\033[0;32mSet {prefix}{var_name} in {filepath}\033[0m") + + +def main(): + if len(sys.argv) >= 2 and sys.argv[1] == "--set": + set_variable(sys.argv[2:]) + return + + if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} ") + print(f" {sys.argv[0]} --set [--export] VAR=VALUE ") + sys.exit(1) + + old_file, new_file = sys.argv[1], sys.argv[2] + old_vars, old_exports = extract_top_level_vars(old_file) + new_vars, _ = extract_top_level_vars(new_file) + + vars_to_apply = {} + + for var_name, old_value in old_vars.items(): + if var_name not in new_vars: + print(f"\033[0;32mPreserving\033[0m {var_name}={old_value} (not in updated script)") + vars_to_apply[var_name] = old_value + elif old_value != new_vars[var_name]: + chosen = ask_user(var_name, old_value, new_vars[var_name]) + if chosen == old_value: + vars_to_apply[var_name] = old_value + # else: same value, nothing to do + + for var_name, value in vars_to_apply.items(): + is_export = var_name in old_exports + apply_var_to_file(new_file, var_name, value, export=is_export) + + +if __name__ == "__main__": + main() diff --git a/bin/setup.sh b/bin/setup.sh new file mode 100644 index 0000000..d32bf9a --- /dev/null +++ b/bin/setup.sh @@ -0,0 +1,171 @@ +#!/bin/bash +# Setup script for ci_tool and helper bash functions. +# +# Run with: +# bash <(curl -fsSL https://raw.githubusercontent.com/Extend-Robotics/er_build_tools/refs/heads/main/bin/setup.sh) + +set -euo pipefail + +Red='\033[0;31m' +Green='\033[0;32m' +Yellow='\033[0;33m' +Cyan='\033[0;36m' +Bold='\033[1m' +Color_Off='\033[0m' + +BRANCH="main" +BASE_URL="https://raw.githubusercontent.com/Extend-Robotics/er_build_tools/refs/heads/${BRANCH}" +HELPER_URL="${BASE_URL}/.helper_bash_functions" +MERGE_VARS_URL="${BASE_URL}/bin/merge_helper_vars.py" +HELPER_PATH="${HOME}/.helper_bash_functions" + +echo -e "${Bold}${Cyan}" +echo "╔══════════════════════════════════════════════╗" +echo "║ ci_tool Setup — Extend Robotics ║" +echo "╚══════════════════════════════════════════════╝" +echo -e "${Color_Off}" + +# --- Step 1: Install/update .helper_bash_functions --- + +echo -e "${Bold}[1/5] Installing helper bash functions...${Color_Off}" +if [ -f "${HELPER_PATH}" ]; then + echo -e "${Yellow} Existing ~/.helper_bash_functions found — updating while preserving your variables...${Color_Off}" + tmp_new=$(mktemp) + curl -fsSL "${HELPER_URL}" -o "${tmp_new}" + merge_script=$(curl -fsSL "${MERGE_VARS_URL}") + python3 <(echo "${merge_script}") "${HELPER_PATH}" "${tmp_new}" + cp "${tmp_new}" "${HELPER_PATH}" + rm -f "${tmp_new}" + echo -e "${Green} Updated ~/.helper_bash_functions (custom variables preserved).${Color_Off}" +else + curl -fsSL "${HELPER_URL}" -o "${HELPER_PATH}" + echo -e "${Green} Installed ~/.helper_bash_functions${Color_Off}" +fi + +# --- Step 2: GitHub token --- + +echo "" +echo -e "${Bold}[2/5] GitHub token${Color_Off}" +echo -e " ci_tool needs a GitHub token with ${Bold}repo${Color_Off} scope to access private repos." +echo -e " Create one at: ${Cyan}https://github.com/settings/tokens${Color_Off}" +echo "" + +current_token="" +if [ -n "${GH_TOKEN:-}" ]; then + current_token="${GH_TOKEN}" + echo -e " ${Green}GH_TOKEN is already set in your environment.${Color_Off}" + echo -n " Keep current token? [Y/n] " + read -r keep_token + if [[ "${keep_token}" =~ ^[nN] ]]; then + current_token="" + fi +fi + +if [ -z "${current_token}" ]; then + echo -n " Enter your GitHub token (ghp_...): " + read -r current_token + if [ -z "${current_token}" ]; then + echo -e " ${Yellow}Skipped. Set it later by editing ~/.helper_bash_functions${Color_Off}" + fi +fi + +if [ -n "${current_token}" ]; then + merge_script=$(curl -fsSL "${MERGE_VARS_URL}") + python3 <(echo "${merge_script}") --set --export "GH_TOKEN=\"${current_token}\"" "${HELPER_PATH}" +fi + +# --- Step 3: Shell integration --- + +echo "" +echo -e "${Bold}[3/5] Shell integration${Color_Off}" +BASHRC="${HOME}/.bashrc" +if [ -f "${BASHRC}" ] && grep -q 'source ~/.helper_bash_functions' "${BASHRC}"; then + echo -e " ${Green}Already sourced in ~/.bashrc${Color_Off}" +else + echo 'source ~/.helper_bash_functions' >> "${BASHRC}" + echo -e " ${Green}Added 'source ~/.helper_bash_functions' to ~/.bashrc${Color_Off}" +fi + +# --- Step 4: Claude Code authentication --- + +echo "" +echo -e "${Bold}[4/5] Claude Code authentication${Color_Off}" +echo -e " ci_tool uses Claude Code to autonomously fix CI failures." +echo -e " Claude must be installed and authenticated on your host machine." +echo "" + +if command -v claude &> /dev/null; then + echo -e " ${Green}Claude Code is installed.${Color_Off}" + if claude -p "say ok" --max-turns 1 &> /dev/null; then + echo -e " ${Green}Claude authentication is working.${Color_Off}" + else + echo -e " ${Yellow}Claude is installed but not authenticated.${Color_Off}" + echo -e " Run ${Bold}claude${Color_Off} in another terminal to authenticate, then press Enter." + echo -n " Press Enter when done (or 's' to skip): " + read -r auth_response + if [[ ! "${auth_response}" =~ ^[sS] ]]; then + if claude -p "say ok" --max-turns 1 &> /dev/null; then + echo -e " ${Green}Claude authentication verified!${Color_Off}" + else + echo -e " ${Yellow}Still not working — you can fix this later.${Color_Off}" + fi + fi + fi +else + echo -e " ${Yellow}Claude Code is not installed.${Color_Off}" + echo -e " Install with: ${Bold}npm install -g @anthropic-ai/claude-code${Color_Off}" + echo -e " Then run ${Bold}claude${Color_Off} to authenticate." +fi + +# --- Step 5: Install ci_tool --- + +echo "" +echo -e "${Bold}[5/5] Installing ci_tool...${Color_Off}" + +CI_TOOL_DIR="${HOME}/.ci_tool" +CI_TOOL_URL="${BASE_URL}/bin/ci_tool" + +mkdir -p "${CI_TOOL_DIR}/ci_tool/ci_context" + +CI_TOOL_FILES=( + "__init__.py" + "__main__.py" + "cli.py" + "ci_fix.py" + "ci_reproduce.py" + "claude_setup.py" + "claude_session.py" + "containers.py" + "preflight.py" + "display_progress.py" + "requirements.txt" +) + +for file in "${CI_TOOL_FILES[@]}"; do + curl -fsSL "${CI_TOOL_URL}/${file}" -o "${CI_TOOL_DIR}/ci_tool/${file}" || { + echo -e " ${Red}Failed to download ${file}${Color_Off}" + exit 1 + } +done + +curl -fsSL "${CI_TOOL_URL}/ci_context/CLAUDE.md" \ + -o "${CI_TOOL_DIR}/ci_tool/ci_context/CLAUDE.md" 2>/dev/null || true + +pip3 install --user --quiet -r "${CI_TOOL_DIR}/ci_tool/requirements.txt" 2>/dev/null || { + echo -e " ${Yellow}Some dependencies may not have installed. ci_tool will retry on first run.${Color_Off}" +} + +echo -e " ${Green}ci_tool installed at ${CI_TOOL_DIR}${Color_Off}" + +# --- Done --- + +echo "" +echo -e "${Bold}${Green}Setup complete!${Color_Off}" +echo "" + +# Source helper functions so GH_TOKEN is available for ci_tool +source "${HELPER_PATH}" 2>/dev/null || true + +echo -e " ${Bold}${Cyan}Launching ci_tool...${Color_Off}" +echo "" +exec python3 "${CI_TOOL_DIR}/ci_tool/__main__.py" diff --git a/docs/plans/2026-02-18-ci-tool-rearchitect-design.md b/docs/plans/2026-02-18-ci-tool-rearchitect-design.md new file mode 100644 index 0000000..66dea63 --- /dev/null +++ b/docs/plans/2026-02-18-ci-tool-rearchitect-design.md @@ -0,0 +1,180 @@ +# CI Tool Rearchitect Design + +## Problem + +When the CI tool is run without a GitHub Actions URL, it fails to ask for a repo URL and branch, then crashes through a cascade of "No such container" errors. This is symptomatic of deeper architectural issues: + +1. **Prompting scattered across modules** — `select_or_create_session` (ci_fix.py), `prompt_for_reproduce_args` (ci_reproduce.py), and `fix_ci` all collect user input at different stages. +2. **No fail-fast after reproduction failure** — The tool continues to container setup even when `reproduce_ci.sh` fails and no container exists. +3. **Absurd fetch chain** — Python curls a public bash wrapper, which curls 3 private scripts, which run bash that does what Python could do directly. +4. **Side-effect mutation** — `select_or_create_session` mutates `parsed["reproduce_args"]` rather than returning clean data. +5. **Dead code and redundant abstractions** — `rename_container`, `parse_fix_args`, `extract_*_from_args` helpers. +6. **Container collision handled in 3 places** — `prompt_for_session_name`, `reproduce_ci`, and `fix_ci` all handle existing containers differently. +7. **Preflight skips repo validation** — When no CI URL is provided, `repo_url` is None and the token's repo-access check is silently skipped. + +## Design + +### Entry Points + +``` +setup.sh (bash, one-time setup + hand-off) + Install helpers → configure GH_TOKEN → install ci_tool → exec ci_tool + +ci_tool / ci_fix (Python, primary interface) + All functionality: reproduce, fix with Claude, shell, clean, retest + +reproduce_ci.sh (bash, backward-compatible standalone) + Standalone CI reproduction without Python. No changes. +``` + +### Core Flow: `ci_tool fix` + +``` +gather_session_info() <- All prompts happen here + |- Existing containers? -> Resume menu + '- New session: + |- CI URL? (optional) + |- Repo URL + Branch (if no CI URL) + |- Only needed deps? + '- Session name + +run_all_preflight_checks() <- Always validates repo_url + +reproduce_ci() <- Python does Docker orchestration directly + |- Fetch ci_workspace_setup.sh + ci_repull_and_retest.sh via urllib + |- Validate deps.repos reachable + |- docker create (env vars, volume mounts, graphical forwarding) + |- docker start + |- docker exec ci_workspace_setup.sh + '- Guard: raise if container doesn't exist + +setup_claude_in_container() <- Existing, no major changes + +run_claude_workflow() <- Analysis -> Review -> Fix (or custom/resume) + +drop_to_shell() <- Interactive container shell +``` + +### Data Structures + +`gather_session_info` returns a dict: + +```python +# New session: +{ + "mode": "new", + "container_name": "er_ci_my_branch", + "repo_url": "https://github.com/Extend-Robotics/er_interface", + "branch": "my-branch", + "only_needed_deps": True, + "ci_run_info": {...} or None, +} + +# Resume existing container: +{ + "mode": "resume", + "container_name": "er_ci_existing", + "resume_session_id": "abc123" or None, +} +``` + +### Module Responsibilities + +| Module | Responsibility | +|--------|---------------| +| `ci_fix.py` | `gather_session_info()`, `fix_ci()` linear orchestration, Claude prompts/templates | +| `ci_reproduce.py` | Docker orchestration (create/start/exec), fetch container-side scripts, validate deps.repos | +| `preflight.py` | Validate Docker, GH token (with repo access — always), Claude credentials | +| `claude_setup.py` | Install/configure Claude in container (unchanged) | +| `containers.py` | Low-level Docker helpers (exists, running, exec, cp, remove, list) | +| `cli.py` | Menu dispatcher (minor adapter for `_handle_reproduce`) | +| `display_progress.py` | Stream-json display processor (unchanged, runs in container) | +| `claude_session.py` | Interactive Claude session launcher (unchanged) | + +### `reproduce_ci` New Interface + +```python +def reproduce_ci( + repo_url: str, + branch: str, + container_name: str, + gh_token: str, + only_needed_deps: bool = True, + scripts_branch: str = "main", + graphical: bool = True, +): + """Create a CI reproduction container. + + Fetches container-side scripts from er_build_tools_internal, + creates Docker container with proper env/volumes, runs workspace setup. + + Raises RuntimeError if container doesn't exist after execution. + """ +``` + +Explicit parameters instead of a string arg list. No interactive prompts. +The CLI `reproduce` subcommand uses a thin adapter that parses CLI args or +calls `prompt_for_reproduce_args()` before calling this function. + +### Python Docker Orchestration (replaces bash wrapper) + +Currently Python shells out to a bash wrapper that shells out to another +bash script. The new `reproduce_ci` does the Docker orchestration directly: + +1. **Fetch container-side scripts** via `urllib` with GH token auth header + from `er_build_tools_internal` (configurable branch). +2. **Write to `/tmp/er_reproduce_ci/`** — same location the bash wrapper uses. +3. **Validate deps.repos** is reachable (curl-equivalent HTTP HEAD check). +4. **Build `docker create` args** — env vars (GH_TOKEN, REPO_URL, BRANCH, etc.), + volume mounts (scripts as read-only), network/IPC host, optional graphical + forwarding (X11, NVIDIA). +5. **`docker create`** + **`docker start`** + **`docker exec bash /tmp/ci_workspace_setup.sh`** + via `subprocess.run`. +6. **Container guard** — verify `container_exists()` after execution, raise if not. + +### Rich UI Throughout + +All existing UI stays: +- **InquirerPy** for interactive prompts (select menus, text inputs, confirmations). +- **Rich** for colored console output, panels, status messages. +- **display_progress.py** for Claude stream-json spinner/activity display. + +The reproduce step gets improved UI by moving from plain bash output to rich: +- Spinner with elapsed time during long-running docker exec +- Checkmark confirmations for each setup step +- Colored error messages on failure + +### Preflight Changes + +`run_all_preflight_checks` always requires `repo_url`: +- `gather_session_info` always provides a repo URL (from CI URL extraction or direct prompt) +- For resume mode, preflight is skipped (container already exists) +- The GH token repo-access check always runs for new sessions + +### setup.sh Changes + +Add ci_tool Python package installation and hand-off: +- After existing setup steps (helpers, GH token, shell integration, Claude check) +- `pip install` ci_tool from er_build_tools +- `exec python3 -m ci_tool` to hand off to the Python tool + +### Files Changed + +| File | Change | Repo | +|------|--------|------| +| `ci_fix.py` | Major refactor: `gather_session_info`, linear `fix_ci` flow | er_build_tools | +| `ci_reproduce.py` | Rewrite: Python Docker orchestration, explicit params | er_build_tools | +| `preflight.py` | `repo_url` always required for new sessions | er_build_tools | +| `containers.py` | Remove `rename_container` (dead code) | er_build_tools | +| `cli.py` | Adapt `_handle_reproduce` for new `reproduce_ci` interface | er_build_tools | +| `setup.sh` | Add ci_tool install + hand-off to Python | er_build_tools | + +### Files Unchanged + +| File | Reason | +|------|--------| +| `claude_setup.py` | Works as-is | +| `display_progress.py` | Runs in container, works as-is | +| `claude_session.py` | Works as-is | +| `reproduce_ci.sh` (public wrapper) | Kept for backward compat | +| All `er_build_tools_internal` scripts | No changes needed | diff --git a/docs/plans/2026-02-18-ci-tool-rearchitect-plan.md b/docs/plans/2026-02-18-ci-tool-rearchitect-plan.md new file mode 100644 index 0000000..a425024 --- /dev/null +++ b/docs/plans/2026-02-18-ci-tool-rearchitect-plan.md @@ -0,0 +1,1157 @@ +# CI Tool Rearchitect Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Fix the missing repo/branch prompt bug and rearchitect ci_tool so prompting is consolidated, Docker orchestration is done in Python (not bash), and the tool fails fast on errors. + +**Architecture:** All user prompts happen up-front in `gather_session_info()`. `reproduce_ci()` takes explicit params and does Docker orchestration directly in Python (bypassing the bash wrapper chain). A container existence guard prevents cascading failures. + +**Tech Stack:** Python 3.8+, InquirerPy, Rich, Docker CLI via subprocess, urllib for GitHub API + +--- + +### Task 1: Remove dead code from containers.py + +**Files:** +- Modify: `bin/ci_tool/containers.py:80-83` (remove `rename_container`) + +**Step 1: Remove `rename_container` function** + +Delete the `rename_container` function at line 80-83 of `containers.py`: + +```python +# DELETE this entire function: +def rename_container(old_name, new_name): + """Rename a Docker container.""" + run_command(["docker", "rename", old_name, new_name]) +``` + +**Step 2: Verify no references remain** + +Run: `cd /cortex/er_build_tools && grep -r "rename_container" bin/ci_tool/` +Expected: No matches + +**Step 3: Lint** + +Run: `source ~/.helper_bash_functions && cd /cortex/.catkin_ws/src/er_build_tools/bin/ci_tool && python3 -m pylint containers.py --disable=all --enable=E` +Expected: No errors (warnings OK if pre-existing) + +**Step 4: Commit** + +```bash +git add bin/ci_tool/containers.py +git commit -m "remove unused rename_container from containers.py" +``` + +--- + +### Task 2: Rewrite ci_reproduce.py — Python Docker orchestration + +**Files:** +- Modify: `bin/ci_tool/ci_reproduce.py` (full rewrite) + +**Step 1: Write the new ci_reproduce.py** + +Replace the entire file with: + +```python +#!/usr/bin/env python3 +"""Reproduce CI locally by creating a Docker container.""" +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +from rich.console import Console +from rich.panel import Panel + +from ci_tool.containers import container_exists + +console = Console() + +DEFAULT_SCRIPTS_BRANCH = "main" +SCRIPTS_CACHE_DIR = "/tmp/er_reproduce_ci" +INTERNAL_REPO = "Extend-Robotics/er_build_tools_internal" +DEFAULT_DOCKER_IMAGE = ( + "rostooling/setup-ros-docker:ubuntu-focal-ros-noetic-desktop-latest" +) + +CONTAINER_SIDE_SCRIPTS = [ + "ci_workspace_setup.sh", + "ci_repull_and_retest.sh", +] + + +def fetch_internal_script(script_name, gh_token, scripts_branch): + """Fetch a script from er_build_tools_internal and save to cache dir.""" + url = ( + f"https://raw.githubusercontent.com/{INTERNAL_REPO}" + f"/refs/heads/{scripts_branch}/bin/{script_name}" + ) + request = Request(url, headers={"Authorization": f"token {gh_token}"}) + try: + with urlopen(request, timeout=30) as response: + content = response.read() + except HTTPError as error: + raise RuntimeError( + f"Failed to fetch {script_name} from {INTERNAL_REPO} " + f"(branch: {scripts_branch}): HTTP {error.code}" + ) from error + + cache_dir = Path(SCRIPTS_CACHE_DIR) + cache_dir.mkdir(parents=True, exist_ok=True) + script_path = cache_dir / script_name + script_path.write_bytes(content) + script_path.chmod(0o755) + return str(script_path) + + +def fetch_container_side_scripts(gh_token, scripts_branch): + """Fetch all container-side scripts from er_build_tools_internal.""" + console.print( + f"[cyan]Fetching CI scripts from {INTERNAL_REPO} " + f"(branch: {scripts_branch})...[/cyan]" + ) + script_paths = {} + for script_name in CONTAINER_SIDE_SCRIPTS: + path = fetch_internal_script(script_name, gh_token, scripts_branch) + console.print(f" [green]\u2713[/green] {script_name}") + script_paths[script_name] = path + return script_paths + + +def validate_deps_repos_reachable( + repo_url, branch, gh_token, deps_file="deps.repos" +): + """Validate that deps.repos is reachable at the given branch.""" + repo_path = parse_repo_path(repo_url) + branch_for_raw = branch or "main" + deps_url = ( + f"https://raw.githubusercontent.com/{repo_path}" + f"/{branch_for_raw}/{deps_file}" + ) + console.print(f"[cyan]Validating {deps_file} is reachable...[/cyan]") + request = Request( + deps_url, method="HEAD", headers={"Authorization": f"token {gh_token}"} + ) + try: + with urlopen(request, timeout=10): + pass + except HTTPError as error: + raise RuntimeError( + f"Could not reach {deps_file} at {deps_url} (HTTP {error.code}). " + f"Check that branch '{branch_for_raw}' exists and " + f"{deps_file} is present." + ) from error + console.print( + f" [green]\u2713[/green] {deps_file} reachable " + f"at branch {branch_for_raw}" + ) + + +def parse_repo_path(repo_url): + """Extract 'org/repo' from a GitHub URL.""" + repo_url_clean = repo_url.rstrip("/").removesuffix(".git") + return repo_url_clean.split("github.com/")[1] + + +def parse_repo_parts(repo_url): + """Extract org, repo_name, and cleaned URL from a GitHub URL.""" + repo_url_clean = repo_url.rstrip("/").removesuffix(".git") + repo_path = repo_url_clean.split("github.com/")[1] + org, repo_name = repo_path.split("/", 1) + return org, repo_name, repo_url_clean + + +def build_docker_create_command( + container_name, + script_paths, + gh_token, + repo_url_clean, + repo_name, + org, + branch, + only_needed_deps, + graphical, +): + """Build the full docker create command with all args.""" + docker_args = [ + "docker", "create", + "--name", container_name, + "--network=host", + "--ipc=host", + "-v", f"{script_paths['ci_workspace_setup.sh']}:/tmp/ci_workspace_setup.sh:ro", + "-v", f"{script_paths['ci_repull_and_retest.sh']}:/tmp/ci_repull_and_retest.sh:ro", + "-e", f"GH_TOKEN={gh_token}", + "-e", f"REPO_URL={repo_url_clean}", + "-e", f"REPO_NAME={repo_name}", + "-e", f"ORG={org}", + "-e", "DEPS_FILE=deps.repos", + "-e", f"BRANCH={branch}", + "-e", f"ONLY_NEEDED_DEPS={'true' if only_needed_deps else 'false'}", + "-e", "SKIP_TESTS=false", + "-e", "ADDITIONAL_COMMAND=", + ] + + if graphical: + display = os.environ.get("DISPLAY", "") + if display: + console.print("[cyan]Enabling graphical forwarding...[/cyan]") + subprocess.run( + ["xhost", "+local:"], check=False, capture_output=True + ) + docker_args.extend([ + "--runtime", "nvidia", + "--gpus", "all", + "--privileged", + "--security-opt", "seccomp=unconfined", + "-v", "/tmp/.X11-unix:/tmp/.X11-unix:rw", + "-e", f"DISPLAY={display}", + "-e", "QT_X11_NO_MITSHM=1", + "-e", "NVIDIA_DRIVER_CAPABILITIES=all", + "-e", "NVIDIA_VISIBLE_DEVICES=all", + ]) + + docker_args.extend([DEFAULT_DOCKER_IMAGE, "sleep", "infinity"]) + return docker_args + + +def reproduce_ci( + repo_url, + branch, + container_name, + gh_token, + only_needed_deps=True, + scripts_branch=DEFAULT_SCRIPTS_BRANCH, + graphical=True, +): + """Create a CI reproduction container. + + Fetches container-side scripts from er_build_tools_internal, + creates Docker container with proper env/volumes, runs workspace setup. + + Raises RuntimeError if container doesn't exist after execution. + """ + console.print(Panel("[bold]Reproducing CI Locally[/bold]", expand=False)) + + script_paths = fetch_container_side_scripts(gh_token, scripts_branch) + validate_deps_repos_reachable(repo_url, branch, gh_token) + + org, repo_name, repo_url_clean = parse_repo_parts(repo_url) + console.print(f" Organization: {org}") + console.print(f" Repository: {repo_name}") + + create_command = build_docker_create_command( + container_name, script_paths, gh_token, repo_url_clean, + repo_name, org, branch, only_needed_deps, graphical, + ) + + console.print(f"\n[cyan]Creating container '{container_name}'...[/cyan]") + subprocess.run(create_command, check=True) + console.print(f" [green]\u2713[/green] Container created") + + subprocess.run(["docker", "start", container_name], check=True) + console.print(f" [green]\u2713[/green] Container started") + + console.print("\n[cyan]Running CI workspace setup...[/cyan]") + workspace_setup_exit_code = 0 + try: + result = subprocess.run( + ["docker", "exec", container_name, + "bash", "/tmp/ci_workspace_setup.sh"], + check=False, + ) + workspace_setup_exit_code = result.returncode + except KeyboardInterrupt: + console.print( + "\n[yellow]Interrupted \u2014 continuing with whatever test " + "output was captured[/yellow]" + ) + + if workspace_setup_exit_code != 0: + console.print( + f"\n[yellow]CI workspace setup exited with code " + f"{workspace_setup_exit_code} " + f"(expected \u2014 tests likely failed)[/yellow]" + ) + + if not container_exists(container_name): + raise RuntimeError( + f"Container '{container_name}' was not created. " + "Check the output above for errors." + ) + console.print( + f"\n[green]\u2713 Container '{container_name}' is ready[/green]" + ) + + +def prompt_for_reproduce_args(): + """Interactively ask user for reproduce arguments. + + Used by the CLI 'reproduce' subcommand only. + Returns (repo_url, branch, only_needed_deps). + """ + from InquirerPy import inquirer + + repo_url = inquirer.text( + message="Repository URL:", + validate=lambda url: url.startswith("https://github.com/"), + invalid_message="Must be a GitHub URL (https://github.com/...)", + ).execute() + + branch = inquirer.text( + message="Branch name:", + validate=lambda b: len(b.strip()) > 0, + invalid_message="Branch name cannot be empty", + ).execute() + + only_needed_deps = not inquirer.confirm( + message="Build everything (slower, disable --only-needed-deps)?", + default=False, + ).execute() + + return repo_url, branch, only_needed_deps +``` + +**Step 2: Lint** + +Run: `source ~/.helper_bash_functions && cd /cortex/.catkin_ws/src/er_build_tools/bin/ci_tool && python3 -m pylint ci_reproduce.py --disable=all --enable=E` +Expected: No errors + +**Step 3: Commit** + +```bash +git add bin/ci_tool/ci_reproduce.py +git commit -m "rewrite ci_reproduce.py with Python Docker orchestration + +Replaces the bash wrapper fetch chain with direct Python Docker +orchestration. reproduce_ci() now takes explicit params, fetches +container-side scripts via urllib, and raises on failure." +``` + +--- + +### Task 3: Refactor ci_fix.py — consolidated prompting and linear flow + +**Files:** +- Modify: `bin/ci_tool/ci_fix.py` (major refactor) + +**Step 1: Write the new ci_fix.py** + +Replace the entire file. Key changes: +- New `gather_session_info()` consolidates all up-front prompts +- `fix_ci()` is a linear sequence with no scattered prompts +- Removed: `parse_fix_args`, `select_or_create_session`, the args-based path +- Uses new `reproduce_ci` interface with explicit params + +```python +#!/usr/bin/env python3 +"""Fix CI test failures using Claude Code inside a container.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +from urllib.request import Request, urlopen + +from InquirerPy import inquirer +from rich.console import Console +from rich.panel import Panel + +from ci_tool.claude_setup import ( + copy_ci_context, + copy_claude_credentials, + copy_display_script, + inject_rerun_tests_function, + inject_resume_function, + is_claude_installed_in_container, + save_package_list, + setup_claude_in_container, +) +from ci_tool.containers import ( + container_exists, + container_is_running, + docker_exec, + docker_exec_interactive, + list_ci_containers, + remove_container, + sanitize_container_name, + start_container, +) +from ci_tool.ci_reproduce import reproduce_ci +from ci_tool.preflight import run_all_preflight_checks, PreflightError + +console = Console() + +SUMMARY_FORMAT = ( + "When done, print EXACTLY this format:\n\n" + "--- SUMMARY ---\n" + "Problem: \n" + "Fix: \n" + "Assumptions: \n\n" + "--- COMMIT MESSAGE ---\n" + "\n" + "--- END ---" +) + +ROS_SOURCE_PREAMBLE = ( + "You are inside a CI reproduction container at /ros_ws. " + "Source the ROS workspace: " + "`source /opt/ros/noetic/setup.bash && source /ros_ws/install/setup.bash`." + "\n\n" +) + +ANALYSIS_PROMPT_TEMPLATE = ( + ROS_SOURCE_PREAMBLE + + "The CI tests have already been run. Analyse the failures:\n" + "1. Examine the test output in /ros_ws/test_output.log\n" + "2. For each failing test, report:\n" + " - Package and test name\n" + " - The error/assertion message\n" + " - Your hypothesis for the root cause\n" + "3. Suggest a fix strategy for each failure\n\n" + "Do NOT make any code changes. Only analyse and report.\n" + "{extra_context}" +) + +CI_COMPARE_EXTRA_CONTEXT_TEMPLATE = ( + "\nAlso investigate the CI run: {ci_run_url}\n" + "- Verify local and CI are on the same commit:\n" + " - Local: check HEAD in the repo under /ros_ws/src/\n" + " - CI: `gh api repos/{owner_repo}/actions/runs/{run_id}" + " --jq '.head_sha'`\n" + " - If they differ, determine whether the missing/extra commits " + "explain the failure\n" + "- Fetch CI logs: `gh run view {run_id} --log-failed` " + "(use `--log` for full output if needed)\n" + "- Compare CI failures with local test results\n" +) + +FIX_PROMPT_TEMPLATE = ( + "The user has reviewed your analysis. Their feedback:\n" + "{user_feedback}\n\n" + "Now fix the CI failures based on this understanding.\n" + "Rebuild the affected packages and re-run the failing tests to verify.\n" + "Iterate until all tests pass.\n\n" + + SUMMARY_FORMAT +) + +FIX_MODE_CHOICES = [ + {"name": "Fix CI failures (from test_output.log)", "value": "fix_from_log"}, + {"name": "Compare with GitHub Actions CI run", "value": "compare_ci_run"}, + {"name": "Custom prompt", "value": "custom"}, +] + +CLAUDE_STDERR_LOG = "/ros_ws/.claude_stderr.log" + + +def read_container_state(container_name): + """Read the ci_fix state file from a container. Returns dict or None.""" + result = subprocess.run( + ["docker", "exec", container_name, + "cat", "/ros_ws/.ci_fix_state.json"], + capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + return None + try: + return json.loads(result.stdout) + except json.JSONDecodeError: + return None + + +def extract_run_id_from_url(ci_run_url): + """Extract the numeric run ID from a GitHub Actions URL. + + Handles URLs like: + https://github.com/org/repo/actions/runs/12345678901 + https://github.com/org/repo/actions/runs/12345678901/job/98765 + """ + parts = ci_run_url.rstrip("/").split("/runs/") + if len(parts) < 2: + raise ValueError(f"Cannot extract run ID from URL: {ci_run_url}") + run_id = parts[1].split("/")[0] + if not run_id.isdigit(): + raise ValueError(f"Run ID is not numeric: {run_id}") + return run_id + + +def extract_info_from_ci_url(ci_run_url): + """Extract repo URL, branch, and run ID from a GitHub Actions URL.""" + run_id = extract_run_id_from_url(ci_run_url) + + owner_repo = ci_run_url.split("github.com/")[1].split("/actions/")[0] + repo_url = f"https://github.com/{owner_repo}" + + token = os.environ.get("GH_TOKEN") or os.environ.get("ER_SETUP_TOKEN") + if not token: + raise ValueError( + "No GitHub token found (GH_TOKEN or ER_SETUP_TOKEN)" + ) + + api_url = ( + f"https://api.github.com/repos/{owner_repo}/actions/runs/{run_id}" + ) + request = Request(api_url, headers={ + "Authorization": f"token {token}", + "Accept": "application/vnd.github.v3+json", + }) + with urlopen(request) as response: + data = json.loads(response.read()) + + return { + "repo_url": repo_url, + "owner_repo": owner_repo, + "branch": data["head_branch"], + "run_id": run_id, + "ci_run_url": ci_run_url, + } + + +def prompt_for_session_name(branch_hint=None): + """Ask user for a session name. Returns full container name (er_ci_). + + Exits if the container already exists. + """ + default = sanitize_container_name(branch_hint) if branch_hint else "" + name = inquirer.text( + message="Session name (used for container naming):", + default=default, + validate=lambda n: len(n.strip()) > 0, + invalid_message="Session name cannot be empty", + ).execute().strip() + + container_name = f"er_ci_{sanitize_container_name(name)}" + + if container_exists(container_name): + console.print( + f"[red]Container '{container_name}' already exists. " + f"Choose a different name or clean up first.[/red]" + ) + sys.exit(1) + + return container_name + + +def gather_session_info(): + """Collect all session information up front via interactive prompts. + + Returns a dict with 'mode' key: + - mode='new': container_name, repo_url, branch, only_needed_deps, + ci_run_info (or None) + - mode='resume': container_name, resume_session_id (or None) + """ + existing = list_ci_containers() + + if existing: + choices = [{"name": "Start new session", "value": "_new"}] + for container in existing: + choices.append({ + "name": ( + f"Resume '{container['name']}' ({container['status']})" + ), + "value": container["name"], + }) + + selection = inquirer.select( + message="Select a session:", + choices=choices, + ).execute() + + if selection != "_new": + if not container_is_running(selection): + start_container(selection) + + resume_session_id = None + state = read_container_state(selection) + if state and state.get("session_id"): + session_id = state["session_id"] + phase = state.get("phase", "unknown") + attempt = state.get("attempt_count", 0) + console.print( + f" [dim]Previous session: {phase} " + f"(attempt {attempt}, id: {session_id})[/dim]" + ) + + resume_choice = inquirer.select( + message=( + "Resume previous Claude session or start fresh?" + ), + choices=[ + { + "name": f"Resume session ({phase})", + "value": "resume", + }, + { + "name": "Start fresh fix attempt", + "value": "fresh", + }, + ], + ).execute() + + if resume_choice == "resume": + resume_session_id = session_id + + return { + "mode": "resume", + "container_name": selection, + "resume_session_id": resume_session_id, + } + + # New session: collect all info + ci_run_info = None + ci_run_url = inquirer.text( + message="GitHub Actions run URL (leave blank to skip):", + default="", + ).execute().strip() + + if ci_run_url: + ci_run_info = extract_info_from_ci_url(ci_run_url) + repo_url = ci_run_info["repo_url"] + branch = ci_run_info["branch"] + console.print(f" [green]Repo:[/green] {repo_url}") + console.print(f" [green]Branch:[/green] {branch}") + console.print(f" [green]Run ID:[/green] {ci_run_info['run_id']}") + else: + repo_url = inquirer.text( + message="Repository URL:", + validate=lambda url: url.startswith("https://github.com/"), + invalid_message="Must be a GitHub URL (https://github.com/...)", + ).execute() + + branch = inquirer.text( + message="Branch name:", + validate=lambda b: len(b.strip()) > 0, + invalid_message="Branch name cannot be empty", + ).execute() + + only_needed_deps = not inquirer.confirm( + message="Build everything (slower, disable --only-needed-deps)?", + default=False, + ).execute() + + container_name = prompt_for_session_name(branch if branch else None) + + return { + "mode": "new", + "container_name": container_name, + "repo_url": repo_url, + "branch": branch, + "only_needed_deps": only_needed_deps, + "ci_run_info": ci_run_info, + } + + +def select_fix_mode(): + """Let the user choose how Claude should fix CI failures. + + Returns (ci_run_info_or_none, custom_prompt_or_none). + """ + mode = inquirer.select( + message="How should Claude fix CI?", + choices=FIX_MODE_CHOICES, + default="fix_from_log", + ).execute() + + if mode == "fix_from_log": + return None, None + + if mode == "compare_ci_run": + ci_run_url = inquirer.text( + message="GitHub Actions run URL:", + validate=lambda url: "/runs/" in url, + invalid_message=( + "URL must contain /runs/ " + "(e.g. https://github.com/org/repo/actions/runs/12345)" + ), + ).execute() + return extract_info_from_ci_url(ci_run_url), None + + custom_prompt = inquirer.text( + message="Enter your custom prompt for Claude:" + ).execute() + return None, custom_prompt + + +def build_analysis_prompt(ci_run_info): + """Build the analysis prompt, optionally including CI compare context.""" + if ci_run_info: + extra_context = CI_COMPARE_EXTRA_CONTEXT_TEMPLATE.format( + **ci_run_info + ) + else: + extra_context = "" + return ANALYSIS_PROMPT_TEMPLATE.format(extra_context=extra_context) + + +def run_claude_streamed(container_name, prompt): + """Run Claude non-interactively with stream-json output.""" + escaped_prompt = prompt.replace("'", "'\\''") + claude_command = ( + f"cd /ros_ws && IS_SANDBOX=1 claude --dangerously-skip-permissions " + f"-p '{escaped_prompt}' --verbose --output-format stream-json " + f"2>{CLAUDE_STDERR_LOG} | ci_fix_display" + ) + docker_exec(container_name, claude_command, check=False) + + +def run_claude_resumed(container_name, session_id, prompt): + """Resume a Claude session with a new prompt, streaming output.""" + escaped_prompt = prompt.replace("'", "'\\''") + claude_command = ( + f"cd /ros_ws && IS_SANDBOX=1 claude --dangerously-skip-permissions " + f"--resume '{session_id}' -p '{escaped_prompt}' " + f"--verbose --output-format stream-json " + f"2>{CLAUDE_STDERR_LOG} | ci_fix_display" + ) + docker_exec(container_name, claude_command, check=False) + + +def prompt_user_for_feedback(): + """Ask user to review Claude's analysis and provide corrections.""" + feedback = inquirer.text( + message=( + "Review the analysis above. " + "Provide corrections or context (Enter to accept as-is):" + ), + default="", + ).execute().strip() + if not feedback: + return "Analysis looks correct, proceed with fixing." + return feedback + + +def refresh_claude_config(container_name): + """Refresh Claude config in an existing container.""" + console.print( + "[green]Claude already installed \u2014 refreshing config...[/green]" + ) + copy_claude_credentials(container_name) + copy_ci_context(container_name) + copy_display_script(container_name) + inject_resume_function(container_name) + inject_rerun_tests_function(container_name) + + +def run_claude_workflow(container_name, ci_run_info): + """Run the Claude analysis -> feedback -> fix workflow.""" + if ci_run_info: + custom_prompt = None + else: + ci_run_info, custom_prompt = select_fix_mode() + + if custom_prompt: + console.print( + "\n[bold cyan]Launching Claude Code (custom prompt)...[/bold cyan]" + ) + run_claude_streamed(container_name, custom_prompt) + else: + # Analysis phase + analysis_prompt = build_analysis_prompt(ci_run_info) + console.print( + "\n[bold cyan]Launching Claude Code " + "\u2014 analysis phase...[/bold cyan]" + ) + console.print( + "[dim]Claude will analyse failures before " + "attempting fixes[/dim]\n" + ) + run_claude_streamed(container_name, analysis_prompt) + + # User review + console.print() + user_feedback = prompt_user_for_feedback() + + # Fix phase (resume session) + state = read_container_state(container_name) + session_id = state["session_id"] if state else None + if session_id: + console.print( + "\n[bold cyan]Resuming Claude " + "\u2014 fix phase...[/bold cyan]" + ) + console.print( + "[dim]Claude will now fix the failures[/dim]\n" + ) + fix_prompt = FIX_PROMPT_TEMPLATE.format( + user_feedback=user_feedback + ) + run_claude_resumed(container_name, session_id, fix_prompt) + else: + console.print( + "\n[yellow]No session ID from analysis phase \u2014 " + "cannot resume. Dropping to shell.[/yellow]" + ) + + # Show outcome + state = read_container_state(container_name) + if state: + phase = state.get("phase", "unknown") + session_id = state.get("session_id") + attempt = state.get("attempt_count", 1) + console.print( + f"\n[bold]Claude finished \u2014 " + f"phase: {phase}, attempt: {attempt}[/bold]" + ) + if session_id: + console.print(f"[dim]Session ID: {session_id}[/dim]") + else: + console.print( + "\n[yellow]Could not read state file from container[/yellow]" + ) + + +def drop_to_shell(container_name): + """Drop user into an interactive container shell.""" + console.print("\n[bold green]Dropping into container shell.[/bold green]") + console.print("[cyan]Useful commands:[/cyan]") + console.print( + " [bold]rerun_tests[/bold] " + "\u2014 rebuild and re-run CI tests locally" + ) + console.print( + " [bold]resume_claude[/bold] " + "\u2014 resume the Claude session interactively" + ) + console.print(" [bold]git diff[/bold] \u2014 review changes") + console.print( + " [bold]git add && git commit[/bold] \u2014 commit fixes" + ) + console.print(" [dim]Repo is at /ros_ws/src/[/dim]\n") + docker_exec_interactive(container_name) + + +def fix_ci(args): + """Main fix workflow: gather -> preflight -> reproduce -> Claude -> shell. + + Args are accepted for backward compat but ignored (interactive only). + """ + console.print( + Panel("[bold cyan]CI Fix with Claude[/bold cyan]", expand=False) + ) + + # Step 1: Gather all session info up front + session = gather_session_info() + container_name = session["container_name"] + + if session["mode"] == "new": + # Step 2: Preflight checks + try: + gh_token = run_all_preflight_checks( + repo_url=session["repo_url"] + ) + except PreflightError as error: + console.print( + f"\n[bold red]Preflight failed:[/bold red] {error}" + ) + sys.exit(1) + + # Step 3: Reproduce CI in container + if container_exists(container_name): + remove_container(container_name) + reproduce_ci( + repo_url=session["repo_url"], + branch=session["branch"], + container_name=container_name, + gh_token=gh_token, + only_needed_deps=session["only_needed_deps"], + ) + save_package_list(container_name) + + # Step 4: Setup Claude in container + if is_claude_installed_in_container(container_name): + refresh_claude_config(container_name) + else: + setup_claude_in_container(container_name) + + # Step 5: Run Claude + resume_session_id = session.get("resume_session_id") + if resume_session_id: + console.print( + "\n[bold cyan]Resuming Claude session...[/bold cyan]" + ) + console.print( + "[dim]You are now in an interactive Claude session[/dim]\n" + ) + docker_exec( + container_name, + "cd /ros_ws && IS_SANDBOX=1 claude " + "--dangerously-skip-permissions " + f'--resume "{resume_session_id}"', + interactive=True, check=False, + ) + else: + run_claude_workflow( + container_name, session.get("ci_run_info") + ) + + # Step 6: Drop to shell + drop_to_shell(container_name) +``` + +**Step 2: Lint** + +Run: `source ~/.helper_bash_functions && cd /cortex/.catkin_ws/src/er_build_tools/bin/ci_tool && python3 -m pylint ci_fix.py --disable=all --enable=E` +Expected: No errors + +**Step 3: Commit** + +```bash +git add bin/ci_tool/ci_fix.py +git commit -m "refactor ci_fix.py: consolidated prompting and linear flow + +gather_session_info() collects all user input up front. fix_ci() is +now a linear sequence: gather -> preflight -> reproduce -> claude -> shell. +Fixes the missing repo/branch prompt when CI URL is left blank." +``` + +--- + +### Task 4: Adapt cli.py for new reproduce_ci interface + +**Files:** +- Modify: `bin/ci_tool/cli.py` + +**Step 1: Update `_handle_reproduce`** + +The `reproduce` CLI subcommand needs a thin adapter since `reproduce_ci` now takes explicit params. Replace `_handle_reproduce` (and add necessary imports): + +In `cli.py`, replace the `_handle_reproduce` function (lines 62-64): + +```python +def _handle_reproduce(args): + import os + from ci_tool.ci_reproduce import ( + reproduce_ci, + prompt_for_reproduce_args, + DEFAULT_SCRIPTS_BRANCH, + ) + from ci_tool.containers import ( + DEFAULT_CONTAINER_NAME, + container_exists, + container_is_running, + remove_container, + ) + from ci_tool.preflight import ( + validate_docker_available, + validate_gh_token, + PreflightError, + ) + + try: + validate_docker_available() + except PreflightError as error: + console.print(f"\n[bold red]Preflight failed:[/bold red] {error}") + sys.exit(1) + + repo_url, branch, only_needed_deps = prompt_for_reproduce_args() + container_name = DEFAULT_CONTAINER_NAME + + if container_exists(container_name): + from InquirerPy import inquirer + action = inquirer.select( + message=f"Container '{container_name}' already exists. What to do?", + choices=[ + {"name": "Remove and recreate", "value": "recreate"}, + {"name": "Keep existing (skip creation)", "value": "keep"}, + {"name": "Cancel", "value": "cancel"}, + ], + ).execute() + + if action == "cancel": + return + if action == "recreate": + remove_container(container_name) + if action == "keep": + if not container_is_running(container_name): + import subprocess + subprocess.run( + ["docker", "start", container_name], check=True + ) + console.print( + f"[green]Using existing container " + f"'{container_name}'[/green]" + ) + return + + try: + gh_token = validate_gh_token(repo_url=repo_url) + except PreflightError as error: + console.print(f"\n[bold red]Preflight failed:[/bold red] {error}") + sys.exit(1) + + reproduce_ci( + repo_url=repo_url, + branch=branch, + container_name=container_name, + gh_token=gh_token, + only_needed_deps=only_needed_deps, + ) +``` + +**Step 2: Lint** + +Run: `source ~/.helper_bash_functions && cd /cortex/.catkin_ws/src/er_build_tools/bin/ci_tool && python3 -m pylint cli.py --disable=all --enable=E` +Expected: No errors + +**Step 3: Commit** + +```bash +git add bin/ci_tool/cli.py +git commit -m "adapt cli.py reproduce handler for new reproduce_ci interface" +``` + +--- + +### Task 5: Update setup.sh — install ci_tool and hand-off + +**Files:** +- Modify: `bin/setup.sh` + +**Step 1: Add ci_tool install step and hand-off** + +After the existing Step 4 (Claude Code authentication) and before the "Done" section, add a new step 5 that installs the ci_tool Python package. Then change the "Done" section to exec into ci_tool. + +After line 118 (`fi` closing the Claude auth block), add: + +```bash +# --- Step 5: Install ci_tool --- + +echo "" +echo -e "${Bold}[5/5] Installing ci_tool...${Color_Off}" + +CI_TOOL_DIR="${HOME}/.ci_tool" +CI_TOOL_URL="${BASE_URL}/bin/ci_tool" + +if [ -d "${CI_TOOL_DIR}/ci_tool" ]; then + echo -e " ${Green}ci_tool already installed at ${CI_TOOL_DIR}${Color_Off}" + echo -e " ${Cyan}Updating...${Color_Off}" +fi + +mkdir -p "${CI_TOOL_DIR}" + +# Download ci_tool package files +CI_TOOL_FILES=( + "__init__.py" + "__main__.py" + "cli.py" + "ci_fix.py" + "ci_reproduce.py" + "claude_setup.py" + "claude_session.py" + "containers.py" + "preflight.py" + "display_progress.py" + "requirements.txt" +) + +mkdir -p "${CI_TOOL_DIR}/ci_tool/ci_context" +for file in "${CI_TOOL_FILES[@]}"; do + curl -fsSL "${CI_TOOL_URL}/${file}" -o "${CI_TOOL_DIR}/ci_tool/${file}" || { + echo -e " ${Red}Failed to download ${file}${Color_Off}" + exit 1 + } +done + +# Download CI context CLAUDE.md +curl -fsSL "${CI_TOOL_URL}/ci_context/CLAUDE.md" \ + -o "${CI_TOOL_DIR}/ci_tool/ci_context/CLAUDE.md" 2>/dev/null || true + +# Install Python dependencies +pip3 install --user --quiet -r "${CI_TOOL_DIR}/ci_tool/requirements.txt" 2>/dev/null || { + echo -e " ${Yellow}Some dependencies may not have installed. ci_tool will retry on first run.${Color_Off}" +} + +echo -e " ${Green}ci_tool installed at ${CI_TOOL_DIR}${Color_Off}" +``` + +Then update the "Done" section to hand off: + +```bash +# --- Done --- + +echo "" +echo -e "${Bold}${Green}Setup complete!${Color_Off}" +echo "" +echo -e " Reload your shell or run:" +echo -e " ${Bold}source ~/.helper_bash_functions${Color_Off}" +echo "" + +# Source helper functions so GH_TOKEN is available for ci_tool +source "${HELPER_PATH}" 2>/dev/null || true + +echo -e " ${Bold}${Cyan}Launching ci_tool...${Color_Off}" +echo "" +exec python3 "${CI_TOOL_DIR}/ci_tool/__main__.py" +``` + +**Step 2: Update step numbering** + +Change step header counts from `[1/4]`, `[2/4]`, `[3/4]`, `[4/4]` to `[1/5]`, `[2/5]`, `[3/5]`, `[4/5]`. + +**Step 3: Commit** + +```bash +git add bin/setup.sh +git commit -m "setup.sh: install ci_tool and hand off after setup" +``` + +--- + +### Task 6: Lint all changed files + +**Files:** +- All modified Python files + +**Step 1: Run linters on all changed files** + +Run: `source ~/.helper_bash_functions && cd /cortex/.catkin_ws/src/er_build_tools/bin/ci_tool && er_python_linters_here` +Expected: No errors or warnings. Fix any that appear. + +**Step 2: Commit lint fixes if any** + +```bash +git add -A bin/ci_tool/ +git commit -m "fix lint issues from rearchitect" +``` + +--- + +### Task 7: Manual integration test + +**Step 1: Test new session without CI URL** + +Run: `python3 -m ci_tool` +Select: "Fix CI with Claude" +Leave CI URL blank. +Expected: Prompted for "Repository URL:" and "Branch name:" (the core bug fix). +Enter valid repo/branch. Verify preflight runs with repo validation, container is created, workspace setup runs. + +**Step 2: Test new session with CI URL** + +Run: `python3 -m ci_tool` +Select: "Fix CI with Claude" +Enter a valid GitHub Actions URL. +Expected: Repo/branch auto-extracted, not prompted again. Preflight validates repo. Full flow works. + +**Step 3: Test resume existing session** + +Run: `python3 -m ci_tool` (with existing container from previous test) +Select existing container from resume menu. +Expected: Container starts, Claude resume/fresh choice shown, flow continues without reproduction step. + +**Step 4: Test standalone reproduce** + +Run: `python3 -m ci_tool` +Select: "Reproduce CI (create container)" +Expected: Prompted for repo, branch, only-needed-deps. Container created via Python Docker orchestration (no bash wrapper chain). + +**Step 5: Test failure guard** + +Intentionally provide a bad repo URL (e.g. `https://github.com/fake/nonexistent`). +Expected: Preflight fails with clear error about repo access. Tool exits cleanly, no cascade of "No such container" errors. diff --git a/docs/plans/2026-02-18-ci-tool-session-handoff.md b/docs/plans/2026-02-18-ci-tool-session-handoff.md new file mode 100644 index 0000000..84ef082 --- /dev/null +++ b/docs/plans/2026-02-18-ci-tool-session-handoff.md @@ -0,0 +1,82 @@ +# CI Tool Rearchitect — Session Handoff + +**Branch:** `ERD-1633_reproduce_ci_locally_tool` in `/cortex/er_build_tools` +**Date:** 2026-02-18 + +## What Was Done + +Full rearchitect of the `bin/ci_tool` Python package: + +1. **Removed dead code** — `rename_container` from containers.py +2. **Rewrote ci_reproduce.py** — Python Docker orchestration replaces bash wrapper chain. `reproduce_ci()` is a pure function (no interactive prompts). Fetches scripts from `er_build_tools_internal` via urllib, creates Docker containers directly. +3. **Refactored ci_fix.py** — `gather_session_info()` consolidates all prompting up front. Fixes the original bug where skipping the CI URL didn't prompt for repo/branch. +4. **Adapted cli.py** — `_handle_container_collision()` extracted. Container collision handling is the caller's responsibility, not `reproduce_ci`'s. +5. **Updated setup.sh** — Installs ci_tool and hands off to Python. +6. **Fixed display_progress.py** — Rewrote event handler to match Claude Code's actual stream-json format (`{"type":"assistant","message":{"content":[...]}}` not Anthropic API format). Added `force_terminal=True`. +7. **Fixed claude_setup.py** — Added `IS_SANDBOX=1` to `resume_claude` function. +8. **All files lint clean** — pylint 10.00/10 across all changed modules. + +## Outstanding Issues to Debug on Test Machine + +### 1. Empty workspace after reproduce (HIGH PRIORITY) + +The `ci_workspace_setup.sh` runs inside the container but the workspace ends up empty (`/ros_ws/src/` has nothing). Need to: + +```bash +# Check env vars were passed correctly to the container +docker exec er_ci_main env | grep -E 'REPO_URL|REPO_NAME|ORG|BRANCH|GH_TOKEN|DEPS_FILE' + +# Check if the setup script is mounted +docker exec er_ci_main ls -la /tmp/ci_workspace_setup.sh + +# Re-run setup manually to see errors +docker exec er_ci_main bash /tmp/ci_workspace_setup.sh +``` + +The `_docker_exec_workspace_setup()` in ci_reproduce.py treats all non-zero exit codes as "expected if tests failed" but doesn't distinguish setup failures from test failures. + +### 2. Display progress — no spinners + +The display now shows text and tool names (format fix worked) but has no animated spinners. Rich's `Live` display was removed because it swallowed all output in docker exec. The `-t` flag is now passed to docker exec. Could try re-adding a spinner now that `-t` is set, or use a simpler periodic timer approach. + +### 3. resume_claude auth (just pushed fix) + +Added `IS_SANDBOX=1` to the `resume_claude` bash function. Without it, Claude shows the login screen instead of resuming. Needs testing. + +### 4. CDN caching + +`ci_tool()` in `.helper_bash_functions` fetches Python files from `raw.githubusercontent.com` which caches aggressively (sometimes minutes). For rapid iteration, either: +- Copy files directly: `cp /cortex/er_build_tools/bin/ci_tool/*.py ~/.ci_tool/ci_tool/` +- Or run locally: `cd /cortex/er_build_tools/bin && python3 -m ci_tool` + +### 5. Test repo + +Use `https://github.com/Extend-Robotics/er_ci_test_fixture` for integration testing (noted in TODO.md). + +## Key Files + +- `bin/ci_tool/ci_reproduce.py` — Docker orchestration, script fetching, prompting +- `bin/ci_tool/ci_fix.py` — Claude workflow, session management, prompting +- `bin/ci_tool/cli.py` — Menu routing, container collision handling +- `bin/ci_tool/containers.py` — Low-level Docker helpers +- `bin/ci_tool/display_progress.py` — Stream-json event display +- `bin/ci_tool/claude_setup.py` — Claude installation and config in containers +- `bin/ci_tool/TODO.md` — Future work items +- `.helper_bash_functions` — Bash wrapper that fetches and runs ci_tool + +## Key Design Decisions + +- `reproduce_ci()` is pure — callers handle container collisions and preflight +- `gather_session_info()` collects ALL user input before any work starts +- `prompt_for_repo_and_branch()` is shared between ci_fix.py and ci_reproduce.py +- `DEFAULT_SCRIPTS_BRANCH` reads `CI_TOOL_SCRIPTS_BRANCH` env var, defaults to `ERD-1633_reproduce_ci_locally` (change to `main` when internal scripts are merged) +- Graphical mode fails fast if DISPLAY not set (CLAUDE.md: no fallback behavior) +- `force_terminal=True` on Rich Console + `docker exec -t` for display + +## CLAUDE.md Rules + +- KISS, YAGNI, SOLID +- Self-documenting variable names, minimal comments +- Fail fast — no fallback behaviour or silent defaults +- Linters must pass (pylint 10.00/10) +- ROS1 Noetic, Python 3.8 compatibility required diff --git a/docs/plans/2026-02-25-ci-analyse-mode-design.md b/docs/plans/2026-02-25-ci-analyse-mode-design.md new file mode 100644 index 0000000..45d89c4 --- /dev/null +++ b/docs/plans/2026-02-25-ci-analyse-mode-design.md @@ -0,0 +1,140 @@ +# CI Analyse Mode Design + +## Goal + +Add an "Analyse CI" mode to ci_tool that diagnoses CI failures. Offers two +sub-modes: a fast remote-only analysis (fetch + filter GH Actions logs, diagnose +with Claude haiku), and a full parallel analysis that also reproduces locally in +Docker. + +## User Flow + +1. User selects **"Analyse CI"** from main menu (or `ci_tool analyse`) +2. Provides a **GitHub Actions URL** (required) +3. Sub-menu: **"Remote only (fast)"** vs **"Remote + local reproduction"** + +### Remote only (fast) + +4. Fetches GH Actions logs, filters with Python regex, sends reduced context + to Claude haiku on host for diagnosis +5. Prints structured report +6. Done — no container, no Docker + +### Remote + local reproduction + +4. Provides build options (only-needed-deps) and session name +5. Two parallel threads start with a Rich Live split-panel display: + - **Top panel ("Remote CI Logs"):** Same as remote-only pipeline above + - **Bottom panel ("Local Reproduction"):** `reproduce_ci()` in Docker, + then Claude analyses local `test_output.log` inside the container +6. Once both complete, prints combined summary +7. Asks: "Proceed to fix with Claude?" — if yes, transitions into existing + `run_claude_workflow()` fix phase (container already set up) + +## Architecture + +### New file: `bin/ci_tool/ci_analyse.py` + +Entry point `analyse_ci(args)`: +- Prompts for GH Actions URL (required) +- Sub-menu for analysis depth +- Remote-only: runs remote pipeline, prints report, exits +- Full: runs preflight, launches parallel threads with split display + +### Remote Analysis Pipeline + +``` +gh run view {run_id} --log-failed + | + v + Python regex filter (ci_log_filter.py) + - Extract ERROR/FAIL/assertion/[FAIL] blocks + - Keep ~5 lines context around each match + - Strip ANSI codes, timestamps, build noise + | + v + ~50-200 lines (vs thousands raw) + | + v + claude --model haiku -p "Analyse these CI failures..." + | + v + Structured diagnosis +``` + +### Local Reproduction Pipeline (full mode only) + +``` +reproduce_ci() (existing) + - Create Docker container + - Clone repo, install deps, build, run tests + | + v + setup_claude_in_container() (existing) + | + v + Claude analysis with ANALYSIS_PROMPT_TEMPLATE (existing) + | + v + Local analysis results +``` + +### New file: `bin/ci_tool/ci_log_filter.py` + +Python regex-based log filter. Extracts failure-relevant lines with surrounding +context, strips ANSI codes and timestamps. Reduces thousands of raw log lines +to ~50-200 lines for Claude haiku. + +### New file: `bin/ci_tool/ci_analyse_display.py` + +`SplitPanelDisplay` class for the full parallel mode: +- `rich.live.Live` with `rich.layout.Layout` (two rows) +- Thread-safe `append_remote()` / `append_local()` methods +- Auto-refreshes at 4 Hz + +### Display Layout (full mode only) + +``` ++------------------------------------------+ +| Remote CI Logs | +| Package: my_pkg | +| Test: test_something | +| Error: AssertionError: expected 5, got 3 | +| Diagnosis: ... | ++------------------------------------------+ +| Local Reproduction | +| [Building workspace...] | +| [Running tests...] | +| [Analysing failures...] | ++------------------------------------------+ +``` + +## Changes to Existing Files + +- **`cli.py`**: Add `{"name": "Analyse CI", "value": "analyse"}` to + `MENU_CHOICES`. Add `"analyse": _handle_analyse` to `dispatch_subcommand`. + Add `_handle_analyse()` handler. Update `HELP_TEXT`. +- **No changes** to `ci_fix.py`, `ci_reproduce.py`, or other modules. + +## Error Handling + +- **GH logs unavailable:** Fail fast with clear error message +- **One thread fails (full mode):** Show partial results with warning +- **Claude haiku on host fails:** Fall back to displaying filtered logs raw +- **Container build fails (full mode):** Show error in bottom panel, remote + analysis still completes + +## Fix Transition (full mode only) + +After both panels complete: +1. Print combined report +2. Prompt: "Proceed to fix with Claude?" + - Yes: transition into existing `run_claude_workflow()` with container ready + - No: offer shell access or exit + +## Dependencies + +- `rich` (already in requirements.txt) — Live, Layout, Panel, Text +- `gh` CLI (already required) — for `gh run view --log-failed` +- `claude` on host — for haiku analysis of filtered remote logs +- `threading` (stdlib) — parallel execution (full mode only) diff --git a/docs/plans/2026-02-25-ci-analyse-mode-plan.md b/docs/plans/2026-02-25-ci-analyse-mode-plan.md new file mode 100644 index 0000000..cea7fea --- /dev/null +++ b/docs/plans/2026-02-25-ci-analyse-mode-plan.md @@ -0,0 +1,855 @@ +# CI Analyse Mode Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Add an "Analyse CI" menu item with two sub-modes: a fast remote-only analysis (fetch + filter GH Actions logs + Claude haiku diagnosis), and a full parallel mode that also reproduces locally in Docker with a split Rich Live display. + +**Architecture:** New `ci_analyse.py` module. Sub-menu after URL input: "Remote only (fast)" skips Docker entirely, just fetches/filters/diagnoses. "Remote + local reproduction" runs both pipelines in parallel with `SplitPanelDisplay`. Shared helpers for fetching/filtering/diagnosing reused by both paths. Existing modules untouched. + +**Tech Stack:** Python 3.6+, Rich (Live, Layout, Panel, Text), threading, subprocess, `gh` CLI, `claude` CLI on host + +--- + +### Task 1: Add log filtering module `ci_log_filter.py` + +Pre-processes raw GH Actions logs to extract only failure-relevant lines, reducing tokens before sending to Claude. + +**Files:** +- Create: `bin/ci_tool/ci_log_filter.py` + +**Step 1: Create the log filter module** + +```python +#!/usr/bin/env python3 +"""Filter CI logs to extract failure-relevant lines.""" +from __future__ import annotations + +import re + +# Patterns that indicate a failure or error in ROS/colcon CI output +FAILURE_PATTERNS = [ + re.compile(r'(?i)\bFAILURE\b'), + re.compile(r'(?i)\bFAILED\b'), + re.compile(r'(?i)\bERROR\b'), + re.compile(r'(?i)\b(?:Assertion|Assert)Error\b'), + re.compile(r'(?i)\bassert\b.*(?:!=|==|is not|not in)'), + re.compile(r'\[FAIL\]'), + re.compile(r'\[ERROR\]'), + re.compile(r'(?i)ERRORS?:?\s*\d+'), + re.compile(r'(?i)failures?:?\s*\d+'), + re.compile(r'(?i)Traceback \(most recent call last\)'), + re.compile(r'(?i)raise\s+\w+Error'), + re.compile(r'(?i)E\s+\w+Error:'), + re.compile(r'---\s*\>\s*'), +] + +ANSI_ESCAPE = re.compile(r'\x1b\[[0-9;]*m') + +CONTEXT_LINES_BEFORE = 3 +CONTEXT_LINES_AFTER = 5 +MAX_OUTPUT_LINES = 300 + + +def strip_ansi(text): + """Remove ANSI escape codes from text.""" + return ANSI_ESCAPE.sub('', text) + + +def strip_gh_log_timestamps(line): + """Strip GitHub Actions log timestamp prefixes like '2024-01-15T10:30:00.1234567Z '.""" + return re.sub(r'^\d{4}-\d{2}-\d{2}T[\d:.]+Z\s*', '', line) + + +def filter_ci_logs(raw_logs): + """Extract failure-relevant lines from raw CI logs with surrounding context. + + Returns a string of filtered lines ready for analysis, or empty string if + no failure patterns found. + """ + lines = raw_logs.split('\n') + cleaned_lines = [strip_gh_log_timestamps(strip_ansi(line)) for line in lines] + + matching_line_indices = set() + for line_index, line in enumerate(cleaned_lines): + for pattern in FAILURE_PATTERNS: + if pattern.search(line): + matching_line_indices.add(line_index) + break + + if not matching_line_indices: + return "" + + included_line_indices = set() + for match_index in sorted(matching_line_indices): + context_start = max(0, match_index - CONTEXT_LINES_BEFORE) + context_end = min(len(cleaned_lines), match_index + CONTEXT_LINES_AFTER + 1) + for context_index in range(context_start, context_end): + included_line_indices.add(context_index) + + result_lines = [] + previous_index = -2 + for line_index in sorted(included_line_indices): + if line_index > previous_index + 1: + result_lines.append("---") + result_lines.append(cleaned_lines[line_index]) + previous_index = line_index + + if len(result_lines) > MAX_OUTPUT_LINES: + result_lines = result_lines[:MAX_OUTPUT_LINES] + result_lines.append(f"\n... (truncated at {MAX_OUTPUT_LINES} lines)") + + return '\n'.join(result_lines) +``` + +**Step 2: Verify pylint passes** + +Run: `pylint --rcfile=pylintrc bin/ci_tool/ci_log_filter.py` +Expected: Score 10.0/10 + +**Step 3: Commit** + +```bash +git add bin/ci_tool/ci_log_filter.py +git commit -m "Add CI log filter module for pre-processing GH Actions logs" +``` + +--- + +### Task 2: Add split-panel display class `ci_analyse_display.py` + +Thread-safe Rich Live Layout with two panels. Only used by the "full" parallel mode. + +**Files:** +- Create: `bin/ci_tool/ci_analyse_display.py` + +**Step 1: Create the display module** + +```python +#!/usr/bin/env python3 +"""Split-panel Rich Live display for parallel CI analysis.""" +from __future__ import annotations + +import threading + +from rich.console import Console +from rich.layout import Layout +from rich.live import Live +from rich.panel import Panel +from rich.text import Text + + +class SplitPanelDisplay: + """Thread-safe split-panel display using Rich Live Layout. + + Top panel: remote CI log analysis + Bottom panel: local reproduction progress + """ + + def __init__(self): + self._console = Console() + self._lock = threading.Lock() + self._remote_lines = [] + self._local_lines = [] + self._remote_status = "Waiting..." + self._local_status = "Waiting..." + self._live = None + + def _build_panel_content(self, lines, status): + """Build panel content from accumulated lines and current status.""" + if not lines: + return Text(status, style="dim") + text = Text() + for line in lines[-30:]: + text.append(line + "\n") + text.append(f"\n[{status}]", style="dim") + return text + + def _build_layout(self): + """Build the full layout with both panels.""" + layout = Layout() + with self._lock: + remote_content = self._build_panel_content( + self._remote_lines, self._remote_status + ) + local_content = self._build_panel_content( + self._local_lines, self._local_status + ) + layout.split_column( + Layout( + Panel( + remote_content, + title="Remote CI Logs", + border_style="cyan", + ), + name="remote", + ), + Layout( + Panel( + local_content, + title="Local Reproduction", + border_style="green", + ), + name="local", + ), + ) + return layout + + def append_remote(self, line): + """Append a line to the remote panel (thread-safe).""" + with self._lock: + self._remote_lines.append(line) + + def append_local(self, line): + """Append a line to the local panel (thread-safe).""" + with self._lock: + self._local_lines.append(line) + + def set_remote_status(self, status): + """Update the remote panel status text (thread-safe).""" + with self._lock: + self._remote_status = status + + def set_local_status(self, status): + """Update the local panel status text (thread-safe).""" + with self._lock: + self._local_status = status + + def start(self): + """Start the live display. Returns the Live context for use with 'with'.""" + self._live = Live( + self._build_layout(), + console=self._console, + refresh_per_second=4, + ) + return self._live + + def refresh(self): + """Refresh the display with current state.""" + if self._live: + self._live.update(self._build_layout()) + + def get_remote_lines(self): + """Return a copy of all remote lines.""" + with self._lock: + return list(self._remote_lines) + + def get_local_lines(self): + """Return a copy of all local lines.""" + with self._lock: + return list(self._local_lines) +``` + +**Step 2: Verify pylint passes** + +Run: `pylint --rcfile=pylintrc bin/ci_tool/ci_analyse_display.py` +Expected: Score 10.0/10 + +**Step 3: Commit** + +```bash +git add bin/ci_tool/ci_analyse_display.py +git commit -m "Add split-panel display for parallel CI analysis" +``` + +--- + +### Task 3: Add main analyse module `ci_analyse.py` + +Core module with two code paths: remote-only (fast) and full parallel. Shared helpers for fetching, filtering, and diagnosing are used by both. + +**Files:** +- Create: `bin/ci_tool/ci_analyse.py` + +**Step 1: Create the analyse module** + +```python +#!/usr/bin/env python3 +"""Analyse CI failures from remote logs, optionally with local reproduction.""" +from __future__ import annotations + +import json +import subprocess +import sys +import threading +import time + +from InquirerPy import inquirer +from rich.console import Console +from rich.panel import Panel + +from ci_tool.ci_analyse_display import SplitPanelDisplay +from ci_tool.ci_fix import ( + build_analysis_prompt, + drop_to_shell, + extract_info_from_ci_url, + prompt_for_session_name, + refresh_claude_config, + run_claude_workflow, +) +from ci_tool.ci_log_filter import filter_ci_logs +from ci_tool.ci_reproduce import _parse_repo_url, reproduce_ci +from ci_tool.claude_setup import ( + copy_learnings_to_container, + is_claude_installed_in_container, + save_package_list, + setup_claude_in_container, +) +from ci_tool.containers import container_exists, remove_container +from ci_tool.preflight import run_all_preflight_checks, PreflightError + +console = Console() + +REMOTE_ANALYSIS_PROMPT = ( + "You are analysing CI failure logs from GitHub Actions. " + "The logs have been pre-filtered to show only failure-relevant lines.\n\n" + "For each failure, report:\n" + "- Package and test name\n" + "- The error/assertion message\n" + "- Your hypothesis for the root cause\n" + "- Suggested fix strategy\n\n" + "Be concise. Here are the filtered CI logs:\n\n{filtered_logs}" +) + +ANALYSE_DEPTH_CHOICES = [ + {"name": "Remote only (fast — no Docker)", "value": "remote_only"}, + {"name": "Remote + local reproduction (parallel)", "value": "full"}, +] + + +def _prompt_for_ci_url(): + """Ask user for the GitHub Actions run URL (required).""" + ci_run_url = inquirer.text( + message="GitHub Actions run URL:", + validate=lambda url: "/runs/" in url, + invalid_message=( + "URL must contain /runs/ " + "(e.g. https://github.com/org/repo/actions/runs/12345)" + ), + ).execute().strip() + + ci_run_info = extract_info_from_ci_url(ci_run_url) + console.print(f" [green]Repo:[/green] {ci_run_info['repo_url']}") + console.print(f" [green]Branch:[/green] {ci_run_info['branch']}") + console.print(f" [green]Run ID:[/green] {ci_run_info['run_id']}") + return ci_run_info + + +def _prompt_for_analyse_depth(): + """Ask user whether to do remote-only or full parallel analysis.""" + return inquirer.select( + message="Analysis depth:", + choices=ANALYSE_DEPTH_CHOICES, + default="remote_only", + ).execute() + + +# --------------------------------------------------------------------------- +# Shared helpers (used by both remote-only and full mode) +# --------------------------------------------------------------------------- + +def _fetch_failed_logs(run_id, owner_repo): + """Fetch failed job logs from GitHub Actions via gh CLI.""" + result = subprocess.run( + ["gh", "run", "view", run_id, "--log-failed", + "--repo", owner_repo], + capture_output=True, text=True, check=False, timeout=60, + ) + if result.returncode != 0: + raise RuntimeError( + f"Failed to fetch CI logs (exit {result.returncode}): " + f"{result.stderr.strip()[:300]}" + ) + if not result.stdout.strip(): + raise RuntimeError("GH Actions returned empty log output") + return result.stdout + + +def _run_claude_haiku_on_host(prompt): + """Run Claude haiku on the host to analyse filtered logs.""" + escaped_prompt = prompt.replace("'", "'\\''") + result = subprocess.run( + ["claude", "--model", "haiku", "-p", escaped_prompt, + "--max-turns", "1"], + capture_output=True, text=True, check=False, timeout=120, + ) + if result.returncode != 0: + raise RuntimeError( + f"Claude haiku failed (exit {result.returncode}): " + f"{result.stderr.strip()[:300]}" + ) + return result.stdout.strip() + + +def _fetch_filter_and_diagnose(ci_run_info): + """Fetch GH Actions logs, filter, and diagnose with Claude haiku. + + Returns (filtered_logs, diagnosis) tuple. + diagnosis is None if Claude haiku fails (filtered_logs still returned). + """ + run_id = ci_run_info["run_id"] + owner_repo = ci_run_info["owner_repo"] + + console.print("[cyan]Fetching CI logs...[/cyan]") + raw_logs = _fetch_failed_logs(run_id, owner_repo) + console.print(f" Fetched {len(raw_logs)} chars of log output") + + console.print("[cyan]Filtering logs...[/cyan]") + filtered_logs = filter_ci_logs(raw_logs) + if not filtered_logs: + console.print("[yellow]No failure patterns found in CI logs.[/yellow]") + return "", None + + filtered_line_count = len(filtered_logs.split('\n')) + console.print(f" Filtered to {filtered_line_count} lines") + + console.print("[cyan]Analysing with Claude haiku...[/cyan]") + try: + analysis_prompt = REMOTE_ANALYSIS_PROMPT.format( + filtered_logs=filtered_logs + ) + diagnosis = _run_claude_haiku_on_host(analysis_prompt) + except (RuntimeError, subprocess.TimeoutExpired) as error: + console.print( + f"[yellow]Claude haiku failed: {error}[/yellow]\n" + "[yellow]Showing filtered logs instead.[/yellow]" + ) + diagnosis = None + + return filtered_logs, diagnosis + + +# --------------------------------------------------------------------------- +# Remote-only mode +# --------------------------------------------------------------------------- + +def _run_remote_only(ci_run_info): + """Fast remote-only analysis: fetch, filter, diagnose, print report.""" + filtered_logs, diagnosis = _fetch_filter_and_diagnose(ci_run_info) + + console.print() + console.print(Panel("[bold cyan]Remote CI Analysis[/bold cyan]", expand=False)) + + if diagnosis: + console.print(diagnosis) + elif filtered_logs: + console.print(filtered_logs) + else: + console.print("[yellow]No failures found in CI logs.[/yellow]") + + +# --------------------------------------------------------------------------- +# Full parallel mode +# --------------------------------------------------------------------------- + +def _gather_full_mode_session_info(ci_run_info): + """Collect extra session info needed for local reproduction.""" + only_needed_deps = not inquirer.confirm( + message="Build everything (slower, disable --only-needed-deps)?", + default=False, + ).execute() + + container_name = prompt_for_session_name(ci_run_info["branch"]) + + return { + "ci_run_info": ci_run_info, + "repo_url": ci_run_info["repo_url"], + "branch": ci_run_info["branch"], + "only_needed_deps": only_needed_deps, + "container_name": container_name, + } + + +def _remote_analysis_thread(ci_run_info, display): + """Thread: fetch GH Actions logs, filter, analyse with Claude haiku.""" + run_id = ci_run_info["run_id"] + owner_repo = ci_run_info["owner_repo"] + + try: + display.set_remote_status("Fetching CI logs...") + display.refresh() + raw_logs = _fetch_failed_logs(run_id, owner_repo) + display.append_remote(f"Fetched {len(raw_logs)} chars of log output") + + display.set_remote_status("Filtering logs...") + display.refresh() + filtered_logs = filter_ci_logs(raw_logs) + if not filtered_logs: + display.append_remote("No failure patterns found in CI logs") + display.set_remote_status("Done (no failures found)") + display.refresh() + return + + filtered_line_count = len(filtered_logs.split('\n')) + display.append_remote(f"Filtered to {filtered_line_count} lines") + + display.set_remote_status("Analysing with Claude haiku...") + display.refresh() + analysis_prompt = REMOTE_ANALYSIS_PROMPT.format( + filtered_logs=filtered_logs + ) + diagnosis = _run_claude_haiku_on_host(analysis_prompt) + for line in diagnosis.split('\n'): + display.append_remote(line) + + display.set_remote_status("Done") + display.refresh() + + except (RuntimeError, subprocess.TimeoutExpired) as error: + display.append_remote(f"ERROR: {error}") + display.set_remote_status("Failed") + display.refresh() + + +def _local_reproduction_thread(session, gh_token, display): + """Thread: reproduce CI locally in Docker, then analyse.""" + container_name = session["container_name"] + + try: + display.set_local_status("Creating container & running CI...") + display.refresh() + + if container_exists(container_name): + remove_container(container_name) + reproduce_ci( + repo_url=session["repo_url"], + branch=session["branch"], + container_name=container_name, + gh_token=gh_token, + only_needed_deps=session["only_needed_deps"], + ) + save_package_list(container_name) + display.append_local("Container ready, build and tests complete") + + display.set_local_status("Setting up Claude in container...") + display.refresh() + if is_claude_installed_in_container(container_name): + refresh_claude_config(container_name) + else: + setup_claude_in_container(container_name) + + org, repo_name, _ = _parse_repo_url(session["repo_url"]) + if org and repo_name: + copy_learnings_to_container(container_name, org, repo_name) + + display.set_local_status("Analysing local test failures with Claude...") + display.refresh() + analysis_prompt = build_analysis_prompt(session["ci_run_info"]) + _run_container_analysis(container_name, analysis_prompt, display) + + display.set_local_status("Done") + display.refresh() + + except (RuntimeError, KeyboardInterrupt) as error: + display.append_local(f"ERROR: {error}") + display.set_local_status("Failed") + display.refresh() + + +def _run_container_analysis(container_name, prompt, display): + """Run Claude analysis inside container, capturing output to local panel.""" + escaped_prompt = prompt.replace("'", "'\\''") + claude_command = ( + f"cd /ros_ws && IS_SANDBOX=1 claude --dangerously-skip-permissions " + f"-p '{escaped_prompt}' --max-turns 10 --output-format stream-json " + f"2>/ros_ws/.claude_stderr.log" + ) + process = subprocess.Popen( + ["docker", "exec", "-e", "IS_SANDBOX=1", container_name, + "bash", "-c", claude_command], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + ) + for raw_line in process.stdout: + raw_line = raw_line.strip() + if not raw_line: + continue + try: + event = json.loads(raw_line) + _handle_stream_event(event, display) + except json.JSONDecodeError: + pass + process.wait() + + +def _handle_stream_event(event, display): + """Handle a Claude stream-json event, appending text to the local panel.""" + if event.get("type") != "assistant": + return + for block in event.get("message", {}).get("content", []): + if block.get("type") == "text": + text = block.get("text", "") + if text.strip(): + for text_line in text.split('\n'): + display.append_local(text_line) + display.refresh() + + +def _print_combined_report(display): + """Print the combined analysis report after both threads complete.""" + console.print("\n") + console.print(Panel("[bold cyan]Combined Analysis Report[/bold cyan]", expand=False)) + + remote_lines = display.get_remote_lines() + local_lines = display.get_local_lines() + + if remote_lines: + console.print("\n[bold]Remote CI Analysis:[/bold]") + for line in remote_lines: + console.print(f" {line}") + + if local_lines: + console.print("\n[bold]Local Reproduction Analysis:[/bold]") + for line in local_lines: + console.print(f" {line}") + + if not remote_lines and not local_lines: + console.print("[yellow]No analysis results from either source.[/yellow]") + + +def _offer_fix_transition(container_name, ci_run_info): + """Ask user if they want to proceed to fix mode.""" + proceed = inquirer.confirm( + message="Proceed to fix with Claude?", + default=True, + ).execute() + + if proceed: + console.print("\n[bold cyan]Transitioning to fix mode...[/bold cyan]") + run_claude_workflow(container_name, ci_run_info) + drop_to_shell(container_name) + else: + shell_choice = inquirer.confirm( + message="Drop into container shell?", + default=True, + ).execute() + if shell_choice: + drop_to_shell(container_name) + + +def _run_full_parallel(ci_run_info, gh_token): + """Full parallel analysis with split display: remote + local.""" + session = _gather_full_mode_session_info(ci_run_info) + display = SplitPanelDisplay() + + remote_thread = threading.Thread( + target=_remote_analysis_thread, + args=(ci_run_info, display), + daemon=True, + ) + local_thread = threading.Thread( + target=_local_reproduction_thread, + args=(session, gh_token, display), + daemon=True, + ) + + try: + with display.start() as _live: + remote_thread.start() + local_thread.start() + while remote_thread.is_alive() or local_thread.is_alive(): + display.refresh() + time.sleep(0.25) + display.refresh() + except KeyboardInterrupt: + console.print("\n[yellow]Interrupted.[/yellow]") + return + + _print_combined_report(display) + _offer_fix_transition(session["container_name"], ci_run_info) + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +def analyse_ci(_args): + """Main analyse workflow: URL -> depth choice -> run analysis -> report.""" + console.print( + Panel("[bold cyan]Analyse CI[/bold cyan]", expand=False) + ) + + ci_run_info = _prompt_for_ci_url() + analyse_depth = _prompt_for_analyse_depth() + + if analyse_depth == "remote_only": + _run_remote_only(ci_run_info) + return + + # Full mode needs preflight checks (Docker, token, Claude credentials) + try: + gh_token = run_all_preflight_checks( + repo_url=ci_run_info["repo_url"] + ) + except PreflightError as error: + console.print( + f"\n[bold red]Preflight failed:[/bold red] {error}" + ) + sys.exit(1) + + _run_full_parallel(ci_run_info, gh_token) +``` + +**Step 2: Verify pylint passes** + +Run: `pylint --rcfile=pylintrc bin/ci_tool/ci_analyse.py` +Expected: Score 10.0/10 + +**Step 3: Commit** + +```bash +git add bin/ci_tool/ci_analyse.py +git commit -m "Add CI analysis module with remote-only and full parallel modes" +``` + +--- + +### Task 4: Wire up the menu in `cli.py` + +Add the "Analyse CI" menu item and dispatcher entry. + +**Files:** +- Modify: `bin/ci_tool/cli.py:14-22` (MENU_CHOICES) +- Modify: `bin/ci_tool/cli.py:24-40` (HELP_TEXT) +- Modify: `bin/ci_tool/cli.py:73-80` (dispatch_subcommand handlers) +- Add: `_handle_analyse()` function + +**Step 1: Add "Analyse CI" to MENU_CHOICES** + +In `bin/ci_tool/cli.py`, insert after "Reproduce CI" (line 15): + +```python +MENU_CHOICES = [ + {"name": "Reproduce CI (create container)", "value": "reproduce"}, + {"name": "Analyse CI", "value": "analyse"}, + {"name": "Fix CI with Claude", "value": "fix"}, + {"name": "Claude session (interactive)", "value": "claude"}, + {"name": "Shell into container", "value": "shell"}, + {"name": "Re-run tests in container", "value": "retest"}, + {"name": "Clean up containers", "value": "clean"}, + {"name": "Exit", "value": "exit"}, +] +``` + +**Step 2: Update HELP_TEXT** + +Add `analyse` to the commands list: + +``` +Commands: + analyse Analyse CI failures (remote-only or with local reproduction) + fix Fix CI failures with Claude + reproduce Reproduce CI environment in Docker + claude Interactive Claude session in container + shell Shell into an existing CI container + retest Re-run tests in a CI container + clean Remove CI containers +``` + +**Step 3: Add handler to dispatch_subcommand** + +Add `"analyse": _handle_analyse` to the handlers dict: + +```python +handlers = { + "reproduce": _handle_reproduce, + "analyse": _handle_analyse, + "fix": _handle_fix, + "claude": _handle_claude, + "shell": _handle_shell, + "retest": _handle_retest, + "clean": _handle_clean, +} +``` + +**Step 4: Add _handle_analyse function** + +Add alongside the other handler functions: + +```python +def _handle_analyse(args): + from ci_tool.ci_analyse import analyse_ci + analyse_ci(args) +``` + +**Step 5: Verify pylint passes** + +Run: `pylint --rcfile=pylintrc bin/ci_tool/cli.py` +Expected: Score 10.0/10 + +**Step 6: Commit** + +```bash +git add bin/ci_tool/cli.py +git commit -m "Add 'Analyse CI' to main menu and command dispatch" +``` + +--- + +### Task 5: Manual end-to-end test + +No unit tests yet — test the full workflow manually. + +**Step 1: Verify the module loads** + +Run: `cd /cortex/er_build_tools && python3 -c "from ci_tool.ci_analyse import analyse_ci; print('OK')"` +Expected: `OK` + +**Step 2: Verify menu shows new option** + +Run: `cd /cortex/er_build_tools && python3 -m ci_tool --help` +Expected: Output includes `analyse` in the commands list + +**Step 3: Run full pylint on the package** + +Run: `pylint --rcfile=pylintrc bin/ci_tool/` +Expected: Score 10.0/10 + +**Step 4: Test remote-only mode with a real GH Actions URL (if available)** + +Run: `cd /cortex/er_build_tools && python3 -m ci_tool analyse` +- Select a GH Actions URL +- Choose "Remote only (fast)" +- Expected: fetches logs, filters, sends to Claude haiku, prints diagnosis + +**Step 5: Commit any fixes needed** + +```bash +git add -u +git commit -m "Fix issues found during manual testing of analyse mode" +``` + +--- + +### Task 6: Update CLAUDE.md project docs + +**Files:** +- Modify: `CLAUDE.md` — add `analyse` to Running section and new files to Project Structure + +**Step 1: Add to Running section** + +``` +ci_tool # interactive menu +ci_tool analyse # analyse CI failures (remote-only or with local reproduction) +ci_fix # shortcut for ci_tool fix +``` + +**Step 2: Add to Project Structure** + +Under `bin/ci_tool/`, add: + +``` + ci_analyse.py # CI analysis: remote-only or parallel with local reproduction + ci_analyse_display.py # Split-panel Rich Live display for parallel analysis + ci_log_filter.py # Pre-filter GH Actions logs to reduce token usage +``` + +**Step 3: Verify pylint still passes** + +Run: `pylint --rcfile=pylintrc bin/ci_tool/` +Expected: Score 10.0/10 + +**Step 4: Commit** + +```bash +git add CLAUDE.md +git commit -m "Document new 'analyse' command in CLAUDE.md" +```