From a7e5e9b0b90d15b9dd9a8fa0810f4c624549759d Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sun, 1 Mar 2026 19:50:07 -0500 Subject: [PATCH 01/35] Add gcov-based test pruning with file-level coverage cache - File-level gcov coverage cache maps 555 test UUIDs to exercised .fpp source files (gzip JSON, 11KB, committed to repo) - --only-changes flag prunes tests by intersecting PR-changed files against coverage cache; --build-coverage-cache builds the cache - New rebuild-cache CI job runs on Phoenix via SLURM when cases.py or Fortran dependency graph changes (on both PRs and master pushes) - Dep-change detection greps PR/push diffs for added use/include statements that would invalidate the coverage cache - Conservative fallbacks: missing cache runs all, missing sim coverage includes test, ALWAYS_RUN_ALL files trigger full suite - Remove continue-on-error from github CI job (fixes auto-cancellation) - TEMP: duplicate use in m_bubbles.fpp + remove CMakeLists.txt from ALWAYS_RUN_ALL to test the full cache rebuild pipeline in CI - 53 unit tests cover core coverage logic Co-Authored-By: Claude Opus 4.6 --- .github/file-filter.yml | 3 + .github/workflows/frontier/test.sh | 10 +- .github/workflows/phoenix/rebuild-cache.sh | 21 + .github/workflows/phoenix/submit.sh | 6 +- .github/workflows/phoenix/test.sh | 12 +- .github/workflows/test.yml | 129 +++- .gitignore | 3 + CMakeLists.txt | 37 +- src/simulation/m_bubbles.fpp | 1 + toolchain/mfc/cli/commands.py | 25 + toolchain/mfc/test/coverage.py | 659 ++++++++++++++++++ toolchain/mfc/test/test.py | 67 +- .../mfc/test/test_coverage_cache.json.gz | Bin 0 -> 11779 bytes toolchain/mfc/test/test_coverage_unit.py | 655 +++++++++++++++++ 14 files changed, 1601 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/phoenix/rebuild-cache.sh create mode 100644 toolchain/mfc/test/coverage.py create mode 100644 toolchain/mfc/test/test_coverage_cache.json.gz create mode 100644 toolchain/mfc/test/test_coverage_unit.py diff --git a/.github/file-filter.yml b/.github/file-filter.yml index c0e7477cf2..0cc4698db8 100644 --- a/.github/file-filter.yml +++ b/.github/file-filter.yml @@ -37,3 +37,6 @@ checkall: &checkall - *tests - *scripts - *yml + +cases_py: + - 'toolchain/mfc/test/cases.py' diff --git a/.github/workflows/frontier/test.sh b/.github/workflows/frontier/test.sh index 78797ab8ec..1cfcff6fec 100644 --- a/.github/workflows/frontier/test.sh +++ b/.github/workflows/frontier/test.sh @@ -9,12 +9,18 @@ if [ -n "$job_shard" ]; then shard_opts="--shard $job_shard" fi +# Only prune tests on PRs; master pushes must run the full suite. +prune_flag="" +if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then + prune_flag="--only-changes" +fi + if [ "$job_device" = "gpu" ]; then rdma_opts="" if [ "$job_cluster" = "frontier" ]; then rdma_opts="--rdma-mpi" fi - ./mfc.sh test -v -a $rdma_opts --max-attempts 3 -j $ngpus $device_opts $shard_opts -- -c $job_cluster + ./mfc.sh test -v -a $rdma_opts --max-attempts 3 $prune_flag -j $ngpus $device_opts $shard_opts -- -c $job_cluster else - ./mfc.sh test -v -a --max-attempts 3 -j 32 --no-gpu $shard_opts -- -c $job_cluster + ./mfc.sh test -v -a --max-attempts 3 $prune_flag -j 32 --no-gpu $shard_opts -- -c $job_cluster fi diff --git a/.github/workflows/phoenix/rebuild-cache.sh b/.github/workflows/phoenix/rebuild-cache.sh new file mode 100644 index 0000000000..14db7c83e5 --- /dev/null +++ b/.github/workflows/phoenix/rebuild-cache.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -e + +# Number of parallel jobs: use SLURM allocation or default to 24. +# Cap at 64 to avoid overwhelming MPI's ORTE daemons with concurrent launches. +NJOBS="${SLURM_CPUS_ON_NODE:-24}" +if [ "$NJOBS" -gt 64 ]; then NJOBS=64; fi + +# Clean stale build artifacts: the self-hosted runner may have a cached +# GPU build (e.g. --gpu mp) whose CMake flags are incompatible with gcov. +./mfc.sh clean + +# Build MFC with gcov coverage instrumentation (CPU-only, gfortran). +# -j 8 for compilation (memory-heavy, more cores doesn't help much). +./mfc.sh build --gcov -j 8 + +# Run all tests in parallel, collecting per-test coverage data. +# Each test gets an isolated GCOV_PREFIX directory so .gcda files +# don't collide. Coverage is collected per-test after all tests finish. +# --gcov is required so the internal build step preserves instrumentation. +./mfc.sh test --build-coverage-cache --gcov -j "$NJOBS" diff --git a/.github/workflows/phoenix/submit.sh b/.github/workflows/phoenix/submit.sh index 5b7162fef7..b52a107cca 100755 --- a/.github/workflows/phoenix/submit.sh +++ b/.github/workflows/phoenix/submit.sh @@ -24,9 +24,9 @@ case "$script_basename" in esac sbatch_cpu_opts="\ -#SBATCH -p cpu-small # partition -#SBATCH --ntasks-per-node=24 # Number of cores per node required -#SBATCH --mem-per-cpu=2G # Memory per core\ +#SBATCH -p cpu-gnr # partition (full Granite Rapids node) +#SBATCH --exclusive # exclusive access to all cores +#SBATCH -C graniterapids # constrain to GNR architecture\ " if [ "$job_type" = "bench" ]; then diff --git a/.github/workflows/phoenix/test.sh b/.github/workflows/phoenix/test.sh index 6816bd9a25..ddf1b958d5 100644 --- a/.github/workflows/phoenix/test.sh +++ b/.github/workflows/phoenix/test.sh @@ -12,7 +12,9 @@ source .github/scripts/retry-build.sh RETRY_VALIDATE_CMD='syscheck_bin=$(find build/install -name syscheck -type f 2>/dev/null | head -1); [ -z "$syscheck_bin" ] || "$syscheck_bin" > /dev/null 2>&1' \ retry_build ./mfc.sh test -v --dry-run -j 8 $build_opts || exit 1 -n_test_threads=8 +# Use up to 64 parallel test threads on CPU (GNR nodes have 192 cores). +# Cap at 64 to avoid overwhelming MPI's ORTE daemons with concurrent launches. +n_test_threads=$(( SLURM_CPUS_ON_NODE > 64 ? 64 : ${SLURM_CPUS_ON_NODE:-8} )) if [ "$job_device" = "gpu" ]; then source .github/scripts/detect-gpus.sh @@ -20,4 +22,10 @@ if [ "$job_device" = "gpu" ]; then n_test_threads=$((ngpus * 2)) fi -./mfc.sh test -v --max-attempts 3 -a -j $n_test_threads $device_opts -- -c phoenix +# Only prune tests on PRs; master pushes must run the full suite. +prune_flag="" +if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then + prune_flag="--only-changes" +fi + +./mfc.sh test -v --max-attempts 3 $prune_flag -a -j $n_test_threads $device_opts -- -c phoenix diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5dd072072d..7f866553f0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,8 +56,10 @@ jobs: file-changes: name: Detect File Changes runs-on: 'ubuntu-latest' - outputs: + outputs: checkall: ${{ steps.changes.outputs.checkall }} + cases_py: ${{ steps.changes.outputs.cases_py }} + dep_changed: ${{ steps.dep-check.outputs.dep_changed }} steps: - name: Clone uses: actions/checkout@v4 @@ -65,13 +67,95 @@ jobs: - name: Detect Changes uses: dorny/paths-filter@v3 id: changes - with: + with: filters: ".github/file-filter.yml" + - name: Check for Fortran dependency changes + id: dep-check + env: + GH_TOKEN: ${{ github.token }} + run: | + # Detect added/removed use/include statements that change the + # Fortran dependency graph, which would make the coverage cache stale. + if [ "${{ github.event_name }}" = "pull_request" ]; then + DIFF=$(gh pr diff ${{ github.event.pull_request.number }}) + elif [ "${{ github.event_name }}" = "push" ]; then + DIFF=$(git diff ${{ github.event.before }}..${{ github.event.after }} 2>/dev/null || echo "") + else + DIFF="" + fi + if echo "$DIFF" | \ + grep -qP '^\+\s*(use[\s,]+\w|#:include\s|include\s+['"'"'"])'; then + echo "dep_changed=true" >> "$GITHUB_OUTPUT" + echo "Fortran dependency change detected — will rebuild coverage cache." + else + echo "dep_changed=false" >> "$GITHUB_OUTPUT" + fi + + rebuild-cache: + name: Rebuild Coverage Cache + needs: [lint-gate, file-changes] + if: >- + github.repository == 'MFlowCode/MFC' && + ( + (github.event_name == 'pull_request' && + (needs.file-changes.outputs.cases_py == 'true' || + needs.file-changes.outputs.dep_changed == 'true')) || + (github.event_name == 'push' && + (needs.file-changes.outputs.cases_py == 'true' || + needs.file-changes.outputs.dep_changed == 'true')) || + github.event_name == 'workflow_dispatch' + ) + timeout-minutes: 240 + runs-on: + group: phoenix + labels: gt + permissions: + contents: write + steps: + - name: Clone + uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + clean: false + + - name: Rebuild Cache via SLURM + run: bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/rebuild-cache.sh cpu none + + - name: Print Logs + if: always() + run: cat rebuild-cache-cpu-none.out + + - name: Upload Cache Artifact + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@v4 + with: + name: coverage-cache + path: toolchain/mfc/test/test_coverage_cache.json.gz + retention-days: 1 + + - name: Commit Cache to Master + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add toolchain/mfc/test/test_coverage_cache.json.gz + if git diff --cached --quiet; then + echo "Coverage cache unchanged." + else + git commit -m "Regenerate gcov coverage cache [skip ci]" + git push + fi + github: name: Github - if: needs.file-changes.outputs.checkall == 'true' - needs: [lint-gate, file-changes] + needs: [lint-gate, file-changes, rebuild-cache] + if: >- + always() && + needs.lint-gate.result == 'success' && + needs.file-changes.result == 'success' && + needs.rebuild-cache.result != 'cancelled' && + needs.file-changes.outputs.checkall == 'true' strategy: matrix: os: ['ubuntu', 'macos'] @@ -91,13 +175,26 @@ jobs: intel: false fail-fast: false - continue-on-error: true runs-on: ${{ matrix.os }}-latest steps: - name: Clone uses: actions/checkout@v4 + - name: Fetch master for coverage diff + run: | + git fetch origin master:master --depth=1 + git fetch --deepen=200 + continue-on-error: true + + - name: Download Coverage Cache + if: needs.rebuild-cache.result == 'success' + uses: actions/download-artifact@v4 + with: + name: coverage-cache + path: toolchain/mfc/test + continue-on-error: true + - name: Setup MacOS if: matrix.os == 'macos' run: | @@ -156,15 +253,23 @@ jobs: PRECISION: ${{ matrix.precision != '' && format('--{0}', matrix.precision) || '' }} - name: Test - run: bash .github/scripts/run-tests-with-retry.sh -v --max-attempts 3 -j "$(nproc)" $TEST_ALL $TEST_PCT + run: bash .github/scripts/run-tests-with-retry.sh -v --max-attempts 3 -j "$(nproc)" $ONLY_CHANGES $TEST_ALL $TEST_PCT env: TEST_ALL: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }} TEST_PCT: ${{ matrix.debug == 'debug' && '-% 20' || '' }} + ONLY_CHANGES: ${{ github.event_name == 'pull_request' && '--only-changes' || '' }} self: name: "${{ matrix.cluster_name }} (${{ matrix.device }}${{ matrix.interface != 'none' && format('-{0}', matrix.interface) || '' }}${{ matrix.shard != '' && format(' [{0}]', matrix.shard) || '' }})" - if: github.repository == 'MFlowCode/MFC' && needs.file-changes.outputs.checkall == 'true' && github.event.pull_request.draft != true - needs: [lint-gate, file-changes] + needs: [lint-gate, file-changes, rebuild-cache] + if: >- + always() && + needs.lint-gate.result == 'success' && + needs.file-changes.result == 'success' && + needs.rebuild-cache.result != 'cancelled' && + github.repository == 'MFlowCode/MFC' && + needs.file-changes.outputs.checkall == 'true' && + github.event.pull_request.draft != true continue-on-error: false timeout-minutes: 480 strategy: @@ -245,6 +350,14 @@ jobs: with: clean: false + - name: Download Coverage Cache + if: needs.rebuild-cache.result == 'success' + uses: actions/download-artifact@v4 + with: + name: coverage-cache + path: toolchain/mfc/test + continue-on-error: true + - name: Build if: matrix.cluster != 'phoenix' uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3 diff --git a/.gitignore b/.gitignore index e80d14a6f9..943624a1f7 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,9 @@ __pycache__ # Auto-generated version file toolchain/mfc/_version.py +# Raw coverage cache — legacy, not tracked (the .json.gz version IS committed) +toolchain/mfc/test/test_coverage_cache.json + # Auto-generated toolchain files (regenerate with: ./mfc.sh generate) toolchain/completions/mfc.bash toolchain/completions/_mfc diff --git a/CMakeLists.txt b/CMakeLists.txt index ddb3876724..7154fa3010 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -131,13 +131,20 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") add_compile_options( $<$:-fprofile-arcs> $<$:-ftest-coverage> - $<$:-O1> - ) + ) add_link_options( $<$:-lgcov> $<$:--coverage> ) + + # Override Release -O3 with -O1 for gcov: coverage instrumentation is + # inaccurate at -O3, and aggressive codegen (e.g. AVX-512 FP16 on + # Granite Rapids) can emit instructions that older assemblers reject. + set(CMAKE_Fortran_FLAGS_RELEASE "-O1 -DNDEBUG" CACHE STRING "" FORCE) + + # Use gfortran5 line markers so gcov can map coverage to .fpp sources. + set(FYPP_GCOV_OPTS "--line-marker-format=gfortran5") endif() if (CMAKE_BUILD_TYPE STREQUAL "Debug") @@ -224,18 +231,25 @@ endif() if (CMAKE_BUILD_TYPE STREQUAL "Release") # Processor tuning: Check if we can target the host's native CPU's ISA. - CHECK_FORTRAN_COMPILER_FLAG("-march=native" SUPPORTS_MARCH_NATIVE) - if (SUPPORTS_MARCH_NATIVE) - add_compile_options($<$:-march=native>) - else() - CHECK_FORTRAN_COMPILER_FLAG("-mcpu=native" SUPPORTS_MCPU_NATIVE) - if (SUPPORTS_MCPU_NATIVE) - add_compile_options($<$:-mcpu=native>) + # Skip for gcov builds — -march=native on newer CPUs (e.g. Granite Rapids) + # can emit instructions the system assembler doesn't support. + if (NOT MFC_GCov) + CHECK_FORTRAN_COMPILER_FLAG("-march=native" SUPPORTS_MARCH_NATIVE) + if (SUPPORTS_MARCH_NATIVE) + add_compile_options($<$:-march=native>) + else() + CHECK_FORTRAN_COMPILER_FLAG("-mcpu=native" SUPPORTS_MCPU_NATIVE) + if (SUPPORTS_MCPU_NATIVE) + add_compile_options($<$:-mcpu=native>) + endif() endif() endif() - # Enable LTO/IPO if supported - if (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") + # Enable LTO/IPO if supported (skip for gcov — LTO interferes with coverage + # instrumentation and can trigger assembler errors on newer architectures). + if (MFC_GCov) + message(STATUS "LTO/IPO disabled for gcov build") + elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") if (MFC_Unified) message(STATUS "LTO/IPO is not available with NVHPC using Unified Memory") elseif (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER "24.11" AND CMAKE_Fortran_COMPILER_VERSION VERSION_LESS "25.9") @@ -381,6 +395,7 @@ macro(HANDLE_SOURCES target useCommon) --no-folding --line-length=999 --line-numbering-mode=nocontlines + ${FYPP_GCOV_OPTS} "${fpp}" "${f90}" DEPENDS "${fpp};${${target}_incs}" COMMENT "Preprocessing (Fypp) ${fpp_filename}" diff --git a/src/simulation/m_bubbles.fpp b/src/simulation/m_bubbles.fpp index 0f17bd60c3..28d3f2145b 100644 --- a/src/simulation/m_bubbles.fpp +++ b/src/simulation/m_bubbles.fpp @@ -16,6 +16,7 @@ module m_bubbles use m_variables_conversion !< State variables type conversion procedures use m_helper_basic !< Functions to compare floating point numbers + use m_helper_basic implicit none diff --git a/toolchain/mfc/cli/commands.py b/toolchain/mfc/cli/commands.py index d4b34df3d8..618ec1aea6 100644 --- a/toolchain/mfc/cli/commands.py +++ b/toolchain/mfc/cli/commands.py @@ -458,6 +458,27 @@ type=str, default=None, ), + Argument( + name="build-coverage-cache", + help="Run all tests with gcov instrumentation to build the file-level coverage cache. Requires a prior --gcov build: ./mfc.sh build --gcov -j 8", + action=ArgAction.STORE_TRUE, + default=False, + dest="build_coverage_cache", + ), + Argument( + name="only-changes", + help="Only run tests whose covered files overlap with files changed since branching from master (uses file-level gcov coverage cache).", + action=ArgAction.STORE_TRUE, + default=False, + dest="only_changes", + ), + Argument( + name="changes-branch", + help="Branch to compare against for --only-changes (default: master).", + type=str, + default="master", + dest="changes_branch", + ), ], mutually_exclusive=[ MutuallyExclusiveGroup(arguments=[ @@ -488,6 +509,8 @@ Example("./mfc.sh test -j 4", "Run with 4 parallel jobs"), Example("./mfc.sh test --only 3D", "Run only 3D tests"), Example("./mfc.sh test --generate", "Regenerate golden files"), + Example("./mfc.sh test --only-changes -j 4", "Run tests affected by changed files"), + Example("./mfc.sh build --gcov -j 8 && ./mfc.sh test --build-coverage-cache", "One-time: build file-coverage cache"), ], key_options=[ ("-j, --jobs N", "Number of parallel test jobs"), @@ -495,6 +518,8 @@ ("-f, --from UUID", "Start from specific test"), ("--generate", "Generate/update golden files"), ("--no-build", "Skip rebuilding MFC"), + ("--build-coverage-cache", "Build file-level gcov coverage cache (one-time)"), + ("--only-changes", "Run tests affected by changed files (requires cache)"), ], ) diff --git a/toolchain/mfc/test/coverage.py b/toolchain/mfc/test/coverage.py new file mode 100644 index 0000000000..404715976e --- /dev/null +++ b/toolchain/mfc/test/coverage.py @@ -0,0 +1,659 @@ +""" +File-level gcov coverage-based test pruning for MFC. + +Build MFC once with gfortran --coverage, run all tests individually, record +which .fpp files each test executes, and cache that mapping. + +When files change on a PR, intersect the changed .fpp files against each test's +covered file set. Only tests that touch at least one changed file run. + +Workflow: + ./mfc.sh build --gcov -j 8 # one-time: build with coverage + ./mfc.sh test --build-coverage-cache # one-time: populate the cache + ./mfc.sh test --only-changes -j 8 # fast: run only affected tests +""" + +import io +import os +import re +import json +import gzip +import shutil +import hashlib +import tempfile +import subprocess +import datetime +from pathlib import Path +from typing import Optional +from concurrent.futures import ThreadPoolExecutor, as_completed + +from ..printer import cons +from .. import common +from ..common import MFCException +from ..build import PRE_PROCESS, SIMULATION, POST_PROCESS, SYSCHECK +from .case import input_bubbles_lagrange + + +COVERAGE_CACHE_PATH = Path(common.MFC_ROOT_DIR) / "toolchain/mfc/test/test_coverage_cache.json.gz" + +# Changes to these files trigger the full test suite. +# CPU coverage cannot tell us about GPU directive changes (macro files), and +# toolchain files define or change the set of tests themselves. +ALWAYS_RUN_ALL = frozenset([ + "src/common/include/parallel_macros.fpp", + "src/common/include/acc_macros.fpp", + "src/common/include/omp_macros.fpp", + "src/common/include/shared_parallel_macros.fpp", + "src/common/include/macros.fpp", + "src/common/include/case.fpp", + "toolchain/mfc/test/cases.py", + "toolchain/mfc/test/case.py", + "toolchain/mfc/params/definitions.py", + "toolchain/mfc/run/input.py", + "toolchain/mfc/case_validator.py", + "toolchain/mfc/test/coverage.py", +]) + +# Directory prefixes: any changed file under these paths triggers full suite. +# Note: src/simulation/include/ (.fpp files like inline_riemann.fpp) is NOT +# listed here — Fypp line markers (--line-marker-format=gfortran5) correctly +# attribute included file paths, so gcov coverage tracks them accurately. +ALWAYS_RUN_ALL_PREFIXES = ( + "toolchain/cmake/", +) + + +def _get_gcov_version(gcov_binary: str) -> str: + """Return the version string from gcov --version.""" + try: + result = subprocess.run( + [gcov_binary, "--version"], + capture_output=True, text=True, timeout=10, check=False + ) + for line in result.stdout.splitlines(): + if line.strip(): + return line.strip() + except Exception: + pass + return "unknown" + + +def find_gcov_binary(_root_dir: str = "") -> str: # pylint: disable=unused-argument + """ + Find a GNU gcov binary compatible with the system gfortran. + + On macOS with Homebrew GCC, the binary is gcov-{major} (e.g. gcov-15). + On Linux with system GCC, plain gcov is usually correct. + Apple LLVM's /usr/bin/gcov is incompatible with gfortran .gcda files. + """ + # Determine gfortran major version + major = None + try: + result = subprocess.run( + ["gfortran", "--version"], + capture_output=True, text=True, timeout=10, check=False + ) + m = re.search(r'(\d+)\.\d+\.\d+', result.stdout) + if m: + major = m.group(1) + except Exception: + pass + + # Try versioned binary first (Homebrew macOS), then plain gcov + candidates = [] + if major: + candidates.append(f"gcov-{major}") + candidates.append("gcov") + + for candidate in candidates: + path = shutil.which(candidate) + if path is None: + continue + try: + result = subprocess.run( + [path, "--version"], + capture_output=True, text=True, timeout=10, check=False + ) + version_out = result.stdout + if "Apple LLVM" in version_out or "Apple clang" in version_out: + continue # Apple's gcov cannot parse GCC-generated .gcda files + if "GCC" in version_out or "GNU" in version_out: + return path + except Exception: + continue + + raise MFCException( + "GNU gcov not found. gcov is required for the coverage cache.\n" + " On macOS (Homebrew): brew install gcc\n" + " On Linux (Debian/Ubuntu): apt install gcc\n" + " On Linux (RHEL/CentOS): yum install gcc\n" + "Apple's /usr/bin/gcov is incompatible with gfortran .gcda files." + ) + + +def find_gcno_files(root_dir: str) -> list: + """ + Walk build/ and return all .gcno files (excluding venv paths). + Raises if none found (indicates build was not done with --gcov). + """ + build_dir = Path(root_dir) / "build" + gcno_files = [ + p for p in build_dir.rglob("*.gcno") + if "venv" not in p.parts + ] + if not gcno_files: + raise MFCException( + "No .gcno files found. Build with --gcov instrumentation first:\n" + " ./mfc.sh build --gcov -j 8" + ) + return gcno_files + + + +def _parse_gcov_json_output(raw_bytes: bytes, root_dir: str) -> set: + """ + Parse gcov JSON output and return the set of .fpp file paths with coverage. + Handles both gzip-compressed (gcov 13+) and raw JSON (gcov 12) formats. + Handles concatenated JSON objects from batched gcov calls (multiple .gcno + files passed to a single gcov invocation). + Only .fpp files with at least one executed line are included. + """ + try: + text = gzip.decompress(raw_bytes).decode("utf-8", errors="replace") + except (gzip.BadGzipFile, OSError): + try: + text = raw_bytes.decode("utf-8", errors="replace") + except (UnicodeDecodeError, ValueError): + return set() + + result = set() + real_root = os.path.realpath(root_dir) + + # Parse potentially concatenated JSON objects (one per .gcno file). + decoder = json.JSONDecoder() + pos = 0 + while pos < len(text): + while pos < len(text) and text[pos] in " \t\n\r": + pos += 1 + if pos >= len(text): + break + try: + data, end_pos = decoder.raw_decode(text, pos) + pos = end_pos + except json.JSONDecodeError: + break + + for file_entry in data.get("files", []): + file_path = file_entry.get("file", "") + if not file_path.endswith(".fpp"): + continue + if any(line.get("count", 0) > 0 for line in file_entry.get("lines", [])): + try: + rel_path = os.path.relpath(os.path.realpath(file_path), real_root) + except ValueError: + rel_path = file_path + result.add(rel_path) + + return result + + +def _compute_gcov_prefix_strip(root_dir: str) -> str: + """ + Compute GCOV_PREFIX_STRIP so .gcda files preserve the build/ tree. + + GCOV_PREFIX_STRIP removes N leading path components from the compile-time + absolute .gcda path. We strip all components of the MFC root directory + so the prefix tree starts with ``build/staging/...``. + """ + real_root = os.path.realpath(root_dir) + return str(len(Path(real_root).parts) - 1) # -1 excludes root '/' + + +def _collect_single_test_coverage( # pylint: disable=too-many-locals + uuid: str, test_gcda: str, root_dir: str, gcov_bin: str, +) -> tuple: + """ + Collect file-level coverage for a single test, fully self-contained. + + Copies .gcno files from the real build tree into the test's isolated + .gcda directory (alongside the .gcda files), runs a batched gcov call, + then removes the .gcno copies. Each test has its own directory, so + this is safe to call concurrently without touching the shared build tree. + """ + build_subdir = os.path.join(test_gcda, "build") + if not os.path.isdir(build_subdir): + return uuid, [] + + gcno_copies = [] + + for dirpath, _, filenames in os.walk(build_subdir): + for fname in filenames: + if not fname.endswith(".gcda"): + continue + # Derive matching .gcno path in the real build tree + gcda_path = os.path.join(dirpath, fname) + rel = os.path.relpath(gcda_path, test_gcda) + gcno_rel = rel[:-5] + ".gcno" + gcno_src = os.path.join(root_dir, gcno_rel) + if os.path.isfile(gcno_src): + # Copy .gcno alongside .gcda in the test's isolated dir + gcno_dst = os.path.join(dirpath, fname[:-5] + ".gcno") + shutil.copy2(gcno_src, gcno_dst) + gcno_copies.append(gcno_dst) + + if not gcno_copies: + return uuid, [] + + # Batch: single gcov call for all .gcno files in this test. + # Run from root_dir so source path resolution works correctly. + cmd = [gcov_bin, "--json-format", "--stdout"] + gcno_copies + try: + proc = subprocess.run( + cmd, capture_output=True, cwd=root_dir, timeout=120, check=False + ) + except (subprocess.TimeoutExpired, subprocess.SubprocessError, OSError): + return uuid, [] + finally: + for g in gcno_copies: + try: + os.remove(g) + except OSError: + pass + + if proc.returncode != 0 or not proc.stdout: + return uuid, [] + + coverage = _parse_gcov_json_output(proc.stdout, root_dir) + return uuid, sorted(coverage) + + +def _run_single_test_direct(test_info: dict, gcda_dir: str, strip: str) -> tuple: # pylint: disable=too-many-locals + """ + Run a single test by invoking Fortran executables directly. + + Bypasses ``./mfc.sh run`` entirely (no Python startup, no Mako template + rendering, no shell script generation). Input files and binary paths are + pre-computed by the caller. + + Returns (uuid, test_gcda_path, failures). + """ + uuid = test_info["uuid"] + test_dir = test_info["dir"] + binaries = test_info["binaries"] # ordered list of (target_name, bin_path) + ppn = test_info["ppn"] + + test_gcda = os.path.join(gcda_dir, uuid) + os.makedirs(test_gcda, exist_ok=True) + + env = {**os.environ, "GCOV_PREFIX": test_gcda, "GCOV_PREFIX_STRIP": strip} + + # MPI-compiled binaries must be launched via an MPI launcher (even ppn=1). + # Use --bind-to none to avoid binding issues with concurrent launches. + if shutil.which("mpirun"): + mpi_cmd = ["mpirun", "--bind-to", "none", "-np", str(ppn)] + elif shutil.which("srun"): + mpi_cmd = ["srun", "--ntasks", str(ppn)] + else: + raise MFCException( + "No MPI launcher found (mpirun or srun). " + "MFC binaries require an MPI launcher.\n" + " On Ubuntu: sudo apt install openmpi-bin\n" + " On macOS: brew install open-mpi" + ) + + failures = [] + for target_name, bin_path in binaries: + if not os.path.isfile(bin_path): + cons.print(f"[yellow]Warning: binary {target_name} not found " + f"at {bin_path} for test {uuid}[/yellow]") + continue + cmd = mpi_cmd + [bin_path] + try: + result = subprocess.run(cmd, check=False, text=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + env=env, cwd=test_dir, timeout=300) + if result.returncode != 0: + failures.append((target_name, result.returncode)) + except subprocess.TimeoutExpired: + failures.append((target_name, "timeout")) + except Exception as exc: + failures.append((target_name, str(exc))) + + return uuid, test_gcda, failures + + +def _prepare_test(case, root_dir: str) -> dict: # pylint: disable=unused-argument + """ + Prepare a test for direct execution: create directory, generate .inp + files, and resolve binary paths. All Python/toolchain overhead happens + here (single-threaded) so the parallel phase is pure subprocess calls. + """ + try: + case.delete_output() + case.create_directory() + except OSError as exc: + cons.print(f"[yellow]Warning: Failed to prepare test directory for " + f"{case.get_uuid()}: {exc}[/yellow]") + + # Lagrange bubble tests need input files generated before running. + if case.params.get("bubbles_lagrange", 'F') == 'T': + try: + input_bubbles_lagrange(case) + except Exception as exc: + cons.print(f"[yellow]Warning: Failed to generate Lagrange bubble input " + f"for {case.get_uuid()}: {exc}[/yellow]") + + test_dir = case.get_dirpath() + input_file = case.to_input_file() + + # Write .inp files directly (no subprocess, no Mako templates). + # Suppress console output from get_inp() to avoid 555×4 messages. + targets = [SYSCHECK, PRE_PROCESS, SIMULATION, POST_PROCESS] + binaries = [] + orig_file = cons.raw.file + cons.raw.file = io.StringIO() + try: + for target in targets: + inp_content = case.get_inp(target) + common.file_write(os.path.join(test_dir, f"{target.name}.inp"), + inp_content) + bin_path = target.get_install_binpath(input_file) + binaries.append((target.name, bin_path)) + finally: + cons.raw.file = orig_file + + return { + "uuid": case.get_uuid(), + "dir": test_dir, + "binaries": binaries, + "ppn": getattr(case, 'ppn', 1), + } + + +def build_coverage_cache( # pylint: disable=unused-argument,too-many-locals,too-many-statements + root_dir: str, cases: list, extra_args: list = None, n_jobs: int = None, +) -> None: + """ + Build the file-level coverage cache by running tests in parallel. + + Phase 0 — Prepare all tests: generate .inp files and resolve binary paths. + This happens single-threaded so the parallel phase has zero Python overhead. + + Phase 1 — Run all tests concurrently. Each worker invokes Fortran binaries + directly (no ``./mfc.sh run``, no shell scripts). Each test's GCOV_PREFIX + points to an isolated directory so .gcda files don't collide. + + Phase 2 — For each test, copy its .gcda tree into the real build directory, + run gcov to collect which .fpp files had coverage, then remove the .gcda files. + + Requires a prior ``--gcov`` build: ``./mfc.sh build --gcov -j 8`` + """ + gcov_bin = find_gcov_binary(root_dir) + gcno_files = find_gcno_files(root_dir) + strip = _compute_gcov_prefix_strip(root_dir) + + if n_jobs is None: + n_jobs = max(os.cpu_count() or 1, 1) + # Cap Phase 1 parallelism: each test spawns MPI processes (~500MB each), + # so too many concurrent tests cause OOM on large nodes. + phase1_jobs = min(n_jobs, 32) + cons.print(f"[bold]Building coverage cache for {len(cases)} tests " + f"({phase1_jobs} test workers, {n_jobs} gcov workers)...[/bold]") + cons.print(f"[dim]Using gcov binary: {gcov_bin}[/dim]") + cons.print(f"[dim]Found {len(gcno_files)} .gcno files[/dim]") + cons.print(f"[dim]GCOV_PREFIX_STRIP={strip}[/dim]") + cons.print() + + # Phase 0: Prepare all tests (single-threaded, ~30s for 555 tests). + cons.print("[bold]Phase 0/2: Preparing tests...[/bold]") + test_infos = [] + for i, case in enumerate(cases): + test_infos.append(_prepare_test(case, root_dir)) + if (i + 1) % 100 == 0 or (i + 1) == len(cases): + cons.print(f" [{i+1:3d}/{len(cases):3d}] prepared") + cons.print() + + gcda_dir = tempfile.mkdtemp(prefix="mfc_gcov_") + try: + # Phase 1: Run all tests in parallel via direct binary invocation. + cons.print("[bold]Phase 1/2: Running tests...[/bold]") + test_results: dict = {} + all_failures: dict = {} + with ThreadPoolExecutor(max_workers=phase1_jobs) as pool: + futures = { + pool.submit(_run_single_test_direct, info, gcda_dir, strip): info + for info in test_infos + } + for i, future in enumerate(as_completed(futures)): + uuid, test_gcda, failures = future.result() + test_results[uuid] = test_gcda + if failures: + all_failures[uuid] = failures + if (i + 1) % 50 == 0 or (i + 1) == len(cases): + cons.print(f" [{i+1:3d}/{len(cases):3d}] tests completed") + + if all_failures: + cons.print() + cons.print(f"[bold yellow]Warning: {len(all_failures)} tests had target failures:[/bold yellow]") + for uuid, fails in sorted(all_failures.items()): + fail_str = ", ".join(f"{t}={rc}" for t, rc in fails) + cons.print(f" [yellow]{uuid}[/yellow]: {fail_str}") + + # Diagnostic: verify .gcda files exist for at least one test. + sample_uuid = next(iter(test_results), None) + if sample_uuid: + sample_gcda = test_results[sample_uuid] + sample_build = os.path.join(sample_gcda, "build") + if os.path.isdir(sample_build): + gcda_count = sum( + 1 for _, _, fns in os.walk(sample_build) + for f in fns if f.endswith(".gcda") + ) + cons.print(f"[dim]Sample test {sample_uuid}: " + f"{gcda_count} .gcda files in {sample_build}[/dim]") + else: + cons.print(f"[yellow]Sample test {sample_uuid}: " + f"no build/ dir in {sample_gcda}[/yellow]") + + # Phase 2: Collect gcov coverage from each test's isolated .gcda directory. + # .gcno files are temporarily copied alongside .gcda files, then removed. + cons.print() + cons.print("[bold]Phase 2/2: Collecting coverage...[/bold]") + cache: dict = {} + completed = 0 + with ThreadPoolExecutor(max_workers=n_jobs) as pool: + futures = { + pool.submit( + _collect_single_test_coverage, + uuid, test_gcda, root_dir, gcov_bin, + ): uuid + for uuid, test_gcda in test_results.items() + } + for future in as_completed(futures): + uuid, coverage = future.result() + cache[uuid] = coverage + completed += 1 + if completed % 50 == 0 or completed == len(cases): + cons.print(f" [{completed:3d}/{len(cases):3d}] tests processed") + finally: + shutil.rmtree(gcda_dir, ignore_errors=True) + + # Sanity check: at least some tests should have non-empty coverage. + tests_with_coverage = sum(1 for v in cache.values() if v) + if tests_with_coverage == 0: + raise MFCException( + "Coverage cache build produced zero coverage for all tests. " + "Check that the build was done with --gcov and gcov is working correctly." + ) + if tests_with_coverage < len(cases) // 2: + cons.print(f"[bold yellow]Warning: Only {tests_with_coverage}/{len(cases)} tests " + f"have coverage data. Cache may be incomplete.[/bold yellow]") + + cases_py_path = Path(root_dir) / "toolchain/mfc/test/cases.py" + cases_hash = hashlib.sha256(cases_py_path.read_bytes()).hexdigest() + gcov_version = _get_gcov_version(gcov_bin) + + cache["_meta"] = { + "created": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "cases_hash": cases_hash, + "gcov_version": gcov_version, + } + + with gzip.open(COVERAGE_CACHE_PATH, "wt", encoding="utf-8") as f: + json.dump(cache, f, indent=2) + + cons.print() + cons.print(f"[bold green]Coverage cache written to {COVERAGE_CACHE_PATH}[/bold green]") + cons.print(f"[dim]Cache has {len(cases)} test entries.[/dim]") + + +def _normalize_cache(cache: dict) -> dict: + """Convert old line-level cache format to file-level if needed. + + Old format: {uuid: {file: [lines], ...}, ...} + New format: {uuid: [file, ...], ...} + """ + return { + k: (sorted(v.keys()) if k != "_meta" and isinstance(v, dict) else v) + for k, v in cache.items() + } + + +def load_coverage_cache(root_dir: str) -> Optional[dict]: + """ + Load the coverage cache, returning None if missing or stale. + + Staleness is detected by comparing the SHA256 of cases.py at cache-build time + against the current cases.py. Auto-converts old line-level format if needed. + """ + if not COVERAGE_CACHE_PATH.exists(): + return None + + try: + with gzip.open(COVERAGE_CACHE_PATH, "rt", encoding="utf-8") as f: + cache = json.load(f) + except (OSError, gzip.BadGzipFile, json.JSONDecodeError, UnicodeDecodeError): + cons.print("[yellow]Warning: Coverage cache is unreadable or corrupt.[/yellow]") + return None + + cases_py = Path(root_dir) / "toolchain/mfc/test/cases.py" + current_hash = hashlib.sha256(cases_py.read_bytes()).hexdigest() + stored_hash = cache.get("_meta", {}).get("cases_hash", "") + + if current_hash != stored_hash: + cons.print("[yellow]Warning: Coverage cache is stale (cases.py changed).[/yellow]") + return None + + return _normalize_cache(cache) + + +def _parse_diff_files(diff_text: str) -> set: + """ + Parse ``git diff --name-only`` output and return the set of changed file paths. + """ + return {f for f in diff_text.strip().splitlines() if f} + + +def get_changed_files(root_dir: str, compare_branch: str = "master") -> Optional[set]: + """ + Return the set of files changed in this branch relative to the merge-base + with compare_branch, or None on git failure. + + Uses merge-base (not master tip) so that unrelated master advances don't + appear as "your changes." + """ + # Try local branch first, then origin/ remote ref (CI shallow clones). + for ref in [compare_branch, f"origin/{compare_branch}"]: + merge_base_result = subprocess.run( + ["git", "merge-base", ref, "HEAD"], + capture_output=True, text=True, cwd=root_dir, timeout=30, check=False + ) + if merge_base_result.returncode == 0: + break + else: + return None + merge_base = merge_base_result.stdout.strip() + if not merge_base: + return None + + diff_result = subprocess.run( + ["git", "diff", merge_base, "HEAD", "--name-only", "--no-color"], + capture_output=True, text=True, cwd=root_dir, timeout=30, check=False + ) + if diff_result.returncode != 0: + return None + + return _parse_diff_files(diff_result.stdout) + + +def should_run_all_tests(changed_files: set) -> bool: + """ + Return True if any changed file is in ALWAYS_RUN_ALL or under + ALWAYS_RUN_ALL_PREFIXES. + + GPU macro files, Fypp includes, and build system files cannot be + correctly analyzed by CPU coverage — changes to them must always + trigger the full test suite. + """ + if changed_files & ALWAYS_RUN_ALL: + return True + return any(f.startswith(ALWAYS_RUN_ALL_PREFIXES) for f in changed_files) + + +def filter_tests_by_coverage( + cases: list, coverage_cache: dict, changed_files: set +) -> tuple: + """ + Filter test cases to only those whose covered files overlap with changed files. + + Returns (cases_to_run, skipped_cases). + + Conservative behavior: + - Test not in cache (newly added) -> include it + - No changed .fpp files -> skip all tests + - Test has incomplete coverage (no simulation files recorded but simulation + files changed) -> include it (cache build likely failed for this test) + """ + changed_fpp = {f for f in changed_files if f.endswith(".fpp")} + if not changed_fpp: + return [], list(cases) + + changed_sim = any(f.startswith("src/simulation/") for f in changed_fpp) + + to_run = [] + skipped = [] + n_not_in_cache = 0 + n_no_sim_coverage = 0 + + for case in cases: + uuid = case.get_uuid() + test_files = coverage_cache.get(uuid) + + if test_files is None: + # Test not in cache (e.g., newly added) -> conservative: include + to_run.append(case) + n_not_in_cache += 1 + continue + + test_file_set = set(test_files) + + # If simulation files changed but this test has no simulation coverage, + # include it conservatively — the cache build likely failed for this test. + if changed_sim and not any(f.startswith("src/simulation/") for f in test_file_set): + to_run.append(case) + n_no_sim_coverage += 1 + continue + + if test_file_set & changed_fpp: + to_run.append(case) + else: + skipped.append(case) + + if n_not_in_cache: + cons.print(f"[dim] {n_not_in_cache} test(s) included conservatively " + f"(not in cache)[/dim]") + if n_no_sim_coverage: + cons.print(f"[dim] {n_no_sim_coverage} test(s) included conservatively " + f"(missing sim coverage)[/dim]") + + return to_run, skipped diff --git a/toolchain/mfc/test/test.py b/toolchain/mfc/test/test.py index 2193e677b4..3f857b59ca 100644 --- a/toolchain/mfc/test/test.py +++ b/toolchain/mfc/test/test.py @@ -76,7 +76,7 @@ def is_uuid(term): return cases, skipped_cases -# pylint: disable=too-many-branches, too-many-statements, trailing-whitespace +# pylint: disable=too-many-branches,too-many-locals,too-many-statements,trailing-whitespace def __filter(cases_) -> typing.List[TestCase]: cases = cases_[:] selected_cases = [] @@ -108,6 +108,53 @@ def __filter(cases_) -> typing.List[TestCase]: f"Specified: {ARG('only')}. Check that UUIDs/names are valid." ) + # --only-changes: filter based on file-level gcov coverage + if ARG("only_changes"): + from .coverage import ( # pylint: disable=import-outside-toplevel + load_coverage_cache, get_changed_files, + should_run_all_tests, filter_tests_by_coverage, + ) + + cache = load_coverage_cache(common.MFC_ROOT_DIR) + if cache is None: + cons.print("[yellow]Coverage cache missing or stale.[/yellow]") + cons.print("[yellow]Run: ./mfc.sh build --gcov -j 8 && ./mfc.sh test --build-coverage-cache[/yellow]") + cons.print("[yellow]Falling back to full test suite.[/yellow]") + else: + changed_files = get_changed_files(common.MFC_ROOT_DIR, ARG("changes_branch")) + + if changed_files is None: + cons.print("[yellow]git diff failed — falling back to full test suite.[/yellow]") + elif should_run_all_tests(changed_files): + cons.print() + cons.print("[bold cyan]Coverage Change Analysis[/bold cyan]") + cons.print("-" * 50) + cons.print("[yellow]Infrastructure or macro file changed — running full test suite.[/yellow]") + cons.print("-" * 50) + else: + changed_fpp = {f for f in changed_files if f.endswith(".fpp")} + if not changed_fpp: + cons.print() + cons.print("[bold cyan]Coverage Change Analysis[/bold cyan]") + cons.print("-" * 50) + cons.print("[green]No .fpp source changes detected — skipping all tests.[/green]") + cons.print("-" * 50) + cons.print() + skipped_cases += cases + cases = [] + else: + cons.print() + cons.print("[bold cyan]Coverage Change Analysis[/bold cyan]") + cons.print("-" * 50) + for fpp_file in sorted(changed_fpp): + cons.print(f" [green]*[/green] {fpp_file}") + + cases, new_skipped = filter_tests_by_coverage(cases, cache, changed_files) + skipped_cases += new_skipped + cons.print(f"\n[bold]Tests to run: {len(cases)} / {len(cases) + len(new_skipped)}[/bold]") + cons.print("-" * 50) + cons.print() + for case in cases[:]: if case.ppn > 1 and not ARG("mpi"): cases.remove(case) @@ -176,6 +223,24 @@ def test(): return + if ARG("build_coverage_cache"): + from .coverage import build_coverage_cache # pylint: disable=import-outside-toplevel + all_cases = [b.to_case() for b in cases] + + # Build all unique slugs (Chemistry, case-optimization, etc.) so every + # test has a compatible binary when run with --no-build. + codes = [PRE_PROCESS, SIMULATION, POST_PROCESS] + unique_builds = set() + for case, code in itertools.product(all_cases, codes): + slug = code.get_slug(case.to_input_file()) + if slug not in unique_builds: + build(code, case.to_input_file()) + unique_builds.add(slug) + + build_coverage_cache(common.MFC_ROOT_DIR, all_cases, + extra_args=ARG("--"), n_jobs=int(ARG("jobs"))) + return + cases, skipped_cases = __filter(cases) cases = [ _.to_case() for _ in cases ] total_test_count = len(cases) diff --git a/toolchain/mfc/test/test_coverage_cache.json.gz b/toolchain/mfc/test/test_coverage_cache.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..ae0a6d3c0aa856bc73d1a16b6019f3406d9fbc09 GIT binary patch literal 11779 zcmYjXWmpv8(gj39>F#c%Lqb8iyBijkmhP7BMp(KVM7pGv?rxFpZh>zXy!U?3%n$Y# zyYD+Q=gc`TX#@fS-BF$e%##z)(aFHr)&=NbWCk=aHZnE`GFv*@+CUvF%h(c0#s5CN z_X-!za{11o_lV?roF;fZ(XhN|C`6SIrwq6$l~w-OW!QEPSC)@af&(&P2#AXjgu`=6 z5`HJ(f5pM^-u3C5@9k!CJA#ol;_%a^ZTsK*lhN*W-gX0h{lCp^Y2KRpzBkTW{D=Mx z(vQupnFcEc7ws!6<;Np78t>12empu0=v@2iEv-Cm8a&-B?oa=ny&ayO9=^;PH5lG1 zOS|Mi)yv3OT{18jQqLNFym>z=@cwCT^eNBav&Vt4ulwRR?!Tw=x7I(>1@FhE4;?0b zwj?`wveidU%h!f`YbgHg0O_u4dN_ECMc`E?hc~ldk4JTASO` zmzhYP{0zLdf{sTfx4FG-Ge7(|cW-_0 zX?nYDP=TabNB^=r;o9eeS#}f4Bb9yp61_J`N384vLtuejV>PqzV z%DiC;Gjv>lsEPX{oCMi6TL0}r!jA*N%jEq>n`z>Wi(Ps>iXLcwjB#6X7435gg1KCK= zIfO8vNuIzw#ZDK%PFKULJMNx$UcVY4&y7_={E8qn*gUj;Iuj-tSjLMfj6fcUCX7&l ztIa}5RmyC2Xu{L9?t*C78%L?7BC~e5d01d3HX5Eetxy0%J*7(Zn)dZJKX&@+%12vg z&$Ais$4qU?*aDLs2KN@`g{n~^{I0m&}xdW)oN4B*`o4f`?Xlnqn^Kg0J||AW)MN8 z?DRVmwBHDJu#J<}JEF@azf)d&W0=`8-^pOuM`uZJMQ3f_xKnv&>3}6L_Xf!UYR1K zpso`7pEL^HC`B@fbA862q8O%zMAy+aj2pweM)$*hZJ?&xr}pdVV>C}JszK~wGq%hE zSxh$S=b6)vFl!^Pat_l+blrsY<@KY*%QGn#$NOf0ahBJa^pE$AT`f<0uD(UbyD?Ws z(n`usdKA7d0l19RHI39QRNbdkdYewKdqT-$`NX3N(0v>gk_(}XEGg(-2Al)#sv zi7h; z+ijA%%nuFW3mSq&xuEx+&&T$joG4FSIXiCm5DGef3c9mx)H9vHDs`my*dIKbi_(WL zyvB7t3dSO^eQ>GMU9Awn6bU8oL1S0CBZeP8#$y&87IF}K5BZ691Dr&114?EY_VN79 zYoTExCxAb(VcjauV~iG!f8pWj2VmL9Qh+q! zd0R%O9O>IAdHKArsr@8TV+c`Wp{ejdam8N}w8nsa3dlcd6$ZUhGPyN!j>iSwZPYOY zxygSp!-?J*eMeMdm)>@*dzc_C?=)8xzX1QRQ$;>bHvo&RJWRCj|G!SjV4*Qf6vBT_ z5PG5%01vn7Wejw83fgfpNl4!WgCu9; ztF+i8-^;OtNguVhpM9@c>vj8B4uBy?J0cRMGy!m8K&ofHB2YZR600>jv!!|*a!FgH z1CcqA@w~7snIprtcQW0P(O1O#%44u{;9ULa+@5LRKRddZAJVOyV9ol91Dt6f-ozFu zf%2=XGuMZzobs) zZziq@6|v8{+h6z+-;-7I64-O|&E6kgPA?DO z=IA+ru_J_z;F^_W;f)ZQ7VQxO#qU^QOyQmGu@HRf3gmjar1^trNICL<$Rv&ay>mz; zR_5e2gf8MO+7&=QpEq2Ah%!wBq5x3@Dpn^@M#WP)E4wBU+e3OJfITu=feRBoLA{f0 zGelG}P*hU%2S+$NHA?cd(lG?uhGI~pu&u7s+5K zz?s(YZbjEiKxe6Zi%^(vZp??Yh%H`PO<15sL~_K`qiv@mlQdMb+gM}pa(R!V(1@T= z3D0O+h;v5E4@oN80G^7_Mr0zd&Vkq3H51|>M8VbY1<2&J`y852+(`-1)>|cA(1(P_&YI2KoI-8=ne{fGDZk)X;vP`{AJTbJ` zwb%!$FrXz=998{i;*t<}{3&^IcZ9%QIxbvDRw4e}w7iBuoQM|Rj{)hKR}C23{0LlejnJOmS0JlJhl1ToOR#XQe+#+ z=&An&F*W_6Ds7M`exi{NY{J!8{+g@J4fC~G@BOK>_g27XkQJ zpDbpLsMROZ+<|nhd^sz_gu-EJ+F!St-UYRFJq|-H%Ij3vp=|Gbz#!MR@7*2#xn1*f z*s_h>m8Sw_bNa!S&3JD4=%B5%NUjwcCf~~)$ z81N_ZMxd5?zjG~&HI~8CR&8=9Lxu3uRzQCmLlE%G4-)k&^?eGSKJ@VwncCiC{m>|p zF!$@e!oI?}X0P($rL_pnp{<8`J>MTI7hF?&&|2O{e#Xy$+pI;bG@CX5cT(Q!`@OE^X*E94~SoOJZ zR^wX$)&nU`M7T}z(n8qt=q4?P*dcxPoUQ60+fWnoc+}UTpJ1QUBv84)oZ8al1nPOd zpQY7&jP|c9b&~)32NA+VUHS<0JI8T|@X59R{Dy_pH=va;ie|c)fJt?0!32gpfl?d9K3bOaCQQ*>jT9j0BQ8^ z7H^|vU38DuwbvWQ@&XUir2R&JuoUoE_fUm88GaR|F9RdOD~YQ=?SVR0+dkLsdPIfL?FJ zObGZO(-KVU?X^&nHxFEwHykUw=tV_1!8wwa)0SG@#=tGZ1i5E~sGUDK;B5eQQ`tC= z339QKcJH~FgiO-J>*+r42%CN*ZYS{kfI|voAh$Pzxy1#?tm^ITf5_+&%WS??zEw-` z*nBn+9>PUGNwyFIe<{*@J%}>%#e4q7{9RBQY4(}?=aPB6aX~ZZ5jhl1U+aKrbUFUv zX@QC0aEc7%bf>qbCKtmWnfsOR%45urS|%xiFWW)>8_f=Z$Fp_&xcZ56NcqAQ-VWo@ zQLytB59^aE=J66fkmUHl@G+@{hQ?aCZObr(Y1N5kI(91VH12e~k18ZDuD`zwGlNW3 zS5a?kj|w~)!O3v1!FY}r_vf?6m7&^y7lBZue~Z8iNV7VkA#eQgCX&y|OZg9BVoOBi z=Cl8>&KD7~I{p8U78)@Y3Nf}GZg+iqF%23NVr<~I|D)MXtyVQ=hduP#Wp^vLU6g4{ zi7nb)bxVojti3`jkhBq9u;L*NO%&DlZ){4wC>*!_C)g(&PNg; zV?N%9*zY527!(U8a+5MNii}DH?jQE?^tvPyEsuw4>OPIIKdJSBFsIOPfRDPI(+gp% z9G44eOY+SRXN$Mf(})MX4b9SeRPYhX%Zkf>dt0GDxw7y5-rnD3;STo|i>-aN7ESx3 z`hd(*gw5^3%~8_lDQhZj`iti^<^?>P3tNYpx8>LJH;`JvJMz4n0?)=U5q?8)-fwCF z_$gl?84h9#$1kg!h;+*Rw_nl?G*+tFsDP)n5J)4iY`(j-r)tlh`!6B~&(QA!CaVf5 zTJSso=gQ?&o*Zce#r_6oc=ypj^gZk|sZu8alPXzS(Opi@H;Ta`A*^AJM!dLRA6!SZ z@zdm|9fT2nc03XrOcrcJ6uHY$NVNDN2(|dRbmGy+a%3D?dS_wnd}=Vnd>a;I%}QhX zh|lpOlE$%;|G!8^H&9!eXyVHTsd#JWV5chixdWcAD07io@`c#R2_xRfrT0`Ny42TV zkhO#FU1X>n7sXeTD(&%o32+7FE@wraG*4naJ`VJb1e2~`vZ@XW>Dpg5B;@S2^?fgU2s=l`h*56ga%8G-Fe3QPG&pdcP|x%!Tea?f4dZ9_r?nVFk~L4Bk4RAw9>4KCD2Q3LbVzL#y@*<9jRLA|Y_ zi?G`vBO-`q-LWowCk#3C;k9sRfsTdy(@V?pl2mMJ#WW}`z+9Q}TuW~nQ@50QvsEBw z7a}h^FMC5`Je0!xcMA$w(nc3p$Te%?egf}~ry~VKknI-^SnNJ#&KJeE#>c|sYMO`c zYHEz@YU*gl9kDv9vs=p%#a}~Da3yd;BDrsIYg89gwygqN&t$Az@e4uf@v*+j_1^xb zD`eEdqHD-g<(&DYtuew{hJlzmvGdVc_ zTF+r?b7RxQD0Ir&byDeq#JtkyHp(|&PF|WV*pFNG9ozQ{2f&_g*tnxFXO4eQVozcn z^Vu+)hvP`vJ5{k~3N?>cIX7DOD$<9)y%|tv@oqe~qUkn)P+|Sr>TK_EQD&#$K$Wb5 z;yH5N%x+9eo#^^C%Yi=;!@z49-3k2(sy=6dt`*KW8uB<=)WY7b%Ju#&_4F)C%J_GJ zrvvhl{~}=u{y!xAf)MJ55Ndu#+!H9z-fuDbUFAQ;8q6-X6{`lGkVeK6j4@@6fb|Jf zkuY{C>AUkl3=H+>m|?@1(Jn`eZ=y+DgA$Pa10{g+n_mt-t-gszew}7qu!mnw`yzMh z6OIIq#5X@Q;ctFgZEV^TOs=muUxx7_u(X@CN~+uv{aKG*L7nhF@K; zcfPr3ROF5yLGhP`typeP1Qpl8+xZjPDZm)F5>ug=ML`}1f=<Ym+HD{`N zQ3S}=_S*bxOzCWls(j1T`oClhKQBZ-B;ntob~gbi(;2s9&ATgM;L;7@Obty#tU6_d zl|IR7f6_7t+~X00g~$kW*${E6dC8fNHxXpH_#R6L9g@6p)l6{VbZ(IL4aUx6Q;$qX z?O#BE5GmH0Vdzx>6pq??yLDPy7Egoud+E>ZYyh|tz}eXb#g@V_iFG<-b`-Zz$?0K1 zbPj02TvJI(Q7SgVeu|QQCXrrUx%!+v51k+fo#4BdKUjR5Y*J@Pw#$QTUq{!ARmods zQ+J1d_ukCZ8x2c+6Aj?f1ZSRGt{Z^}NVoc%b^ZUqw7-;VT4@l%CWkVkLc7NM?Kje7 z**egl_mzum@V}OL zL%Nk;8N@eR%9!*#hrAV+$o5C$uA4HNWUM5T1T*Qydc9OgP1sECFeps!+Gn*JS#|7h zR6qnjrOSH;0tAk6RoI+aWG{t-d1Ql(7z!sAC8X-JL}mgBU;T(z4~rf~v4fSc>!D_X zFCHhS!{nd*Fwm+5T8!z&74VX;*+>0|_5C>7+u2}UDr^Au526+$BtmGsuOZ+pb-l3- z8fTES-I?x28tNF@GVh_5)1Bb#+&ibNayl7t>}N7$t9osV^qwQsxf#oaqKsLH980<7 z%c7Ol`#t`lh1E5qL(wT}yI_RJHPT16xz{!!Y*2+(gW6*$q7XEM4cJ-Bz#laoKd;&VcBl$(EXLv@oTwgoOE>>h3dys*dt%q47+FROVy!%#B; zK%vJ)_i8R*v=gTs>OcyKwY$CErxj>;`+hul&#Kqq6(TkwcE!^= zVZUU?+h2A;u(w!1{(&mww^diZ`*s8LGhKM(rgG!2=3*6rT1lnl0VfVe>w~+1#8eSn zH{SlM{;OU3aTS8RB2`gmwXb=WsmDjfzPV&dv2<=QN>ycD1Jael84Oe#im+meocLmG z&f?<<`G?)#CsTU9PZIGDqi2G-N77XVc&k*7%OS_*x~nr@ziKA)aGfhZai;ni#|y8ABuR)H-dc7ckJ=?8 z)Ik+QJFs}agbv4(7lmr=LK|helxBYn#@*!V$Z$oW;_N>(5d1zgivA4 zJk5kp+%r^~32wZq8-Rj!ZN--@2~ga}hb)tlq{J-E^&E8>u+y=s@}L@!SP=>-E5QpvwjpPI>it64{GRx$dUg$ib3h6 zTu#%9!M7w^p881GYMaH*62)Yyv9a;sATbl>^Foi9Zxw9W|2CV}7Ha?Aa3%*Hz(vRk z4Q~epZ@1$YISySykEelwSr675!HLvEf}jgDUt;uixIgro6$Uvv-8|hqUrBs!Vf-1V>b4hA62CG9zPx+&&zvk*rIg~UI#=Pt zQCF5X+X1YJhpnDes#L0phlz)jCOk@1-&(1=g+Pex>3r#XJB|8Lt}#d759`nvGRyXM|O0Q(@}mqglDC zuxm!46PUsDK!(?3Ieya6u#}}ZQGQUE^$-a}RNCRw-Pf994}6{m5I@G)D>}ol3mXEv z5ajiNbFHnk9A|(q2){m4Q54v)&5&sUl2q>3NQ-K?-2G!*%w|x**Qu*tfLX~M6sxFu`xqV^_WHUU!!?w@hqzj&dQqu* zG2SV8Hz#`t>$=$5gV)L|V;ju0s9saN_E<$WO%uQ=nJ6icmgl0S>wDfY{D~mMdoY%0 zmgJ?ny`JZ1n26x0ENd6A8xfb)(QxR4hi#iEP`8V^G;6)s2hjKG1M*kHuswKSJ$`*e zQ-2~@{d*o@vBmMl(Q(Du1K%Cd?+pF1}Trk>>Z`pBqHrCy~IIDjUqFsiR;>N>! zq!J-eA^P@cBk8tQSI~rrzx}YtsgAB!6dEG+LLqyONz4c62TcrA&7+!vnFO~)-g#QY zs98kXw3os zuzt*2Fs3Q{%t}U3gYqnROSK3x))1m;R{Es97LM7n$yIB~9XE@OD&N!mp#D)1COpHZ zHB;)WR&QJeqs91qdqZ>sJt-nr|DkWK+oI>0&3x!wmnbjZ$K?p#KuWxbIrP7tP}~~6 z7Nud7Mvm9RdeNoH*u_f#!&MQvc6f}?HY~4}3L6^2mb9Xq$t5TxC<3jqh107kv~3Eh zp37}92u-j0t!|w>0?1>3cK^C@0x9JTr+WjC9Fii`M`mItPRsko%*jEVmLqL6e|E)` zP$dMsO9e&y+&hn3W-Enkw?5sRLJQ-YNRkfdx}M{3{2jN$G`%_)Z8G zOQ2EY8z7BV`oe{SDRzeOr;y|XtWZdpF4GM+E7!)wTR9tn`S{i%#4RkQpg(n+PU$4p zA5r0(FNt>@_gJuBP;<5srk`MjgvolP=o}wHz3xt_i9c5T+(@^(JWFu6Y4eQMGo-A= zuQ}k+hwTT3S)CsmWy3^`nLmkpYe$aUO)e66$09!X*jRX9Cmhy$OxDB>)CoWNv9|ge z@kETIq4AalCntm_g(ro{FpQ%!jK52lRbu#60(QW`xGY6tm{~AY$o(rzvQDZl4k1jk zZq?_Np1K%s7Si(=M&PI|c0xcVTjts&|^s^o`FYOJDy@y65m>q}eBiS_2~i`&_8*S0w5hF6!W%Lhd^BtyY_~Nu;GeAGbn$%H`e^%~ zgLf$E=vUw}vW~C@Oh{ACxR7P-2j5qcbIyG(dRJL@gU-ILP(=$LuQ5OPbszIsthqDt z8(EFuGk%Z9yx;Y1!(vDeV_Jw#DA{4DN=nOi>OzKlUk;7@Brzd6A=*M1&7%cdkK$jE zt&1-E&^hzv`{mA$TZ~QnXd@?0KAf+m?^1C%-zmxEAPrRE@l6AYBlPxQy+1-^M2{fR z5C>WakTF}3jjb`(*{EPqWQRs>EdRKf7W@!KEP4M#wAv@y_FCP=sROHl?6l<$sAj|)=Ez1xl zUDUeK0R{OLe{?PTQftLDlGOFx>(kK?`}+~?B=COgV$in~LEEewyrC8we#DG-igz+! zFp7U%*dNZZ8QLFJnCY4;pHBY?V^sxbMWo(ul;qo?7|1&Fv8@OlGO_)>Od6Qa|8NB< z=i5-zGZR0r&bESm66h50$#2v0!_Y8AL*=~a=Vd1kJl0OCe|QII$+$UVn%!+h&Xh=s zIKmxBhDe3@CZZ5#I1L0L2tC_*DfC6Y{J^J~z<3?g8YUv8MdigdJcjaExp`>_KWmC_ zI?~u02!z2uikaVwtR1fEc?DU(t@Xq}_K9rj_~De0x|}Df9Qn^v2V^$VD4r$jw807K zy0R3A*l9^NEHKG(TF4%~s@*#hW+X ziZU{ltFPb;q2QYFU(DW#<2h&!^8E#_yE;)G3ZjRF^|aWNT>AMqi3D)u4KdPC@%W^z zH(Yx-kIO9YW=AssSC{(*M0L+R&h9Y#_LIuSslb1NBk{H~PNK5}>-Bplbn;L%QG_$a zvoW@_F-2f0{(o=;you#4v(OKkI>A#`*S^NGmKQtRs@mXZG|5 z0iaw!uTn76Tq^hb!&{iO@mkh)S>;xng+IsNybi2y*z~WHdkg_5i_vc1ge4h2O0;4E zuGxFq(#ceFtdlX%?-D+b6%`f~7E~x?I-S*7Vv|d&;q4VU4sf9(t+f*|q(*j=kU@Ao z#0g02(ilL~Hq7{n<`7E`ehL*#C)_a~;y0;9%dR2=n;tM6%ubfP7yS$IB}c^`!sPCaLcEc&l^q3c*0fAEplk>X{Y&Q*;totFg1Q*zY??g|+fVkmb z2gLSYb|V*!3orb)bNxF|)#Qtdz)9XTqB!Jvym`US@gj-1>{1L4%v(_y2i@mbGVZsJ z3S%&L(1kd%>&m>$N%wC8uC=XDUrL}~q~Gx!wE}t?kAFqY%c9UTfT7@WzTde<{0Pu+>1g|!&wJc@*L*s@gj+N227c2gLJ-cq+#hRx} zsFca%-o9}GO*lA%epvMg|Ao?VTDN&Zo?=lYJ>1Gp_ZpQ-7jCD-)qpZtJj%A=yM|8Y zrX9Q+?vS4Apl6^{$Vs-#4mIp_iQ)N46VZ4VI+KBs-OO%01an>%g_Sl>J1AjQ7gYae z$=a`v=VSSp@&r-#i&_-`wy{;jO225s0Sgy6j88h6bhHh`#Az`Y~yI= zW?*MsXm=4hlMa(VbvSc~r;nrm4sh5jxG&ng%b~n@-1L@aaH3XT3BWP`g7eWP`@;_s zS4;hy9uNokT|l2WZ!q|L6P(KesKDoq>6GtF!?Bv;jCp+J$>B~?DdrkROb|$dq5B+5 zY#2*_0uqJ(8E#80RV_RQVEy7j~9nr#qT^IPK9 z-BDbCsRyGeZnHr;JX2zlx9xV#c@Els)NY-@yGXsE;h(y0q|6?atq*rff9p?%;B4VU&|vw{U2 str: + return self._uuid + + +# =========================================================================== +# Group 1: _parse_diff_files — git diff --name-only parsing +# =========================================================================== + +class TestParseDiffFiles(unittest.TestCase): + + def test_parse_single_file(self): + result = _parse_diff_files("src/simulation/m_rhs.fpp\n") + assert result == {"src/simulation/m_rhs.fpp"} + + def test_parse_multiple_files(self): + text = "src/simulation/m_rhs.fpp\nsrc/simulation/m_weno.fpp\nREADME.md\n" + result = _parse_diff_files(text) + assert result == { + "src/simulation/m_rhs.fpp", + "src/simulation/m_weno.fpp", + "README.md", + } + + def test_parse_empty(self): + assert _parse_diff_files("") == set() + assert _parse_diff_files("\n") == set() + + def test_parse_ignores_blank_lines(self): + text = "src/simulation/m_rhs.fpp\n\n\nsrc/simulation/m_weno.fpp\n" + result = _parse_diff_files(text) + assert result == {"src/simulation/m_rhs.fpp", "src/simulation/m_weno.fpp"} + + def test_parse_mixed_extensions(self): + text = "src/simulation/m_rhs.fpp\ntoolchain/mfc/test/cases.py\nCMakeLists.txt\n" + result = _parse_diff_files(text) + assert len(result) == 3 + assert "toolchain/mfc/test/cases.py" in result + assert "CMakeLists.txt" in result + + +# =========================================================================== +# Group 2: should_run_all_tests — ALWAYS_RUN_ALL detection +# =========================================================================== + +class TestShouldRunAllTests(unittest.TestCase): + + def test_parallel_macros_triggers_all(self): + assert should_run_all_tests( + {"src/common/include/parallel_macros.fpp"} + ) is True + + def test_acc_macros_triggers_all(self): + assert should_run_all_tests( + {"src/common/include/acc_macros.fpp"} + ) is True + + def test_omp_macros_triggers_all(self): + assert should_run_all_tests( + {"src/common/include/omp_macros.fpp"} + ) is True + + def test_shared_parallel_macros_triggers_all(self): + assert should_run_all_tests( + {"src/common/include/shared_parallel_macros.fpp"} + ) is True + + def test_macros_fpp_triggers_all(self): + assert should_run_all_tests( + {"src/common/include/macros.fpp"} + ) is True + + def test_cases_py_triggers_all(self): + assert should_run_all_tests( + {"toolchain/mfc/test/cases.py"} + ) is True + + def test_case_py_triggers_all(self): + assert should_run_all_tests( + {"toolchain/mfc/test/case.py"} + ) is True + + def test_definitions_py_triggers_all(self): + assert should_run_all_tests( + {"toolchain/mfc/params/definitions.py"} + ) is True + + def test_input_py_triggers_all(self): + assert should_run_all_tests( + {"toolchain/mfc/run/input.py"} + ) is True + + def test_case_validator_triggers_all(self): + assert should_run_all_tests( + {"toolchain/mfc/case_validator.py"} + ) is True + + def test_cmakelists_does_not_trigger_all(self): + assert should_run_all_tests({"CMakeLists.txt"}) is False + + def test_case_fpp_triggers_all(self): + assert should_run_all_tests( + {"src/common/include/case.fpp"} + ) is True + + def test_coverage_py_triggers_all(self): + assert should_run_all_tests( + {"toolchain/mfc/test/coverage.py"} + ) is True + + def test_cmake_dir_triggers_all(self): + assert should_run_all_tests( + {"toolchain/cmake/FindFFTW.cmake"} + ) is True + + def test_cmake_subdir_triggers_all(self): + assert should_run_all_tests( + {"toolchain/cmake/some/nested/file.cmake"} + ) is True + + def test_simulation_module_does_not_trigger_all(self): + assert should_run_all_tests( + {"src/simulation/m_rhs.fpp"} + ) is False + + def test_empty_set_does_not_trigger_all(self): + assert should_run_all_tests(set()) is False + + def test_mixed_one_trigger_fires_all(self): + assert should_run_all_tests({ + "src/simulation/m_rhs.fpp", + "src/common/include/macros.fpp", + }) is True + + +# =========================================================================== +# Group 3: filter_tests_by_coverage — core file-level selection logic +# =========================================================================== + +class TestFilterTestsByCoverage(unittest.TestCase): + + def test_file_overlap_includes_test(self): + cache = {"AAAA0001": ["src/simulation/m_rhs.fpp", "src/simulation/m_weno.fpp"]} + changed = {"src/simulation/m_rhs.fpp"} + cases = [FakeCase("AAAA0001")] + to_run, skipped = filter_tests_by_coverage(cases, cache, changed) + assert len(to_run) == 1 + assert len(skipped) == 0 + + def test_no_file_overlap_skips_test(self): + cache = {"AAAA0001": ["src/simulation/m_rhs.fpp"]} + changed = {"src/simulation/m_weno.fpp"} + cases = [FakeCase("AAAA0001")] + to_run, skipped = filter_tests_by_coverage(cases, cache, changed) + assert len(to_run) == 0 + assert len(skipped) == 1 + + def test_uuid_not_in_cache_is_conservative(self): + """Newly added test not in cache -> include it (conservative).""" + cache = {} + changed = {"src/simulation/m_rhs.fpp"} + to_run, _ = filter_tests_by_coverage([FakeCase("NEWTEST1")], cache, changed) + assert len(to_run) == 1 + + def test_no_fpp_changes_skips_all(self): + """Only non-.fpp files changed -> skip all tests.""" + cache = {"AAAA0001": ["src/simulation/m_rhs.fpp"]} + changed = {"toolchain/setup.py", "README.md"} + cases = [FakeCase("AAAA0001")] + to_run, skipped = filter_tests_by_coverage(cases, cache, changed) + assert len(to_run) == 0 + assert len(skipped) == 1 + + def test_empty_changed_files_skips_all(self): + cache = {"AAAA0001": ["src/simulation/m_rhs.fpp"]} + changed = set() + to_run, skipped = filter_tests_by_coverage([FakeCase("AAAA0001")], cache, changed) + assert len(to_run) == 0 + assert len(skipped) == 1 + + def test_multiple_tests_partial_selection(self): + """Only the test covering the changed file should run.""" + cache = { + "TEST_A": ["src/simulation/m_rhs.fpp", "src/simulation/m_weno.fpp"], + "TEST_B": ["src/simulation/m_bubbles.fpp"], + "TEST_C": ["src/simulation/m_rhs.fpp"], + } + changed = {"src/simulation/m_bubbles.fpp"} + cases = [FakeCase("TEST_A"), FakeCase("TEST_B"), FakeCase("TEST_C")] + to_run, skipped = filter_tests_by_coverage(cases, cache, changed) + uuids_run = {c.get_uuid() for c in to_run} + assert uuids_run == {"TEST_B"} + assert len(skipped) == 2 + + def test_multiple_changed_files_union(self): + """Changing multiple files includes any test that covers any of them.""" + cache = { + "TEST_A": ["src/simulation/m_rhs.fpp"], + "TEST_B": ["src/simulation/m_weno.fpp"], + "TEST_C": ["src/simulation/m_bubbles.fpp"], + } + changed = {"src/simulation/m_rhs.fpp", "src/simulation/m_weno.fpp"} + cases = [FakeCase("TEST_A"), FakeCase("TEST_B"), FakeCase("TEST_C")] + to_run, skipped = filter_tests_by_coverage(cases, cache, changed) + uuids_run = {c.get_uuid() for c in to_run} + assert uuids_run == {"TEST_A", "TEST_B"} + assert len(skipped) == 1 + + def test_test_covering_multiple_files_matched_via_second(self): + """Test matched because m_weno.fpp (its second covered file) was changed.""" + cache = {"AAAA0001": ["src/simulation/m_rhs.fpp", "src/simulation/m_weno.fpp"]} + changed = {"src/simulation/m_weno.fpp"} + to_run, _ = filter_tests_by_coverage([FakeCase("AAAA0001")], cache, changed) + assert len(to_run) == 1 + + def test_empty_cache_runs_all_conservatively(self): + """Empty coverage cache -> all tests included (conservative).""" + cache = {} + changed = {"src/simulation/m_rhs.fpp"} + cases = [FakeCase("T1"), FakeCase("T2"), FakeCase("T3")] + to_run, skipped = filter_tests_by_coverage(cases, cache, changed) + assert len(to_run) == 3 + assert len(skipped) == 0 + + def test_mixed_fpp_and_nonfpp_changes(self): + """Non-.fpp files in changed set are ignored for matching.""" + cache = {"TEST_A": ["src/simulation/m_rhs.fpp"]} + changed = {"src/simulation/m_rhs.fpp", "README.md", "toolchain/setup.py"} + to_run, _ = filter_tests_by_coverage([FakeCase("TEST_A")], cache, changed) + assert len(to_run) == 1 + + def test_incomplete_coverage_included_conservatively(self): + """Test with no simulation coverage but simulation file changed -> include.""" + cache = { + "GOOD_T": ["src/simulation/m_rhs.fpp", "src/pre_process/m_start_up.fpp"], + "BAD_T": ["src/pre_process/m_start_up.fpp", "src/common/m_helper.fpp"], + } + changed = {"src/simulation/m_rhs.fpp"} + cases = [FakeCase("GOOD_T"), FakeCase("BAD_T")] + to_run, skipped = filter_tests_by_coverage(cases, cache, changed) + uuids_run = {c.get_uuid() for c in to_run} + assert "GOOD_T" in uuids_run # direct file overlap + assert "BAD_T" in uuids_run # no sim coverage -> conservative include + assert len(skipped) == 0 + + def test_incomplete_coverage_not_triggered_by_preprocess(self): + """Test with no sim coverage is NOT auto-included for pre_process changes.""" + cache = { + "BAD_T": ["src/pre_process/m_start_up.fpp"], + } + changed = {"src/pre_process/m_data_output.fpp"} + to_run, skipped = filter_tests_by_coverage([FakeCase("BAD_T")], cache, changed) + assert len(to_run) == 0 # no sim change, no overlap -> skip + assert len(skipped) == 1 + + +# =========================================================================== +# Group 4: Corner cases from design discussion +# =========================================================================== + +class TestDesignCornerCases(unittest.TestCase): + + def test_gpu_ifdef_file_still_triggers_if_covered(self): + """ + GPU-specific code lives in the same .fpp file as CPU code. + At file level, changing any part of the file triggers tests that cover it. + """ + cache = {"MUSCL_T": ["src/simulation/m_muscl.fpp"]} + changed = {"src/simulation/m_muscl.fpp"} + to_run, _ = filter_tests_by_coverage([FakeCase("MUSCL_T")], cache, changed) + assert len(to_run) == 1 + + def test_macro_file_triggers_all_via_should_run_all(self): + """parallel_macros.fpp in changed files -> should_run_all_tests() is True.""" + assert should_run_all_tests({"src/common/include/parallel_macros.fpp"}) is True + + def test_new_fpp_file_no_coverage_skips(self): + """ + Brand new .fpp file has no coverage in cache. + All tests are skipped (no test covers the new file). + """ + cache = {"AAAA0001": ["src/simulation/m_rhs.fpp"]} + changed = {"src/simulation/m_brand_new.fpp"} + to_run, skipped = filter_tests_by_coverage([FakeCase("AAAA0001")], cache, changed) + assert len(to_run) == 0 + assert len(skipped) == 1 + + def test_non_fpp_always_run_all_detected(self): + """ + End-to-end: diff lists only cases.py (non-.fpp) -> + _parse_diff_files includes it -> should_run_all_tests fires. + """ + files = _parse_diff_files("toolchain/mfc/test/cases.py\n") + assert should_run_all_tests(files) is True + + def test_niche_feature_pruning(self): + """ + Niche features: most tests don't cover m_bubbles.fpp. + Changing it skips tests that don't touch it. + """ + cache = { + "BUBBLE1": ["src/simulation/m_bubbles.fpp", "src/simulation/m_rhs.fpp"], + "BUBBLE2": ["src/simulation/m_bubbles.fpp"], + "BASIC_1": ["src/simulation/m_rhs.fpp", "src/simulation/m_weno.fpp"], + "BASIC_2": ["src/simulation/m_rhs.fpp"], + "BASIC_3": ["src/simulation/m_weno.fpp"], + } + changed = {"src/simulation/m_bubbles.fpp"} + cases = [FakeCase(u) for u in ["BUBBLE1", "BUBBLE2", "BASIC_1", "BASIC_2", "BASIC_3"]] + to_run, skipped = filter_tests_by_coverage(cases, cache, changed) + uuids_run = {c.get_uuid() for c in to_run} + assert uuids_run == {"BUBBLE1", "BUBBLE2"} + assert len(skipped) == 3 + + +# =========================================================================== +# Group 5: _parse_gcov_json_output — gcov JSON parsing (file-level) +# =========================================================================== + +class TestParseGcovJsonOutput(unittest.TestCase): + + def _make_gcov_json(self, files_data: list) -> bytes: + """Build a fake gzip-compressed gcov JSON blob.""" + data = { + "format_version": "2", + "gcc_version": "15.2.0", + "files": files_data, + } + return gzip.compress(json.dumps(data).encode()) + + def test_returns_set_of_covered_fpp_files(self): + compressed = self._make_gcov_json([{ + "file": "/repo/src/simulation/m_rhs.fpp", + "lines": [ + {"line_number": 45, "count": 3}, + {"line_number": 46, "count": 0}, + {"line_number": 47, "count": 1}, + ], + }]) + result = _parse_gcov_json_output(compressed, "/repo") + assert result == {"src/simulation/m_rhs.fpp"} + + def test_ignores_file_with_zero_coverage(self): + compressed = self._make_gcov_json([{ + "file": "/repo/src/simulation/m_rhs.fpp", + "lines": [ + {"line_number": 10, "count": 0}, + {"line_number": 11, "count": 0}, + ], + }]) + result = _parse_gcov_json_output(compressed, "/repo") + assert result == set() + + def test_ignores_f90_files(self): + """Generated .f90 files must not appear in coverage output.""" + compressed = self._make_gcov_json([ + { + "file": "/repo/build/fypp/simulation/m_rhs.fpp.f90", + "lines": [{"line_number": 10, "count": 5}], + }, + { + "file": "/repo/src/simulation/m_rhs.fpp", + "lines": [{"line_number": 45, "count": 1}], + }, + ]) + result = _parse_gcov_json_output(compressed, "/repo") + assert result == {"src/simulation/m_rhs.fpp"} + + def test_handles_raw_json_gcov12(self): + """gcov 12 outputs raw JSON (not gzip). Must parse correctly.""" + data = { + "format_version": "1", + "gcc_version": "12.3.0", + "files": [{ + "file": "/repo/src/simulation/m_rhs.fpp", + "lines": [{"line_number": 45, "count": 3}], + }], + } + raw = json.dumps(data).encode() + result = _parse_gcov_json_output(raw, "/repo") + assert result == {"src/simulation/m_rhs.fpp"} + + def test_handles_invalid_data_gracefully(self): + result = _parse_gcov_json_output(b"not valid gzip or json", "/repo") + assert result == set() + + def test_handles_empty_files_list(self): + compressed = self._make_gcov_json([]) + result = _parse_gcov_json_output(compressed, "/repo") + assert result == set() + + def test_multiple_fpp_files(self): + compressed = self._make_gcov_json([ + { + "file": "/repo/src/simulation/m_rhs.fpp", + "lines": [{"line_number": 45, "count": 1}], + }, + { + "file": "/repo/src/simulation/m_weno.fpp", + "lines": [{"line_number": 200, "count": 2}], + }, + ]) + result = _parse_gcov_json_output(compressed, "/repo") + assert result == {"src/simulation/m_rhs.fpp", "src/simulation/m_weno.fpp"} + + def test_concatenated_json_from_batched_gcov(self): + """Batched gcov calls produce concatenated JSON objects (gcov 12).""" + obj1 = json.dumps({ + "format_version": "1", + "gcc_version": "12.3.0", + "files": [{ + "file": "/repo/src/simulation/m_rhs.fpp", + "lines": [{"line_number": 45, "count": 3}], + }], + }) + obj2 = json.dumps({ + "format_version": "1", + "gcc_version": "12.3.0", + "files": [{ + "file": "/repo/src/simulation/m_weno.fpp", + "lines": [{"line_number": 10, "count": 1}], + }], + }) + raw = (obj1 + "\n" + obj2).encode() + result = _parse_gcov_json_output(raw, "/repo") + assert result == {"src/simulation/m_rhs.fpp", "src/simulation/m_weno.fpp"} + + def test_concatenated_json_skips_zero_coverage(self): + """Batched gcov: files with zero coverage are excluded.""" + obj1 = json.dumps({ + "format_version": "1", + "files": [{ + "file": "/repo/src/simulation/m_rhs.fpp", + "lines": [{"line_number": 45, "count": 3}], + }], + }) + obj2 = json.dumps({ + "format_version": "1", + "files": [{ + "file": "/repo/src/simulation/m_weno.fpp", + "lines": [{"line_number": 10, "count": 0}], + }], + }) + raw = (obj1 + "\n" + obj2).encode() + result = _parse_gcov_json_output(raw, "/repo") + assert result == {"src/simulation/m_rhs.fpp"} + + +# =========================================================================== +# Group 6: _normalize_cache — old format conversion +# =========================================================================== + +class TestNormalizeCache(unittest.TestCase): + + def test_converts_old_line_level_format(self): + """Old format {uuid: {file: [lines]}} -> new format {uuid: [files]}.""" + old_cache = { + "TEST_A": { + "src/simulation/m_rhs.fpp": [45, 46, 47], + "src/simulation/m_weno.fpp": [100, 200], + }, + "TEST_B": { + "src/simulation/m_bubbles.fpp": [10], + }, + "_meta": {"cases_hash": "abc123"}, + } + result = _normalize_cache(old_cache) + assert isinstance(result["TEST_A"], list) + assert set(result["TEST_A"]) == {"src/simulation/m_rhs.fpp", "src/simulation/m_weno.fpp"} + assert result["TEST_B"] == ["src/simulation/m_bubbles.fpp"] + assert result["_meta"] == {"cases_hash": "abc123"} + + def test_new_format_unchanged(self): + """New format {uuid: [files]} passes through unchanged.""" + new_cache = { + "TEST_A": ["src/simulation/m_rhs.fpp", "src/simulation/m_weno.fpp"], + "_meta": {"cases_hash": "abc123"}, + } + result = _normalize_cache(new_cache) + assert result["TEST_A"] == ["src/simulation/m_rhs.fpp", "src/simulation/m_weno.fpp"] + + def test_empty_coverage_dict_becomes_empty_list(self): + """Test with 0 coverage (old format: empty dict) -> empty list.""" + old_cache = {"TEST_A": {}, "_meta": {"cases_hash": "abc"}} + result = _normalize_cache(old_cache) + assert result["TEST_A"] == [] + + +# =========================================================================== +# Group 7: Cache path format +# =========================================================================== + +class TestCachePath(unittest.TestCase): + + def test_cache_path_is_gzipped(self): + """Cache file must use .json.gz so it can be committed to the repo.""" + assert str(COVERAGE_CACHE_PATH).endswith(".json.gz") + + +if __name__ == "__main__": + unittest.main() From 1087b9dc303ada320951c8e6fe05e770bb5c3074 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sun, 1 Mar 2026 21:19:34 -0500 Subject: [PATCH 02/35] Fix post_process failures in coverage cache by applying output params The coverage cache builder's _prepare_test was generating simulation .inp files without post_process output params (cons_vars_wrt, parallel_io, etc.). Without these, simulation doesn't write output files and post_process fails. Extract post_process param dicts into shared constants in case.py (POST_PROCESS_OUTPUT_PARAMS, POST_PROCESS_3D_PARAMS, POST_PROCESS_OFF_PARAMS) and a get_post_process_mods() function. Both the generated case.py template and coverage.py now use the same source of truth. Co-Authored-By: Claude Opus 4.6 --- toolchain/mfc/test/case.py | 60 +++++++++++++++++------- toolchain/mfc/test/coverage.py | 7 ++- toolchain/mfc/test/test_coverage_unit.py | 1 + 3 files changed, 49 insertions(+), 19 deletions(-) diff --git a/toolchain/mfc/test/case.py b/toolchain/mfc/test/case.py index c5ffdd301a..9d27e9df62 100644 --- a/toolchain/mfc/test/case.py +++ b/toolchain/mfc/test/case.py @@ -1,4 +1,4 @@ -import os, glob, hashlib, binascii, subprocess, itertools, dataclasses, shutil +import os, json, glob, hashlib, binascii, subprocess, itertools, dataclasses, shutil from typing import List, Set, Union, Callable, Optional @@ -7,6 +7,44 @@ from ..run import input from ..build import MFCTarget, get_target +# Parameters that enable simulation output writing for post_process. +# When post_process is a target, simulation must write field data so +# post_process has something to read. Used in the generated case.py +# template and by the coverage cache builder. +POST_PROCESS_OUTPUT_PARAMS = { + 'parallel_io': 'T', 'cons_vars_wrt': 'T', + 'prim_vars_wrt': 'T', 'alpha_rho_wrt(1)': 'T', + 'rho_wrt': 'T', 'mom_wrt(1)': 'T', + 'vel_wrt(1)': 'T', 'E_wrt': 'T', + 'pres_wrt': 'T', 'alpha_wrt(1)': 'T', + 'gamma_wrt': 'T', 'heat_ratio_wrt': 'T', + 'pi_inf_wrt': 'T', 'pres_inf_wrt': 'T', + 'c_wrt': 'T', +} + +# Additional output parameters for 3D cases (p != 0). +POST_PROCESS_3D_PARAMS = { + 'fd_order': 1, + 'omega_wrt(1)': 'T', + 'omega_wrt(2)': 'T', + 'omega_wrt(3)': 'T', +} + +# Parameters set when post_process is NOT a target. +POST_PROCESS_OFF_PARAMS = { + 'parallel_io': 'F', + 'prim_vars_wrt': 'F', +} + + +def get_post_process_mods(case_params: dict) -> dict: + """Return parameter modifications needed when post_process is a target.""" + mods = dict(POST_PROCESS_OUTPUT_PARAMS) + if int(case_params.get('p', 0)) != 0: + mods.update(POST_PROCESS_3D_PARAMS) + return mods + + Tend = 0.25 Nt = 50 mydt = 0.0005 @@ -204,25 +242,11 @@ def create_directory(self): mods = {{}} if "post_process" in ARGS["mfc"]["targets"]: - mods = {{ - 'parallel_io' : 'T', 'cons_vars_wrt' : 'T', - 'prim_vars_wrt': 'T', 'alpha_rho_wrt(1)': 'T', - 'rho_wrt' : 'T', 'mom_wrt(1)' : 'T', - 'vel_wrt(1)' : 'T', 'E_wrt' : 'T', - 'pres_wrt' : 'T', 'alpha_wrt(1)' : 'T', - 'gamma_wrt' : 'T', 'heat_ratio_wrt' : 'T', - 'pi_inf_wrt' : 'T', 'pres_inf_wrt' : 'T', - 'c_wrt' : 'T', - }} - + mods = {json.dumps(POST_PROCESS_OUTPUT_PARAMS)} if case['p'] != 0: - mods['fd_order'] = 1 - mods['omega_wrt(1)'] = 'T' - mods['omega_wrt(2)'] = 'T' - mods['omega_wrt(3)'] = 'T' + mods.update({json.dumps(POST_PROCESS_3D_PARAMS)}) else: - mods['parallel_io'] = 'F' - mods['prim_vars_wrt'] = 'F' + mods = {json.dumps(POST_PROCESS_OFF_PARAMS)} print(json.dumps({{**case, **mods}})) """) diff --git a/toolchain/mfc/test/coverage.py b/toolchain/mfc/test/coverage.py index 404715976e..6f2d3b2c66 100644 --- a/toolchain/mfc/test/coverage.py +++ b/toolchain/mfc/test/coverage.py @@ -31,7 +31,7 @@ from .. import common from ..common import MFCException from ..build import PRE_PROCESS, SIMULATION, POST_PROCESS, SYSCHECK -from .case import input_bubbles_lagrange +from .case import input_bubbles_lagrange, get_post_process_mods COVERAGE_CACHE_PATH = Path(common.MFC_ROOT_DIR) / "toolchain/mfc/test/test_coverage_cache.json.gz" @@ -343,6 +343,11 @@ def _prepare_test(case, root_dir: str) -> dict: # pylint: disable=unused-argume cons.print(f"[yellow]Warning: Failed to generate Lagrange bubble input " f"for {case.get_uuid()}: {exc}[/yellow]") + # Apply post_process output params so simulation writes data files that + # post_process reads. Mirrors the generated case.py logic that normally + # runs via ./mfc.sh run (see POST_PROCESS_OUTPUT_PARAMS in case.py). + case.params.update(get_post_process_mods(case.params)) + test_dir = case.get_dirpath() input_file = case.to_input_file() diff --git a/toolchain/mfc/test/test_coverage_unit.py b/toolchain/mfc/test/test_coverage_unit.py index 35f9028a98..09041b7fd5 100644 --- a/toolchain/mfc/test/test_coverage_unit.py +++ b/toolchain/mfc/test/test_coverage_unit.py @@ -72,6 +72,7 @@ class _FakeMFCException(Exception): _case_stub = sys.modules.get("toolchain.mfc.test.case", _make_stub("toolchain.mfc.test.case")) _case_stub.input_bubbles_lagrange = lambda case: None +_case_stub.get_post_process_mods = lambda params: {} # Load coverage.py by injecting stubs into sys.modules so relative imports resolve. _COVERAGE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "coverage.py") From 5c581f6c0cac605d4c60655a5fb9dcac662bb4a1 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sun, 1 Mar 2026 21:35:44 -0500 Subject: [PATCH 03/35] TEMP: strip ALWAYS_RUN_ALL to GPU macros only to exercise pruning in CI Co-Authored-By: Claude Opus 4.6 --- toolchain/mfc/test/coverage.py | 15 ++--- toolchain/mfc/test/test_coverage_unit.py | 70 ++++-------------------- 2 files changed, 15 insertions(+), 70 deletions(-) diff --git a/toolchain/mfc/test/coverage.py b/toolchain/mfc/test/coverage.py index 6f2d3b2c66..44949d3c73 100644 --- a/toolchain/mfc/test/coverage.py +++ b/toolchain/mfc/test/coverage.py @@ -39,28 +39,21 @@ # Changes to these files trigger the full test suite. # CPU coverage cannot tell us about GPU directive changes (macro files), and # toolchain files define or change the set of tests themselves. +# TEMP: stripped to GPU macros only so CI exercises the pruning logic. +# Restore full list before merge. ALWAYS_RUN_ALL = frozenset([ "src/common/include/parallel_macros.fpp", "src/common/include/acc_macros.fpp", "src/common/include/omp_macros.fpp", "src/common/include/shared_parallel_macros.fpp", - "src/common/include/macros.fpp", - "src/common/include/case.fpp", - "toolchain/mfc/test/cases.py", - "toolchain/mfc/test/case.py", - "toolchain/mfc/params/definitions.py", - "toolchain/mfc/run/input.py", - "toolchain/mfc/case_validator.py", - "toolchain/mfc/test/coverage.py", ]) # Directory prefixes: any changed file under these paths triggers full suite. # Note: src/simulation/include/ (.fpp files like inline_riemann.fpp) is NOT # listed here — Fypp line markers (--line-marker-format=gfortran5) correctly # attribute included file paths, so gcov coverage tracks them accurately. -ALWAYS_RUN_ALL_PREFIXES = ( - "toolchain/cmake/", -) +# TEMP: cmake/ prefix removed to exercise pruning in CI. Restore before merge. +ALWAYS_RUN_ALL_PREFIXES = () def _get_gcov_version(gcov_binary: str) -> str: diff --git a/toolchain/mfc/test/test_coverage_unit.py b/toolchain/mfc/test/test_coverage_unit.py index 09041b7fd5..1d6d80d114 100644 --- a/toolchain/mfc/test/test_coverage_unit.py +++ b/toolchain/mfc/test/test_coverage_unit.py @@ -131,7 +131,8 @@ class _FakeMFCException(Exception): .replace("from .. import common", "") .replace("from ..common import MFCException", "MFCException = _globals['MFCException']") .replace("from ..build import PRE_PROCESS, SIMULATION, POST_PROCESS, SYSCHECK", "") - .replace("from .case import input_bubbles_lagrange", "input_bubbles_lagrange = lambda case: None") + .replace("from .case import input_bubbles_lagrange, get_post_process_mods", + "input_bubbles_lagrange = lambda case: None\nget_post_process_mods = lambda params: {}") ) exec(compile(_src, _COVERAGE_PATH, "exec"), _globals) # noqa: S102 @@ -220,58 +221,14 @@ def test_shared_parallel_macros_triggers_all(self): {"src/common/include/shared_parallel_macros.fpp"} ) is True - def test_macros_fpp_triggers_all(self): - assert should_run_all_tests( - {"src/common/include/macros.fpp"} - ) is True - - def test_cases_py_triggers_all(self): - assert should_run_all_tests( - {"toolchain/mfc/test/cases.py"} - ) is True - - def test_case_py_triggers_all(self): - assert should_run_all_tests( - {"toolchain/mfc/test/case.py"} - ) is True + # TEMP: macros.fpp, cases.py, case.py, definitions.py, input.py, + # case_validator.py, case.fpp, coverage.py, cmake/ removed from + # ALWAYS_RUN_ALL to exercise pruning in CI. Restore before merge. - def test_definitions_py_triggers_all(self): + def test_macros_fpp_does_not_trigger_all(self): assert should_run_all_tests( - {"toolchain/mfc/params/definitions.py"} - ) is True - - def test_input_py_triggers_all(self): - assert should_run_all_tests( - {"toolchain/mfc/run/input.py"} - ) is True - - def test_case_validator_triggers_all(self): - assert should_run_all_tests( - {"toolchain/mfc/case_validator.py"} - ) is True - - def test_cmakelists_does_not_trigger_all(self): - assert should_run_all_tests({"CMakeLists.txt"}) is False - - def test_case_fpp_triggers_all(self): - assert should_run_all_tests( - {"src/common/include/case.fpp"} - ) is True - - def test_coverage_py_triggers_all(self): - assert should_run_all_tests( - {"toolchain/mfc/test/coverage.py"} - ) is True - - def test_cmake_dir_triggers_all(self): - assert should_run_all_tests( - {"toolchain/cmake/FindFFTW.cmake"} - ) is True - - def test_cmake_subdir_triggers_all(self): - assert should_run_all_tests( - {"toolchain/cmake/some/nested/file.cmake"} - ) is True + {"src/common/include/macros.fpp"} + ) is False def test_simulation_module_does_not_trigger_all(self): assert should_run_all_tests( @@ -284,7 +241,7 @@ def test_empty_set_does_not_trigger_all(self): def test_mixed_one_trigger_fires_all(self): assert should_run_all_tests({ "src/simulation/m_rhs.fpp", - "src/common/include/macros.fpp", + "src/common/include/parallel_macros.fpp", }) is True @@ -440,13 +397,8 @@ def test_new_fpp_file_no_coverage_skips(self): assert len(to_run) == 0 assert len(skipped) == 1 - def test_non_fpp_always_run_all_detected(self): - """ - End-to-end: diff lists only cases.py (non-.fpp) -> - _parse_diff_files includes it -> should_run_all_tests fires. - """ - files = _parse_diff_files("toolchain/mfc/test/cases.py\n") - assert should_run_all_tests(files) is True + # TEMP: test_non_fpp_always_run_all_detected removed (cases.py not + # in ALWAYS_RUN_ALL during CI pruning test). Restore before merge. def test_niche_feature_pruning(self): """ From 28f2dc180b715e51fc579ae37f89343e0f8be6a2 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sun, 1 Mar 2026 22:08:41 -0500 Subject: [PATCH 04/35] Fix review findings: phase labels, unused params, FileNotFoundError guard Co-Authored-By: Claude Opus 4.6 --- .github/workflows/test.yml | 2 +- toolchain/mfc/test/coverage.py | 32 ++++++++++++++++++-------------- toolchain/mfc/test/test.py | 2 +- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7f866553f0..a7ce7e981b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -111,7 +111,7 @@ jobs: group: phoenix labels: gt permissions: - contents: write + contents: write # Required for Commit Cache to Master on push events steps: - name: Clone uses: actions/checkout@v4 diff --git a/toolchain/mfc/test/coverage.py b/toolchain/mfc/test/coverage.py index 44949d3c73..6134065473 100644 --- a/toolchain/mfc/test/coverage.py +++ b/toolchain/mfc/test/coverage.py @@ -71,7 +71,7 @@ def _get_gcov_version(gcov_binary: str) -> str: return "unknown" -def find_gcov_binary(_root_dir: str = "") -> str: # pylint: disable=unused-argument +def find_gcov_binary() -> str: """ Find a GNU gcov binary compatible with the system gfortran. @@ -368,25 +368,25 @@ def _prepare_test(case, root_dir: str) -> dict: # pylint: disable=unused-argume } -def build_coverage_cache( # pylint: disable=unused-argument,too-many-locals,too-many-statements - root_dir: str, cases: list, extra_args: list = None, n_jobs: int = None, +def build_coverage_cache( # pylint: disable=too-many-locals,too-many-statements + root_dir: str, cases: list, n_jobs: int = None, ) -> None: """ Build the file-level coverage cache by running tests in parallel. - Phase 0 — Prepare all tests: generate .inp files and resolve binary paths. + Phase 1 — Prepare all tests: generate .inp files and resolve binary paths. This happens single-threaded so the parallel phase has zero Python overhead. - Phase 1 — Run all tests concurrently. Each worker invokes Fortran binaries + Phase 2 — Run all tests concurrently. Each worker invokes Fortran binaries directly (no ``./mfc.sh run``, no shell scripts). Each test's GCOV_PREFIX points to an isolated directory so .gcda files don't collide. - Phase 2 — For each test, copy its .gcda tree into the real build directory, + Phase 3 — For each test, copy its .gcda tree into the real build directory, run gcov to collect which .fpp files had coverage, then remove the .gcda files. Requires a prior ``--gcov`` build: ``./mfc.sh build --gcov -j 8`` """ - gcov_bin = find_gcov_binary(root_dir) + gcov_bin = find_gcov_binary() gcno_files = find_gcno_files(root_dir) strip = _compute_gcov_prefix_strip(root_dir) @@ -402,8 +402,8 @@ def build_coverage_cache( # pylint: disable=unused-argument,too-many-locals,too cons.print(f"[dim]GCOV_PREFIX_STRIP={strip}[/dim]") cons.print() - # Phase 0: Prepare all tests (single-threaded, ~30s for 555 tests). - cons.print("[bold]Phase 0/2: Preparing tests...[/bold]") + # Phase 1: Prepare all tests (single-threaded, ~30s for 555 tests). + cons.print("[bold]Phase 1/3: Preparing tests...[/bold]") test_infos = [] for i, case in enumerate(cases): test_infos.append(_prepare_test(case, root_dir)) @@ -413,8 +413,8 @@ def build_coverage_cache( # pylint: disable=unused-argument,too-many-locals,too gcda_dir = tempfile.mkdtemp(prefix="mfc_gcov_") try: - # Phase 1: Run all tests in parallel via direct binary invocation. - cons.print("[bold]Phase 1/2: Running tests...[/bold]") + # Phase 2: Run all tests in parallel via direct binary invocation. + cons.print("[bold]Phase 2/3: Running tests...[/bold]") test_results: dict = {} all_failures: dict = {} with ThreadPoolExecutor(max_workers=phase1_jobs) as pool: @@ -453,10 +453,10 @@ def build_coverage_cache( # pylint: disable=unused-argument,too-many-locals,too cons.print(f"[yellow]Sample test {sample_uuid}: " f"no build/ dir in {sample_gcda}[/yellow]") - # Phase 2: Collect gcov coverage from each test's isolated .gcda directory. + # Phase 3: Collect gcov coverage from each test's isolated .gcda directory. # .gcno files are temporarily copied alongside .gcda files, then removed. cons.print() - cons.print("[bold]Phase 2/2: Collecting coverage...[/bold]") + cons.print("[bold]Phase 3/3: Collecting coverage...[/bold]") cache: dict = {} completed = 0 with ThreadPoolExecutor(max_workers=n_jobs) as pool: @@ -535,7 +535,11 @@ def load_coverage_cache(root_dir: str) -> Optional[dict]: return None cases_py = Path(root_dir) / "toolchain/mfc/test/cases.py" - current_hash = hashlib.sha256(cases_py.read_bytes()).hexdigest() + try: + current_hash = hashlib.sha256(cases_py.read_bytes()).hexdigest() + except FileNotFoundError: + cons.print("[yellow]Warning: cases.py not found; cannot verify cache staleness.[/yellow]") + return None stored_hash = cache.get("_meta", {}).get("cases_hash", "") if current_hash != stored_hash: diff --git a/toolchain/mfc/test/test.py b/toolchain/mfc/test/test.py index 3f857b59ca..9732e3d1fc 100644 --- a/toolchain/mfc/test/test.py +++ b/toolchain/mfc/test/test.py @@ -238,7 +238,7 @@ def test(): unique_builds.add(slug) build_coverage_cache(common.MFC_ROOT_DIR, all_cases, - extra_args=ARG("--"), n_jobs=int(ARG("jobs"))) + n_jobs=int(ARG("jobs"))) return cases, skipped_cases = __filter(cases) From f4b282aba77b01bb7fb390aeb451ae2ee9b6ed1a Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sun, 1 Mar 2026 23:12:07 -0500 Subject: [PATCH 05/35] Increase cache builder per-test timeout from 300s to 600s Heavy 3D QBMM tests with gcov instrumentation need more time. Co-Authored-By: Claude Opus 4.6 --- toolchain/mfc/test/coverage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolchain/mfc/test/coverage.py b/toolchain/mfc/test/coverage.py index 6134065473..413d73971c 100644 --- a/toolchain/mfc/test/coverage.py +++ b/toolchain/mfc/test/coverage.py @@ -304,7 +304,7 @@ def _run_single_test_direct(test_info: dict, gcda_dir: str, strip: str) -> tuple try: result = subprocess.run(cmd, check=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - env=env, cwd=test_dir, timeout=300) + env=env, cwd=test_dir, timeout=600) if result.returncode != 0: failures.append((target_name, result.returncode)) except subprocess.TimeoutExpired: From 3675860f288645888417569fb471ee3bf22bc187 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Mon, 2 Mar 2026 11:04:28 -0500 Subject: [PATCH 06/35] Log failed test output in cache builder for easier debugging Co-Authored-By: Claude Opus 4.6 --- toolchain/mfc/test/coverage.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/toolchain/mfc/test/coverage.py b/toolchain/mfc/test/coverage.py index 413d73971c..7e96f50b3e 100644 --- a/toolchain/mfc/test/coverage.py +++ b/toolchain/mfc/test/coverage.py @@ -306,11 +306,13 @@ def _run_single_test_direct(test_info: dict, gcda_dir: str, strip: str) -> tuple stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env, cwd=test_dir, timeout=600) if result.returncode != 0: - failures.append((target_name, result.returncode)) + # Save last few lines of output for debugging. + tail = "\n".join(result.stdout.strip().splitlines()[-5:]) + failures.append((target_name, result.returncode, tail)) except subprocess.TimeoutExpired: - failures.append((target_name, "timeout")) + failures.append((target_name, "timeout", "")) except Exception as exc: - failures.append((target_name, str(exc))) + failures.append((target_name, str(exc), "")) return uuid, test_gcda, failures @@ -434,8 +436,13 @@ def build_coverage_cache( # pylint: disable=too-many-locals,too-many-statements cons.print() cons.print(f"[bold yellow]Warning: {len(all_failures)} tests had target failures:[/bold yellow]") for uuid, fails in sorted(all_failures.items()): - fail_str = ", ".join(f"{t}={rc}" for t, rc in fails) + fail_str = ", ".join(f"{t}={rc}" for t, rc, _ in fails) cons.print(f" [yellow]{uuid}[/yellow]: {fail_str}") + for target_name, _rc, tail in fails: + if tail: + cons.print(f" {target_name} output (last 5 lines):") + for line in tail.splitlines(): + cons.print(f" {line}") # Diagnostic: verify .gcda files exist for at least one test. sample_uuid = next(iter(test_results), None) From ca70390a729f64a9e135b57c0cc6b59543671707 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Mon, 2 Mar 2026 18:09:35 -0500 Subject: [PATCH 07/35] Fix Rich MarkupError crash when build output contains bracket paths Build errors containing [/tmp/...] paths (e.g. LTO linker output) were misinterpreted as Rich markup closing tags, crashing the error display and masking the actual build failure. Wrap raw output in Text() to prevent markup interpretation. Co-Authored-By: Claude Opus 4.6 --- toolchain/mfc/build.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/toolchain/mfc/build.py b/toolchain/mfc/build.py index 6430f7ad35..08ff6d7510 100644 --- a/toolchain/mfc/build.py +++ b/toolchain/mfc/build.py @@ -1,6 +1,7 @@ import os, typing, hashlib, dataclasses, subprocess, re, time, sys, threading, queue from rich.panel import Panel +from rich.text import Text from rich.progress import Progress, SpinnerColumn, BarColumn, TextColumn, TimeElapsedColumn, TaskProgressColumn from .case import Case @@ -273,14 +274,14 @@ def _show_build_error(result: subprocess.CompletedProcess, stage: str): stdout_text = result.stdout if isinstance(result.stdout, str) else result.stdout.decode('utf-8', errors='replace') stdout_text = stdout_text.strip() if stdout_text: - cons.raw.print(Panel(stdout_text, title="Output", border_style="yellow")) + cons.raw.print(Panel(Text(stdout_text), title="Output", border_style="yellow")) # Show stderr if available if result.stderr: stderr_text = result.stderr if isinstance(result.stderr, str) else result.stderr.decode('utf-8', errors='replace') stderr_text = stderr_text.strip() if stderr_text: - cons.raw.print(Panel(stderr_text, title="Errors", border_style="red")) + cons.raw.print(Panel(Text(stderr_text), title="Errors", border_style="red")) cons.print() From 70ad8b88cf0b86919ef6ab38964abf2d15e11d28 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Mon, 2 Mar 2026 20:36:56 -0500 Subject: [PATCH 08/35] Disable AVX-512 FP16 to fix build on Granite Rapids nodes gfortran 12+ with -march=native on Granite Rapids (GNR) CPUs emits vmovw instructions (AVX-512 FP16) that binutils 2.35 cannot assemble, causing LTO link failures. Add -mno-avx512fp16 when the compiler supports it. FP16 is unused in MFC's double-precision computations. Co-Authored-By: Claude Opus 4.6 --- CMakeLists.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7154fa3010..01da0c7a28 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -237,6 +237,13 @@ if (CMAKE_BUILD_TYPE STREQUAL "Release") CHECK_FORTRAN_COMPILER_FLAG("-march=native" SUPPORTS_MARCH_NATIVE) if (SUPPORTS_MARCH_NATIVE) add_compile_options($<$:-march=native>) + # Disable AVX-512 FP16: gfortran ≥12 emits vmovw instructions on + # Granite Rapids CPUs, but binutils <2.38 cannot assemble them. + # FP16 is unused in MFC's double-precision computations. + CHECK_FORTRAN_COMPILER_FLAG("-mno-avx512fp16" SUPPORTS_MNO_AVX512FP16) + if (SUPPORTS_MNO_AVX512FP16) + add_compile_options($<$:-mno-avx512fp16>) + endif() else() CHECK_FORTRAN_COMPILER_FLAG("-mcpu=native" SUPPORTS_MCPU_NATIVE) if (SUPPORTS_MCPU_NATIVE) From 01aa76850fb10d79a1f7edf89661e562cdce5387 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Mon, 2 Mar 2026 22:30:14 -0500 Subject: [PATCH 09/35] Fix post_process failures in coverage cache builder Adaptive-dt tests (B2EC143C): clamp t_stop so post_process reads only saves that simulation actually produces within t_step_stop. 3D QBMM tests (41C830CC, D33BD146): skip vorticity output params for 3D QBMM configs. Normal tests never run post_process (only pre_process + simulation), so these configs are untested with it. Also increase failure output capture from 5 to 15 lines. Co-Authored-By: Claude Opus 4.6 --- toolchain/mfc/test/coverage.py | 25 +++++++++++++++++++++--- toolchain/mfc/test/test_coverage_unit.py | 11 +++++++++-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/toolchain/mfc/test/coverage.py b/toolchain/mfc/test/coverage.py index 7e96f50b3e..efc8dd44b1 100644 --- a/toolchain/mfc/test/coverage.py +++ b/toolchain/mfc/test/coverage.py @@ -31,7 +31,8 @@ from .. import common from ..common import MFCException from ..build import PRE_PROCESS, SIMULATION, POST_PROCESS, SYSCHECK -from .case import input_bubbles_lagrange, get_post_process_mods +from .case import (input_bubbles_lagrange, get_post_process_mods, + POST_PROCESS_3D_PARAMS) COVERAGE_CACHE_PATH = Path(common.MFC_ROOT_DIR) / "toolchain/mfc/test/test_coverage_cache.json.gz" @@ -306,8 +307,8 @@ def _run_single_test_direct(test_info: dict, gcda_dir: str, strip: str) -> tuple stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env, cwd=test_dir, timeout=600) if result.returncode != 0: - # Save last few lines of output for debugging. - tail = "\n".join(result.stdout.strip().splitlines()[-5:]) + # Save last lines of output for debugging. + tail = "\n".join(result.stdout.strip().splitlines()[-15:]) failures.append((target_name, result.returncode, tail)) except subprocess.TimeoutExpired: failures.append((target_name, "timeout", "")) @@ -343,6 +344,24 @@ def _prepare_test(case, root_dir: str) -> dict: # pylint: disable=unused-argume # runs via ./mfc.sh run (see POST_PROCESS_OUTPUT_PARAMS in case.py). case.params.update(get_post_process_mods(case.params)) + # Adaptive-dt tests: post_process computes n_save = int(t_stop/t_save)+1 + # and iterates over that many save indices. But with small t_step_stop + # the simulation produces far fewer saves. Clamp t_stop so post_process + # only reads saves that actually exist. + if case.params.get('cfl_adap_dt', 'F') == 'T': + t_save = float(case.params.get('t_save', 1.0)) + case.params['t_stop'] = str(t_save) # n_save = 2: indices 0 and 1 + + # Heavy 3D tests: remove vorticity output (omega_wrt + fd_order) for + # 3D QBMM tests. Normal test execution never runs post_process (only + # PRE_PROCESS + SIMULATION; see test.py line ~469), so post_process on + # heavy 3D configs is untested. Vorticity FD computation on large grids + # with many QBMM variables causes post_process to crash (exit code 2). + if (int(case.params.get('p', 0)) > 0 and + case.params.get('qbmm', 'F') == 'T'): + for key in POST_PROCESS_3D_PARAMS: + case.params.pop(key, None) + test_dir = case.get_dirpath() input_file = case.to_input_file() diff --git a/toolchain/mfc/test/test_coverage_unit.py b/toolchain/mfc/test/test_coverage_unit.py index 1d6d80d114..9864247538 100644 --- a/toolchain/mfc/test/test_coverage_unit.py +++ b/toolchain/mfc/test/test_coverage_unit.py @@ -73,6 +73,9 @@ class _FakeMFCException(Exception): _case_stub = sys.modules.get("toolchain.mfc.test.case", _make_stub("toolchain.mfc.test.case")) _case_stub.input_bubbles_lagrange = lambda case: None _case_stub.get_post_process_mods = lambda params: {} +_case_stub.POST_PROCESS_3D_PARAMS = { + 'fd_order': 1, 'omega_wrt(1)': 'T', 'omega_wrt(2)': 'T', 'omega_wrt(3)': 'T', +} # Load coverage.py by injecting stubs into sys.modules so relative imports resolve. _COVERAGE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "coverage.py") @@ -131,8 +134,12 @@ class _FakeMFCException(Exception): .replace("from .. import common", "") .replace("from ..common import MFCException", "MFCException = _globals['MFCException']") .replace("from ..build import PRE_PROCESS, SIMULATION, POST_PROCESS, SYSCHECK", "") - .replace("from .case import input_bubbles_lagrange, get_post_process_mods", - "input_bubbles_lagrange = lambda case: None\nget_post_process_mods = lambda params: {}") + .replace("from .case import (input_bubbles_lagrange, get_post_process_mods,\n" + " POST_PROCESS_3D_PARAMS)", + "input_bubbles_lagrange = lambda case: None\n" + "get_post_process_mods = lambda params: {}\n" + "POST_PROCESS_3D_PARAMS = {'fd_order': 1, 'omega_wrt(1)': 'T', " + "'omega_wrt(2)': 'T', 'omega_wrt(3)': 'T'}") ) exec(compile(_src, _COVERAGE_PATH, "exec"), _globals) # noqa: S102 From bb43adb7c9351fe8cbbe525ce30d5bacc3298fc7 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Tue, 3 Mar 2026 01:51:32 -0500 Subject: [PATCH 10/35] Fix Fortran namelist quoting bug and add .inp existence check str(t_save) produced a Python string that get_inp() quoted with single quotes (t_stop = '0.01'), which is invalid for a Fortran real namelist variable. Keep t_save as a float so it formats unquoted (t_stop = 0.01). Also add a pre-launch check for .inp files in the test runner to distinguish Python-side file-writing failures from Fortran-side missing-file errors in future runs. Co-Authored-By: Claude Opus 4.6 --- toolchain/mfc/test/coverage.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/toolchain/mfc/test/coverage.py b/toolchain/mfc/test/coverage.py index efc8dd44b1..9d67a1401a 100644 --- a/toolchain/mfc/test/coverage.py +++ b/toolchain/mfc/test/coverage.py @@ -301,6 +301,15 @@ def _run_single_test_direct(test_info: dict, gcda_dir: str, strip: str) -> tuple cons.print(f"[yellow]Warning: binary {target_name} not found " f"at {bin_path} for test {uuid}[/yellow]") continue + + # Verify .inp file exists before running (diagnostic for transient + # filesystem issues where the file goes missing between phases). + inp_file = os.path.join(test_dir, f"{target_name}.inp") + if not os.path.isfile(inp_file): + failures.append((target_name, "missing-inp", + f"{inp_file} not found before launch")) + continue + cmd = mpi_cmd + [bin_path] try: result = subprocess.run(cmd, check=False, text=True, @@ -350,7 +359,7 @@ def _prepare_test(case, root_dir: str) -> dict: # pylint: disable=unused-argume # only reads saves that actually exist. if case.params.get('cfl_adap_dt', 'F') == 'T': t_save = float(case.params.get('t_save', 1.0)) - case.params['t_stop'] = str(t_save) # n_save = 2: indices 0 and 1 + case.params['t_stop'] = t_save # n_save = 2: indices 0 and 1 # Heavy 3D tests: remove vorticity output (omega_wrt + fd_order) for # 3D QBMM tests. Normal test execution never runs post_process (only From 97cab676af18dd8592db00d8f22c52d17bb2d020 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Tue, 3 Mar 2026 01:56:15 -0500 Subject: [PATCH 11/35] Restore ALWAYS_RUN_ALL, unit tests, and remove duplicate use statement Reverts TEMP changes made during CI pruning validation: - Restore full ALWAYS_RUN_ALL list (macros.fpp, cases.py, case.py, definitions.py, input.py, case_validator.py, case.fpp, coverage.py) - Restore ALWAYS_RUN_ALL_PREFIXES with toolchain/cmake/ - Restore 11 unit tests that were stripped or inverted - Remove duplicate 'use m_helper_basic' from m_bubbles.fpp Co-Authored-By: Claude Opus 4.6 --- src/simulation/m_bubbles.fpp | 1 - toolchain/mfc/test/coverage.py | 14 +++-- toolchain/mfc/test/test_coverage_unit.py | 67 ++++++++++++++++++++---- 3 files changed, 69 insertions(+), 13 deletions(-) diff --git a/src/simulation/m_bubbles.fpp b/src/simulation/m_bubbles.fpp index 28d3f2145b..0f17bd60c3 100644 --- a/src/simulation/m_bubbles.fpp +++ b/src/simulation/m_bubbles.fpp @@ -16,7 +16,6 @@ module m_bubbles use m_variables_conversion !< State variables type conversion procedures use m_helper_basic !< Functions to compare floating point numbers - use m_helper_basic implicit none diff --git a/toolchain/mfc/test/coverage.py b/toolchain/mfc/test/coverage.py index 9d67a1401a..68f7691c25 100644 --- a/toolchain/mfc/test/coverage.py +++ b/toolchain/mfc/test/coverage.py @@ -41,20 +41,28 @@ # CPU coverage cannot tell us about GPU directive changes (macro files), and # toolchain files define or change the set of tests themselves. # TEMP: stripped to GPU macros only so CI exercises the pruning logic. -# Restore full list before merge. ALWAYS_RUN_ALL = frozenset([ "src/common/include/parallel_macros.fpp", "src/common/include/acc_macros.fpp", "src/common/include/omp_macros.fpp", "src/common/include/shared_parallel_macros.fpp", + "src/common/include/macros.fpp", + "src/common/include/case.fpp", + "toolchain/mfc/test/cases.py", + "toolchain/mfc/test/case.py", + "toolchain/mfc/params/definitions.py", + "toolchain/mfc/run/input.py", + "toolchain/mfc/case_validator.py", + "toolchain/mfc/test/coverage.py", ]) # Directory prefixes: any changed file under these paths triggers full suite. # Note: src/simulation/include/ (.fpp files like inline_riemann.fpp) is NOT # listed here — Fypp line markers (--line-marker-format=gfortran5) correctly # attribute included file paths, so gcov coverage tracks them accurately. -# TEMP: cmake/ prefix removed to exercise pruning in CI. Restore before merge. -ALWAYS_RUN_ALL_PREFIXES = () +ALWAYS_RUN_ALL_PREFIXES = ( + "toolchain/cmake/", +) def _get_gcov_version(gcov_binary: str) -> str: diff --git a/toolchain/mfc/test/test_coverage_unit.py b/toolchain/mfc/test/test_coverage_unit.py index 9864247538..bbe972e0ed 100644 --- a/toolchain/mfc/test/test_coverage_unit.py +++ b/toolchain/mfc/test/test_coverage_unit.py @@ -228,14 +228,58 @@ def test_shared_parallel_macros_triggers_all(self): {"src/common/include/shared_parallel_macros.fpp"} ) is True - # TEMP: macros.fpp, cases.py, case.py, definitions.py, input.py, - # case_validator.py, case.fpp, coverage.py, cmake/ removed from - # ALWAYS_RUN_ALL to exercise pruning in CI. Restore before merge. - - def test_macros_fpp_does_not_trigger_all(self): + def test_macros_fpp_triggers_all(self): assert should_run_all_tests( {"src/common/include/macros.fpp"} - ) is False + ) is True + + def test_cases_py_triggers_all(self): + assert should_run_all_tests( + {"toolchain/mfc/test/cases.py"} + ) is True + + def test_case_py_triggers_all(self): + assert should_run_all_tests( + {"toolchain/mfc/test/case.py"} + ) is True + + def test_definitions_py_triggers_all(self): + assert should_run_all_tests( + {"toolchain/mfc/params/definitions.py"} + ) is True + + def test_input_py_triggers_all(self): + assert should_run_all_tests( + {"toolchain/mfc/run/input.py"} + ) is True + + def test_case_validator_triggers_all(self): + assert should_run_all_tests( + {"toolchain/mfc/case_validator.py"} + ) is True + + def test_cmakelists_does_not_trigger_all(self): + assert should_run_all_tests({"CMakeLists.txt"}) is False + + def test_case_fpp_triggers_all(self): + assert should_run_all_tests( + {"src/common/include/case.fpp"} + ) is True + + def test_coverage_py_triggers_all(self): + assert should_run_all_tests( + {"toolchain/mfc/test/coverage.py"} + ) is True + + def test_cmake_dir_triggers_all(self): + assert should_run_all_tests( + {"toolchain/cmake/FindFFTW.cmake"} + ) is True + + def test_cmake_subdir_triggers_all(self): + assert should_run_all_tests( + {"toolchain/cmake/some/nested/file.cmake"} + ) is True def test_simulation_module_does_not_trigger_all(self): assert should_run_all_tests( @@ -248,7 +292,7 @@ def test_empty_set_does_not_trigger_all(self): def test_mixed_one_trigger_fires_all(self): assert should_run_all_tests({ "src/simulation/m_rhs.fpp", - "src/common/include/parallel_macros.fpp", + "src/common/include/macros.fpp", }) is True @@ -404,8 +448,13 @@ def test_new_fpp_file_no_coverage_skips(self): assert len(to_run) == 0 assert len(skipped) == 1 - # TEMP: test_non_fpp_always_run_all_detected removed (cases.py not - # in ALWAYS_RUN_ALL during CI pruning test). Restore before merge. + def test_non_fpp_always_run_all_detected(self): + """ + End-to-end: diff lists only cases.py (non-.fpp) -> + _parse_diff_files includes it -> should_run_all_tests fires. + """ + files = _parse_diff_files("toolchain/mfc/test/cases.py\n") + assert should_run_all_tests(files) is True def test_niche_feature_pruning(self): """ From c6a007102073b55e468165af0211da332305c22f Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Tue, 3 Mar 2026 09:23:54 -0500 Subject: [PATCH 12/35] Exclude example-based tests from coverage pruning Example tests (98/557) cover zero unique files beyond non-example tests, so they add no value to coverage-based pruning. Skip them in --only-changes mode and exclude from cache builds (~18% faster). Co-Authored-By: Claude Opus 4.6 --- toolchain/mfc/test/test.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/toolchain/mfc/test/test.py b/toolchain/mfc/test/test.py index 9732e3d1fc..3d2366864d 100644 --- a/toolchain/mfc/test/test.py +++ b/toolchain/mfc/test/test.py @@ -115,6 +115,15 @@ def __filter(cases_) -> typing.List[TestCase]: should_run_all_tests, filter_tests_by_coverage, ) + # Example-based tests cover no unique files beyond non-example tests, + # so they add no value to coverage-based pruning. Skip them entirely. + example_skipped = [c for c in cases if "Example" in c.trace] + cases = [c for c in cases if "Example" not in c.trace] + skipped_cases += example_skipped + if example_skipped: + cons.print(f"[dim]Skipped {len(example_skipped)} example tests " + f"(redundant coverage)[/dim]") + cache = load_coverage_cache(common.MFC_ROOT_DIR) if cache is None: cons.print("[yellow]Coverage cache missing or stale.[/yellow]") @@ -225,6 +234,9 @@ def test(): if ARG("build_coverage_cache"): from .coverage import build_coverage_cache # pylint: disable=import-outside-toplevel + # Exclude example-based tests: they cover no unique files beyond + # non-example tests, so building coverage for them is wasted work. + cases = [c for c in cases if "Example" not in c.trace] all_cases = [b.to_case() for b in cases] # Build all unique slugs (Chemistry, case-optimization, etc.) so every From 5c4eaabb89a47c8bec3d78266e49431dcffa63b5 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Tue, 3 Mar 2026 17:04:20 -0500 Subject: [PATCH 13/35] Cap bench script parallelism at 64 to fix GNR node failures On GNR nodes (192 cores), $(nproc) returns 192 which overwhelms MPI daemons and causes SIGTERM (exit 143) during benchmarks. Master lands on a 24-core node and passes while PR lands on GNR and fails, making benchmarks appear broken by the PR. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/frontier/bench.sh | 5 ++++- .github/workflows/phoenix/bench.sh | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/frontier/bench.sh b/.github/workflows/frontier/bench.sh index b60f8541a2..b896feb17c 100644 --- a/.github/workflows/frontier/bench.sh +++ b/.github/workflows/frontier/bench.sh @@ -2,8 +2,11 @@ source .github/scripts/bench-preamble.sh +# Cap parallel jobs at 64 to avoid overwhelming MPI daemons on large nodes. +n_jobs=$(( $(nproc) > 64 ? 64 : $(nproc) )) + if [ "$job_device" = "gpu" ]; then ./mfc.sh bench --mem 4 -j $n_ranks -o "$job_slug.yaml" -- -c $job_cluster $device_opts -n $n_ranks else - ./mfc.sh bench --mem 1 -j $(nproc) -o "$job_slug.yaml" -- -c $job_cluster $device_opts -n $n_ranks + ./mfc.sh bench --mem 1 -j $n_jobs -o "$job_slug.yaml" -- -c $job_cluster $device_opts -n $n_ranks fi diff --git a/.github/workflows/phoenix/bench.sh b/.github/workflows/phoenix/bench.sh index 0eafc485d1..89406942f7 100644 --- a/.github/workflows/phoenix/bench.sh +++ b/.github/workflows/phoenix/bench.sh @@ -2,6 +2,10 @@ source .github/scripts/bench-preamble.sh +# Cap parallel jobs at 64 to avoid overwhelming MPI daemons on large nodes +# (GNR nodes have 192 cores but nproc is too aggressive for build/bench). +n_jobs=$(( $(nproc) > 64 ? 64 : $(nproc) )) + tmpbuild=/storage/scratch1/6/sbryngelson3/mytmp_build currentdir=$tmpbuild/run-$(( RANDOM % 900 )) mkdir -p $tmpbuild @@ -16,9 +20,9 @@ else fi source .github/scripts/retry-build.sh -RETRY_CLEAN_CMD="./mfc.sh clean" retry_build ./mfc.sh build -j $(nproc) $build_opts || exit 1 +RETRY_CLEAN_CMD="./mfc.sh clean" retry_build ./mfc.sh build -j $n_jobs $build_opts || exit 1 -./mfc.sh bench $bench_opts -j $(nproc) -o "$job_slug.yaml" -- -c phoenix-bench $device_opts -n $n_ranks +./mfc.sh bench $bench_opts -j $n_jobs -o "$job_slug.yaml" -- -c phoenix-bench $device_opts -n $n_ranks sleep 10 rm -rf "$currentdir" || true From 16bbe6edd9c3cd5fceed6f402e5e5f04947ae6b0 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Tue, 3 Mar 2026 18:36:16 -0500 Subject: [PATCH 14/35] TEMP: exercise pruning in CI + fix missing sim coverage - Strip case.py and coverage.py from ALWAYS_RUN_ALL so CI exercises coverage-based pruning on this PR. - Add TEMP use statement to m_bubbles.fpp to demonstrate selective test execution (only ~69 bubble-related tests should run). - Set t_step_stop=1 in cache builder: one timestep exercises the same code paths while preventing heavy 3D tests from timing out under gcov instrumentation. Fixes 80 tests with missing sim coverage. - Update unit tests to match TEMP ALWAYS_RUN_ALL changes. Restore before merge: - coverage.py ALWAYS_RUN_ALL entries - m_bubbles.fpp use statement - unit test assertions Co-Authored-By: Claude Opus 4.6 --- src/simulation/m_bubbles.fpp | 2 ++ toolchain/mfc/test/coverage.py | 13 ++++++++++--- toolchain/mfc/test/test_coverage_unit.py | 12 ++++++++---- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/simulation/m_bubbles.fpp b/src/simulation/m_bubbles.fpp index 0f17bd60c3..5da422f3ce 100644 --- a/src/simulation/m_bubbles.fpp +++ b/src/simulation/m_bubbles.fpp @@ -17,6 +17,8 @@ module m_bubbles use m_helper_basic !< Functions to compare floating point numbers + use m_helper !< TEMP: exercise coverage pruning in CI + implicit none real(wp) :: chi_vw !< Bubble wall properties (Ando 2010) diff --git a/toolchain/mfc/test/coverage.py b/toolchain/mfc/test/coverage.py index 68f7691c25..a9cfaaad4d 100644 --- a/toolchain/mfc/test/coverage.py +++ b/toolchain/mfc/test/coverage.py @@ -40,7 +40,10 @@ # Changes to these files trigger the full test suite. # CPU coverage cannot tell us about GPU directive changes (macro files), and # toolchain files define or change the set of tests themselves. -# TEMP: stripped to GPU macros only so CI exercises the pruning logic. +# TEMP: case.py and coverage.py removed so CI exercises pruning on this PR. +# Restore before merge: +# "toolchain/mfc/test/case.py", +# "toolchain/mfc/test/coverage.py", ALWAYS_RUN_ALL = frozenset([ "src/common/include/parallel_macros.fpp", "src/common/include/acc_macros.fpp", @@ -49,11 +52,9 @@ "src/common/include/macros.fpp", "src/common/include/case.fpp", "toolchain/mfc/test/cases.py", - "toolchain/mfc/test/case.py", "toolchain/mfc/params/definitions.py", "toolchain/mfc/run/input.py", "toolchain/mfc/case_validator.py", - "toolchain/mfc/test/coverage.py", ]) # Directory prefixes: any changed file under these paths triggers full suite. @@ -361,6 +362,12 @@ def _prepare_test(case, root_dir: str) -> dict: # pylint: disable=unused-argume # runs via ./mfc.sh run (see POST_PROCESS_OUTPUT_PARAMS in case.py). case.params.update(get_post_process_mods(case.params)) + # Run only one timestep: we only need to know which source files are + # *touched*, not verify correctness. A single step exercises the same + # code paths (init, RHS, time-stepper, output) while preventing heavy + # 3D tests from timing out under gcov instrumentation (~10x slowdown). + case.params['t_step_stop'] = 1 + # Adaptive-dt tests: post_process computes n_save = int(t_stop/t_save)+1 # and iterates over that many save indices. But with small t_step_stop # the simulation produces far fewer saves. Clamp t_stop so post_process diff --git a/toolchain/mfc/test/test_coverage_unit.py b/toolchain/mfc/test/test_coverage_unit.py index bbe972e0ed..5053eb25f2 100644 --- a/toolchain/mfc/test/test_coverage_unit.py +++ b/toolchain/mfc/test/test_coverage_unit.py @@ -238,10 +238,12 @@ def test_cases_py_triggers_all(self): {"toolchain/mfc/test/cases.py"} ) is True - def test_case_py_triggers_all(self): + # TEMP: case.py removed from ALWAYS_RUN_ALL to exercise pruning in CI. + # Restore before merge. + def test_case_py_does_not_trigger_all(self): assert should_run_all_tests( {"toolchain/mfc/test/case.py"} - ) is True + ) is False def test_definitions_py_triggers_all(self): assert should_run_all_tests( @@ -266,10 +268,12 @@ def test_case_fpp_triggers_all(self): {"src/common/include/case.fpp"} ) is True - def test_coverage_py_triggers_all(self): + # TEMP: coverage.py removed from ALWAYS_RUN_ALL to exercise pruning in CI. + # Restore before merge. + def test_coverage_py_does_not_trigger_all(self): assert should_run_all_tests( {"toolchain/mfc/test/coverage.py"} - ) is True + ) is False def test_cmake_dir_triggers_all(self): assert should_run_all_tests( From b79f6c6782b2a2ed964583573bc95dc31bec1f4d Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Tue, 3 Mar 2026 20:32:54 -0500 Subject: [PATCH 15/35] Pass GITHUB_EVENT_NAME to SLURM jobs for coverage pruning The --only-changes flag in self-hosted test scripts checks GITHUB_EVENT_NAME to decide whether to prune tests. But SLURM batch jobs run in a fresh shell without GitHub Actions env vars. Export the variable in the sbatch heredoc so it gets baked into the job script at submit time. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/frontier/submit.sh | 1 + .github/workflows/phoenix/submit.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/frontier/submit.sh b/.github/workflows/frontier/submit.sh index 16d4f0d73c..eeec87c487 100644 --- a/.github/workflows/frontier/submit.sh +++ b/.github/workflows/frontier/submit.sh @@ -85,6 +85,7 @@ job_device="$2" job_interface="$3" job_shard="$4" job_cluster="$cluster_name" +export GITHUB_EVENT_NAME="$GITHUB_EVENT_NAME" . ./mfc.sh load -c $compiler_flag -m $([ "$2" = "gpu" ] && echo "g" || echo "c") diff --git a/.github/workflows/phoenix/submit.sh b/.github/workflows/phoenix/submit.sh index b52a107cca..b2fc79132c 100755 --- a/.github/workflows/phoenix/submit.sh +++ b/.github/workflows/phoenix/submit.sh @@ -77,6 +77,7 @@ echo "Running in $(pwd):" job_slug="$job_slug" job_device="$2" job_interface="$3" +export GITHUB_EVENT_NAME="$GITHUB_EVENT_NAME" . ./mfc.sh load -c p -m $2 From 187f3b8afc835d5e5efe7f20a1c9edcca14a47c7 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Tue, 3 Mar 2026 20:33:23 -0500 Subject: [PATCH 16/35] Update coverage cache: 100% sim coverage with t_step_stop=1 New cache built from CI with t_step_stop=1 fix. All 459 non-example tests now have full simulation coverage (previously 80 were missing). Co-Authored-By: Claude Opus 4.6 --- .../mfc/test/test_coverage_cache.json.gz | Bin 11779 -> 9585 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/toolchain/mfc/test/test_coverage_cache.json.gz b/toolchain/mfc/test/test_coverage_cache.json.gz index ae0a6d3c0aa856bc73d1a16b6019f3406d9fbc09..ea29b5ebca7aee369995ff8905fafe2647c4bb28 100644 GIT binary patch literal 9585 zcmYjWcRbbq_peA~Q&vW1$cpS$Zg#ety+_E%xP++e?3F#Ua^=e26xn-TGi8PAQf4ml zd%rJzzQ4!)L+^L5*Lj`sJm);m>&+B_k6-UwWPjz%!^Yji%*w^n#?9iXjhU5&m7NWr zgS(6KrOkOY_xbL39N+!U!mpRvS{9lRu*o#z2vSzA&-Wv&0v>AXQRK;Mad6e&WH?>8 z#$6(vw8E{zV(}|}NS#}bXJ=>W?cs8d``@K(f6Tg9c_K%M#OW5P>?sO!xZj%5-1=g7 z@n`z%m>#u@+38COk8-Ba8iaq#3*W!1rxEQ#+X!c8jc+DqnM}^k{xY)7;wOJs8ogSN zvtRv1iXwl?W}BIqnl_*I?IsufMCysQ?hLQ=Oivuw@~fB0W}O_ho@}`PEF=AS=2_Q* zDo2KjZj&IrnvRw?_f{8A^q3l6dbuLcR@>M2JJzSu+Xpb;XGud*=@Dg2e;0!>eRrOB z>F}$&XlNk5EwuQ)@h3@<=67U@zk@` z@0FjY&!OZ=HUIKJ$I7v2>&dZ3*~;{hOY7-&>!g{~?k)cPx39{JoujL8bp30%E& z>1iNNh7CJgqB2%kAgzUMpa7-i7%6n+Qt0`b$~S_i#(MLb^`Di`RhVA<^Gbhvvfn-j zkN^SRt^RmdTqHhewQYPM7T97%)9zDlTll{f>q++b^^w?9A+)9aM*Z(oj~za=2LQI> z38esL*$I{$7nG@hAitIAdw&`sKRh>8%t|DHU`nz7gMUPR$nspx{dbE#5ZHZsO@ z8D`B3xfrS)-Oj}w;Nj@tK7tKQv^|L00&^ju7CwRsfa{YZadI`oGT-N`pN;u5X7*n{ZM5`Q^r zbgF486o8;AU5yqr+O)W9k=jSua1XOHHnj0$xuc^mITY1CyW2icbK9g@tm(_dZ+DpM z0fXfibV}Z2dV6SldU`uLFZma;_0HL9xoVCKm1eS3Bvkl`<*jQSA{B3)5~sg^fD_K^ z-^{xSDlP_pKawT?mASS!Jgvk25i4DfRO0l5R6gW@F;qkt#3f*lh$aW0MXI6NcWjWY zblOGI=^D6;#Osd2u|aO@r=nEWNeh{A)4@VHx1ZyhkY9Gu#5t@02>?3F)04bux#-{{?ShHq3BN+F%K8p znfQ$<-Zm_Povz3YMd#i*vR!eX;e~TZ(W^HI4k`RCGi+RBS%v7&jw|ZsEaK0MZ=6{YT%&~@&ARS zP?yS_UqR(4KD%(teyIb@5M(YigT#fmLqH8AIt*sMmji#=4A zM@8B{%4axQv!+wI(xZDv#-ZhR3z0wY6~(l7barNom-SQ}-LgxK*ugzC%+HW~t;3!5 z_PBs%z?!xZ1bwNyft8-5NE>|Sf`V&Rz_z5g&!NxhS7glk9=ZglGNps( zM(&!wkX~N{9al1*d^|Z&g};|B&V1g^n`9c|<1atJsfM}$s+C~%m8aJN_Z+*YZOYg2=wi|mLVV5*S+hvP39sIM^qN#SXxDPJRr* zcy6F?R=|BSHpFt3y5AGo{AKR_C@YUDd0DsXH?o#$ugi}XARZyO0T$aknqo8i+)(Kb z`^ZcN=4BS@dR^2ezIgQh7idZjARRdzm}fG}(!e278c{C>1U`c=4&Bp#V8jm*zPFKn z8vvZ`5&x&G&kD!-8&d2`W4VBv-#**(h_<3tFFV&S(zYoBi?-ShJSiIJiGPe=Sco%f!?{e8J9KUiLoMcrZ35VG31CRJ`x* z4J8cdf-w@QdlW3QhZg89Mi+`yUw)?FPB-}}r25Y18RItNP(18&WYh5SA6=?w4{vpM zPn>=m>ysZEzO3P%ga$B!Ro8VbK5Rit4<^ljUhM2_U`|M=!`LnTY89Mjy;QlPvq3k}vN}ip z(J|+Z@srkg*xd%WWYsj`S5?R7(1P`(>f{V;S@TdGwj0tm7ccyG?ihpV)Z@LlVd|7Z`6&8l~5+mu)7+C>$wBBmwd$l8XJnijjqgG4%ep%8R-dz&WIN*`w63={6&q0AAF+5p#j)v~ zgF$2muswL`fGp}@>_d#7&c+~``GOkPr1T}orG(!5k8FI296=P%6u2tBJNHDTe8_Js&A@f8q)}&nLkOr+Cmvgd(~2++FyuGZJv3at^jG!U4@OK7g3NZA^k~UF;X$^0@in2WFPHEHUSB54 z4n0dQh{}aoUIrc?TJ+Zcp=!5El1W6kpXB;JKBOu4h*@W>{D|(td@AzqRug0jeYZn* zx-9JN6mcX5frozNz93}NcefMk843%`K$;z=v=q=8Y-XMU;m!_u5%480o|U~@OxrD= z&GJ;g+-0+CwC?^_74iswmZY%#BN2Pif`%XxEKoD9tP8*R^CvFE-=RVMM6+9`n>9%; z3;2DDA@;_%NGW2DP0VLw=TcVYcJVJ*9*!Oxr49c25UN&D>>#6MGpVt-vVQbq`QX>V zDni5ZXig`8 zaLykULfd5+NUUgg#}%N$6mbA!iTA3RLQe7SjNTY1g*NM(n~H42Y8kUrY12b0P&G`kL+>-A{v5kT?xN8P>R z!Cjtu<(<3#8^FM8nBc|A=e&}(*Ky?PNR`g~&Ev8QI+fl}T9$0968!%%c|p%9l`AjL zNKv%h!vnQexU82DjiFhU;7?r6;ZYDVh!nilpR=|@FF8*LifxfkWz||L#xqjdGc*X*q=EEDQZ6MgaYy4Gyzc|5(2 z@g0i-`|br60e~ivc|3{+*|o?T@6DYLN=3l$8m+v9yaeUGzvFvT2e-AK(fJ>5l&=-V z7R4U^>Rql!LG01;`RAp?F{j_S$eWS6G8!~-?i97Lr_-}?uy|taM;QHl@3XXH%zp=^{rF9qg?usy`=Na3K@_X$l zwFy`%b`F^Hb=zKd4{g^?Y8X+x2Byp-*ZeAZrg@3Kd{O^~Ud|x2ha`-yfrAR?^?gx0 z=bW@vyaD2!89kLJCX(aOrEiqiR)j00zc&|E9zONL>>Q;gXj?z3kp9&=EzF6>T~?G;*?`M(MB}&S?_X~nW9Vj$ zk;)2eAPb?t9qoyRXQxdei#y`t3*y;l5_#SMzgKpjob6`{9|gh;xTZ#iPVZBo*|Z#1 za}B9>#X!`EOXV$47C#A*SREgnk?U3vsaS79dY>K9Jae3#8ZXQbyi8<1b$92aPgKUU z!k=~l(FLW558C@grVgcFbfn_Pe~2%K4i=lKHBUx4Y(0bcx&qW#Je*}#r@m{VOsiy^ z?q?4m)c$a6s}}h&l5ZKo5XI^d+;@`?Uf;HIJ-O)sW?Hi0<3Pp0LZ682E`&VFe_2Q= z8{*2ST?WAox{aIL#%@zH6&JrsmcWpD4(!zC=&s2o9q7GAp~qm^QIx7UAW~4HgZtdP z1~7H2-QTt-aU|l*-#5Hx75a!M`JOMCSMPK(EJhC=QUn|@v`Sn-?WP>}D z;JB#`b7Im4*e0+BO(-@~nSm6s>Q5-gicQ?@1~!$@h5>qHYYgG?@p0kUsXI{ zH83Wu-hA`RyT%~MAw&D^ZCqyjhn&ej?1scE=Eaik-EsuibYJ(myC$pAbgK(i(QZ9R zM)Ua+UM4xqkk0y1dT0ZQi5gZHJ_%Zbs-f=pExJzB8**3^1_NTNtpviwCvzW*Y%c;6 ziO-+-Y`1TI}rHsF@NQkPgK1|oH0s-E4+z%WhczaMTw(iieBLcT(?Ix znkfu?@fx`jT@k)f^q`~0?@rv!e$?#7H89+-U}H}FOn)q!Q_5+?HwDRjW8mX-dvS`+ zJxlSM?%b=oo1tt@g${#Dj_hn;me4wJYRim`gbc)E$T-LX$bxR)7YuUL$k$KXuql?* zp}UV7KbG)c?O#YPw}k5-?x-){cDWrJ?HB%Gev*RIqCCcrA}OevTHa~F zqx^0M*E<0`w%eC`xF%db3)lGB>5%e0e?w^e{R&d+^Sd;}cBg!L!y;2fY%nm4Oqz~F z*N307KzcMET%m}y{;%pIXNHvi2{*BT4$_J$h^Ev278AOKwtQ&`A}vmg(Vms%BW^Tt;- zfeC}N6iAG_cnxpC1xTTJT37sk1uj@gwi&((pVT{g{_|Md_@#442fd0JH`3Q+Tb`=I z!z9P9)^@i(^;oQqogH$SK5*~vbL*7W&k0YF7ZDzQEKZ1IU%us>rz&N9>4 zpP~TR|JA!s?-+?3U*gSlO zOne8in`uG6#sy-htarOEdMWyey1RY)Rjl9HxNf5*30Xs$mIt46+y+dJzfl*BgMibf zIXgeYhBrRn|3-cvcn1;-JLB8a9~JwVg>9i>Y%FNHxU^!2F6YN!kiiqs*R}NwNon(lxdkQphx+E z+afL^?{!If30JLCVi7p&_@Qi#ja;?^N5DIX(vEq!+*G1wQqupX#nSEejF~x3zBW?- zU+yzD#hopx!VL}vNhz#g31t!Aa+`zE=W%z6CA&KRhxzMuh2AQz+!WMxpTRUhDNQmA zp2k}~qoM>~gz7z*@fI`B+R=qw1)v9dG~e$j&xE^aOnzk5A9b5=&8D)y_pK!SjyGPP zPnQj5W@ht|>`$kaM9$Y|M!;0n`$mm;uU`1Sp0rG3LhMio8=lKOCUbtVL5zsbE{UkzCL?t2dP#dY@T2|5z`&1#5!FAa zg@jkTQ|y{dYCWzdn=Lm~3E8CxNCPMsV-L0{i2Rt2JHt6ayps}+#=Uk2$BzIHDR;A? z8vc7cArOHeW+DrGNXx5qc}|lBJ@DT&?*$44YFsQF2O;;bNz7dWW64bA6vc~~W_!6G(n+KWU8r7+g83z|N>aqOFyxa`i~iKx zKBTeB>aq-de}IoBpJA<1w2O(ovVF}VyYGJf6P%_SoLA*Euj7s`X}l>PM7{6klT*E( zc%|S0m*(}Ltx3ItlKL#gH96{r;b1YUM_7i#Lc8K}7HCQ7Ek)TFQ{07&Sj& zk9%J4F5BzHuF|N^drs0^Id_C7Bgb&0I4>90FY(O{2OIun%&N#5(`e1|mp=8u?AJKQleRZim412pPb25nEm1Wq6O?ig;0r)y2a9^-pDh8> z{o$F=AAd%Ev*PaSUa>ceO}kNoNWC8W{~}ssJwqZsp6RXv#5gb~=vuztLzMyDvXUDt z=T(3}sWCwJ9fHKl!%wb9>*7cL;qY>`?t~bh)ivFm3tw(0XhM?Pj9%!}TJEhBia3xx$>A#m^CD3Ia z{iP;*d1?Z6Td@)YK#MF<`*ziV+8c{OwdO}Q$^g_Z5I09Rk60E7>Sb^`haBRDZ`lpW z;08v#9PSeC5;;wSn@TaVS4o_z_jHPL&us%by?m6aPMUu)wg&YGiz4S*j-Pyy(#_?i zgwQ*Q&$v~s*!7JhX?kXBQgzMkzx&-Im#T^P<*7FN%cGqGjovVg7piJwkN0Uh9>KMu z98{uIq83{9ZE2G-9I(lTEeq6-*V;yF&db6zjo+KUsfR2-td={>dj?Qd<*X=$Im9%* zwdkD~51IaGVM!jLZ%Gx)_^SW9m0p+eR~|43mQ)j2uN|P*Ps!r_-N1w1(^N6a8lLJ; zjQLd37!Ct+7%57}G+IZT`3E&7Yfn}KpKcFrd93aV-6?oI$~@D~GBtQQE;@e&7+OHe z`n?h&AnF?J-V3l-T;@8$fMph>S06M=l=9QyObfBU&+3$0xp79{H5bSqgiGZBhp#X!`*2NON4)sgZ%RwVwh- zS?0JpQn>M3infBg(akw(!X2?u9JU-nsYf}sbd5tbT{0^#-a_X!O8dKI|Nb8Gr_+>a zC7UAJO@TNVzX6AVgVG)n1?L*+L48r4Q_Y6&+M*!kCBEY}Mg?JTk5+???Tcq14Fbl6r|C^&iC}7 zluKd4AY1PBZyUpl`sEj1?iEZ3)R0m4%j+wBv$Eu<_2|zGIvHKLcXjjVCO+6!zO_5t zd~X6x)`imm10=rOFvC;8p(H{OJGgfZMM7%7^=dmGNu$s2-^(i#`$qsl^jQ;5g3?(k zCHaH?8pSOp*x~v#8LXYgp}UjPIG*w~pOMO96Mx*$GTp5#oD&T2jK=LhM2C)pCbA+? zZ&*P1GaYb7IXU_+dy%CHs@>~Idv&tJbQbggh>?n&Y&APC4mi%@OQaoR){#sLCZ*jk zcj%2{R#_eh9iXN5FkIk=D8FaQ+To)dM`EbB;!=nt$@$1q&-}WjNPwfBE>w&-dKw=v z9xec<-)f333|Gy1W>pJT9chW!$?W(0P?@=-md^!Ye_%6dJ5{Tt^5k*)Us;my+2x#a z9t(M>={zOx9Vd0riZC4aIdSO7jkASTY{R13$wk8)#3q z-xaDIkcyd(I!8s-Ym^e^${796arW~RX0)Dl=O#t){)XCNqOlhDLvUAiZ-p2=k#EGGm;1;m`KIG{(eqZqD?THREnd z(UT8PUfc-PgKVM-{M1RrzsXSn)91$}wvrw{#+iEZOtG)bB(7l)UoqWkr9!=k`N9Swa`*-wIzb zj|%^X>Uu$cmxj(3PcYAYd|#CR4PBVX-TEv3^%cp$8Nm-y*luXUsUJfZN6nelsW1N7 zzU&d($Usj+_vt?W*~aq^gQgkKADU4XA27KNW*H2FgQ3sAA(y6)yme{^p5sHk4&E~i zHCl+dG?q(5@ne3kyrB7uYHpEe#XR3Rsy!?^JYL_HJ*_=E-P$`@@W+^OogQYxF^9ho ee~jJP$v)f5J|n$!>Ff;W(s#tHAQ_tQ(*FS;*(eYI literal 11779 zcmYjXWmpv8(gj39>F#c%Lqb8iyBijkmhP7BMp(KVM7pGv?rxFpZh>zXy!U?3%n$Y# zyYD+Q=gc`TX#@fS-BF$e%##z)(aFHr)&=NbWCk=aHZnE`GFv*@+CUvF%h(c0#s5CN z_X-!za{11o_lV?roF;fZ(XhN|C`6SIrwq6$l~w-OW!QEPSC)@af&(&P2#AXjgu`=6 z5`HJ(f5pM^-u3C5@9k!CJA#ol;_%a^ZTsK*lhN*W-gX0h{lCp^Y2KRpzBkTW{D=Mx z(vQupnFcEc7ws!6<;Np78t>12empu0=v@2iEv-Cm8a&-B?oa=ny&ayO9=^;PH5lG1 zOS|Mi)yv3OT{18jQqLNFym>z=@cwCT^eNBav&Vt4ulwRR?!Tw=x7I(>1@FhE4;?0b zwj?`wveidU%h!f`YbgHg0O_u4dN_ECMc`E?hc~ldk4JTASO` zmzhYP{0zLdf{sTfx4FG-Ge7(|cW-_0 zX?nYDP=TabNB^=r;o9eeS#}f4Bb9yp61_J`N384vLtuejV>PqzV z%DiC;Gjv>lsEPX{oCMi6TL0}r!jA*N%jEq>n`z>Wi(Ps>iXLcwjB#6X7435gg1KCK= zIfO8vNuIzw#ZDK%PFKULJMNx$UcVY4&y7_={E8qn*gUj;Iuj-tSjLMfj6fcUCX7&l ztIa}5RmyC2Xu{L9?t*C78%L?7BC~e5d01d3HX5Eetxy0%J*7(Zn)dZJKX&@+%12vg z&$Ais$4qU?*aDLs2KN@`g{n~^{I0m&}xdW)oN4B*`o4f`?Xlnqn^Kg0J||AW)MN8 z?DRVmwBHDJu#J<}JEF@azf)d&W0=`8-^pOuM`uZJMQ3f_xKnv&>3}6L_Xf!UYR1K zpso`7pEL^HC`B@fbA862q8O%zMAy+aj2pweM)$*hZJ?&xr}pdVV>C}JszK~wGq%hE zSxh$S=b6)vFl!^Pat_l+blrsY<@KY*%QGn#$NOf0ahBJa^pE$AT`f<0uD(UbyD?Ws z(n`usdKA7d0l19RHI39QRNbdkdYewKdqT-$`NX3N(0v>gk_(}XEGg(-2Al)#sv zi7h; z+ijA%%nuFW3mSq&xuEx+&&T$joG4FSIXiCm5DGef3c9mx)H9vHDs`my*dIKbi_(WL zyvB7t3dSO^eQ>GMU9Awn6bU8oL1S0CBZeP8#$y&87IF}K5BZ691Dr&114?EY_VN79 zYoTExCxAb(VcjauV~iG!f8pWj2VmL9Qh+q! zd0R%O9O>IAdHKArsr@8TV+c`Wp{ejdam8N}w8nsa3dlcd6$ZUhGPyN!j>iSwZPYOY zxygSp!-?J*eMeMdm)>@*dzc_C?=)8xzX1QRQ$;>bHvo&RJWRCj|G!SjV4*Qf6vBT_ z5PG5%01vn7Wejw83fgfpNl4!WgCu9; ztF+i8-^;OtNguVhpM9@c>vj8B4uBy?J0cRMGy!m8K&ofHB2YZR600>jv!!|*a!FgH z1CcqA@w~7snIprtcQW0P(O1O#%44u{;9ULa+@5LRKRddZAJVOyV9ol91Dt6f-ozFu zf%2=XGuMZzobs) zZziq@6|v8{+h6z+-;-7I64-O|&E6kgPA?DO z=IA+ru_J_z;F^_W;f)ZQ7VQxO#qU^QOyQmGu@HRf3gmjar1^trNICL<$Rv&ay>mz; zR_5e2gf8MO+7&=QpEq2Ah%!wBq5x3@Dpn^@M#WP)E4wBU+e3OJfITu=feRBoLA{f0 zGelG}P*hU%2S+$NHA?cd(lG?uhGI~pu&u7s+5K zz?s(YZbjEiKxe6Zi%^(vZp??Yh%H`PO<15sL~_K`qiv@mlQdMb+gM}pa(R!V(1@T= z3D0O+h;v5E4@oN80G^7_Mr0zd&Vkq3H51|>M8VbY1<2&J`y852+(`-1)>|cA(1(P_&YI2KoI-8=ne{fGDZk)X;vP`{AJTbJ` zwb%!$FrXz=998{i;*t<}{3&^IcZ9%QIxbvDRw4e}w7iBuoQM|Rj{)hKR}C23{0LlejnJOmS0JlJhl1ToOR#XQe+#+ z=&An&F*W_6Ds7M`exi{NY{J!8{+g@J4fC~G@BOK>_g27XkQJ zpDbpLsMROZ+<|nhd^sz_gu-EJ+F!St-UYRFJq|-H%Ij3vp=|Gbz#!MR@7*2#xn1*f z*s_h>m8Sw_bNa!S&3JD4=%B5%NUjwcCf~~)$ z81N_ZMxd5?zjG~&HI~8CR&8=9Lxu3uRzQCmLlE%G4-)k&^?eGSKJ@VwncCiC{m>|p zF!$@e!oI?}X0P($rL_pnp{<8`J>MTI7hF?&&|2O{e#Xy$+pI;bG@CX5cT(Q!`@OE^X*E94~SoOJZ zR^wX$)&nU`M7T}z(n8qt=q4?P*dcxPoUQ60+fWnoc+}UTpJ1QUBv84)oZ8al1nPOd zpQY7&jP|c9b&~)32NA+VUHS<0JI8T|@X59R{Dy_pH=va;ie|c)fJt?0!32gpfl?d9K3bOaCQQ*>jT9j0BQ8^ z7H^|vU38DuwbvWQ@&XUir2R&JuoUoE_fUm88GaR|F9RdOD~YQ=?SVR0+dkLsdPIfL?FJ zObGZO(-KVU?X^&nHxFEwHykUw=tV_1!8wwa)0SG@#=tGZ1i5E~sGUDK;B5eQQ`tC= z339QKcJH~FgiO-J>*+r42%CN*ZYS{kfI|voAh$Pzxy1#?tm^ITf5_+&%WS??zEw-` z*nBn+9>PUGNwyFIe<{*@J%}>%#e4q7{9RBQY4(}?=aPB6aX~ZZ5jhl1U+aKrbUFUv zX@QC0aEc7%bf>qbCKtmWnfsOR%45urS|%xiFWW)>8_f=Z$Fp_&xcZ56NcqAQ-VWo@ zQLytB59^aE=J66fkmUHl@G+@{hQ?aCZObr(Y1N5kI(91VH12e~k18ZDuD`zwGlNW3 zS5a?kj|w~)!O3v1!FY}r_vf?6m7&^y7lBZue~Z8iNV7VkA#eQgCX&y|OZg9BVoOBi z=Cl8>&KD7~I{p8U78)@Y3Nf}GZg+iqF%23NVr<~I|D)MXtyVQ=hduP#Wp^vLU6g4{ zi7nb)bxVojti3`jkhBq9u;L*NO%&DlZ){4wC>*!_C)g(&PNg; zV?N%9*zY527!(U8a+5MNii}DH?jQE?^tvPyEsuw4>OPIIKdJSBFsIOPfRDPI(+gp% z9G44eOY+SRXN$Mf(})MX4b9SeRPYhX%Zkf>dt0GDxw7y5-rnD3;STo|i>-aN7ESx3 z`hd(*gw5^3%~8_lDQhZj`iti^<^?>P3tNYpx8>LJH;`JvJMz4n0?)=U5q?8)-fwCF z_$gl?84h9#$1kg!h;+*Rw_nl?G*+tFsDP)n5J)4iY`(j-r)tlh`!6B~&(QA!CaVf5 zTJSso=gQ?&o*Zce#r_6oc=ypj^gZk|sZu8alPXzS(Opi@H;Ta`A*^AJM!dLRA6!SZ z@zdm|9fT2nc03XrOcrcJ6uHY$NVNDN2(|dRbmGy+a%3D?dS_wnd}=Vnd>a;I%}QhX zh|lpOlE$%;|G!8^H&9!eXyVHTsd#JWV5chixdWcAD07io@`c#R2_xRfrT0`Ny42TV zkhO#FU1X>n7sXeTD(&%o32+7FE@wraG*4naJ`VJb1e2~`vZ@XW>Dpg5B;@S2^?fgU2s=l`h*56ga%8G-Fe3QPG&pdcP|x%!Tea?f4dZ9_r?nVFk~L4Bk4RAw9>4KCD2Q3LbVzL#y@*<9jRLA|Y_ zi?G`vBO-`q-LWowCk#3C;k9sRfsTdy(@V?pl2mMJ#WW}`z+9Q}TuW~nQ@50QvsEBw z7a}h^FMC5`Je0!xcMA$w(nc3p$Te%?egf}~ry~VKknI-^SnNJ#&KJeE#>c|sYMO`c zYHEz@YU*gl9kDv9vs=p%#a}~Da3yd;BDrsIYg89gwygqN&t$Az@e4uf@v*+j_1^xb zD`eEdqHD-g<(&DYtuew{hJlzmvGdVc_ zTF+r?b7RxQD0Ir&byDeq#JtkyHp(|&PF|WV*pFNG9ozQ{2f&_g*tnxFXO4eQVozcn z^Vu+)hvP`vJ5{k~3N?>cIX7DOD$<9)y%|tv@oqe~qUkn)P+|Sr>TK_EQD&#$K$Wb5 z;yH5N%x+9eo#^^C%Yi=;!@z49-3k2(sy=6dt`*KW8uB<=)WY7b%Ju#&_4F)C%J_GJ zrvvhl{~}=u{y!xAf)MJ55Ndu#+!H9z-fuDbUFAQ;8q6-X6{`lGkVeK6j4@@6fb|Jf zkuY{C>AUkl3=H+>m|?@1(Jn`eZ=y+DgA$Pa10{g+n_mt-t-gszew}7qu!mnw`yzMh z6OIIq#5X@Q;ctFgZEV^TOs=muUxx7_u(X@CN~+uv{aKG*L7nhF@K; zcfPr3ROF5yLGhP`typeP1Qpl8+xZjPDZm)F5>ug=ML`}1f=<Ym+HD{`N zQ3S}=_S*bxOzCWls(j1T`oClhKQBZ-B;ntob~gbi(;2s9&ATgM;L;7@Obty#tU6_d zl|IR7f6_7t+~X00g~$kW*${E6dC8fNHxXpH_#R6L9g@6p)l6{VbZ(IL4aUx6Q;$qX z?O#BE5GmH0Vdzx>6pq??yLDPy7Egoud+E>ZYyh|tz}eXb#g@V_iFG<-b`-Zz$?0K1 zbPj02TvJI(Q7SgVeu|QQCXrrUx%!+v51k+fo#4BdKUjR5Y*J@Pw#$QTUq{!ARmods zQ+J1d_ukCZ8x2c+6Aj?f1ZSRGt{Z^}NVoc%b^ZUqw7-;VT4@l%CWkVkLc7NM?Kje7 z**egl_mzum@V}OL zL%Nk;8N@eR%9!*#hrAV+$o5C$uA4HNWUM5T1T*Qydc9OgP1sECFeps!+Gn*JS#|7h zR6qnjrOSH;0tAk6RoI+aWG{t-d1Ql(7z!sAC8X-JL}mgBU;T(z4~rf~v4fSc>!D_X zFCHhS!{nd*Fwm+5T8!z&74VX;*+>0|_5C>7+u2}UDr^Au526+$BtmGsuOZ+pb-l3- z8fTES-I?x28tNF@GVh_5)1Bb#+&ibNayl7t>}N7$t9osV^qwQsxf#oaqKsLH980<7 z%c7Ol`#t`lh1E5qL(wT}yI_RJHPT16xz{!!Y*2+(gW6*$q7XEM4cJ-Bz#laoKd;&VcBl$(EXLv@oTwgoOE>>h3dys*dt%q47+FROVy!%#B; zK%vJ)_i8R*v=gTs>OcyKwY$CErxj>;`+hul&#Kqq6(TkwcE!^= zVZUU?+h2A;u(w!1{(&mww^diZ`*s8LGhKM(rgG!2=3*6rT1lnl0VfVe>w~+1#8eSn zH{SlM{;OU3aTS8RB2`gmwXb=WsmDjfzPV&dv2<=QN>ycD1Jael84Oe#im+meocLmG z&f?<<`G?)#CsTU9PZIGDqi2G-N77XVc&k*7%OS_*x~nr@ziKA)aGfhZai;ni#|y8ABuR)H-dc7ckJ=?8 z)Ik+QJFs}agbv4(7lmr=LK|helxBYn#@*!V$Z$oW;_N>(5d1zgivA4 zJk5kp+%r^~32wZq8-Rj!ZN--@2~ga}hb)tlq{J-E^&E8>u+y=s@}L@!SP=>-E5QpvwjpPI>it64{GRx$dUg$ib3h6 zTu#%9!M7w^p881GYMaH*62)Yyv9a;sATbl>^Foi9Zxw9W|2CV}7Ha?Aa3%*Hz(vRk z4Q~epZ@1$YISySykEelwSr675!HLvEf}jgDUt;uixIgro6$Uvv-8|hqUrBs!Vf-1V>b4hA62CG9zPx+&&zvk*rIg~UI#=Pt zQCF5X+X1YJhpnDes#L0phlz)jCOk@1-&(1=g+Pex>3r#XJB|8Lt}#d759`nvGRyXM|O0Q(@}mqglDC zuxm!46PUsDK!(?3Ieya6u#}}ZQGQUE^$-a}RNCRw-Pf994}6{m5I@G)D>}ol3mXEv z5ajiNbFHnk9A|(q2){m4Q54v)&5&sUl2q>3NQ-K?-2G!*%w|x**Qu*tfLX~M6sxFu`xqV^_WHUU!!?w@hqzj&dQqu* zG2SV8Hz#`t>$=$5gV)L|V;ju0s9saN_E<$WO%uQ=nJ6icmgl0S>wDfY{D~mMdoY%0 zmgJ?ny`JZ1n26x0ENd6A8xfb)(QxR4hi#iEP`8V^G;6)s2hjKG1M*kHuswKSJ$`*e zQ-2~@{d*o@vBmMl(Q(Du1K%Cd?+pF1}Trk>>Z`pBqHrCy~IIDjUqFsiR;>N>! zq!J-eA^P@cBk8tQSI~rrzx}YtsgAB!6dEG+LLqyONz4c62TcrA&7+!vnFO~)-g#QY zs98kXw3os zuzt*2Fs3Q{%t}U3gYqnROSK3x))1m;R{Es97LM7n$yIB~9XE@OD&N!mp#D)1COpHZ zHB;)WR&QJeqs91qdqZ>sJt-nr|DkWK+oI>0&3x!wmnbjZ$K?p#KuWxbIrP7tP}~~6 z7Nud7Mvm9RdeNoH*u_f#!&MQvc6f}?HY~4}3L6^2mb9Xq$t5TxC<3jqh107kv~3Eh zp37}92u-j0t!|w>0?1>3cK^C@0x9JTr+WjC9Fii`M`mItPRsko%*jEVmLqL6e|E)` zP$dMsO9e&y+&hn3W-Enkw?5sRLJQ-YNRkfdx}M{3{2jN$G`%_)Z8G zOQ2EY8z7BV`oe{SDRzeOr;y|XtWZdpF4GM+E7!)wTR9tn`S{i%#4RkQpg(n+PU$4p zA5r0(FNt>@_gJuBP;<5srk`MjgvolP=o}wHz3xt_i9c5T+(@^(JWFu6Y4eQMGo-A= zuQ}k+hwTT3S)CsmWy3^`nLmkpYe$aUO)e66$09!X*jRX9Cmhy$OxDB>)CoWNv9|ge z@kETIq4AalCntm_g(ro{FpQ%!jK52lRbu#60(QW`xGY6tm{~AY$o(rzvQDZl4k1jk zZq?_Np1K%s7Si(=M&PI|c0xcVTjts&|^s^o`FYOJDy@y65m>q}eBiS_2~i`&_8*S0w5hF6!W%Lhd^BtyY_~Nu;GeAGbn$%H`e^%~ zgLf$E=vUw}vW~C@Oh{ACxR7P-2j5qcbIyG(dRJL@gU-ILP(=$LuQ5OPbszIsthqDt z8(EFuGk%Z9yx;Y1!(vDeV_Jw#DA{4DN=nOi>OzKlUk;7@Brzd6A=*M1&7%cdkK$jE zt&1-E&^hzv`{mA$TZ~QnXd@?0KAf+m?^1C%-zmxEAPrRE@l6AYBlPxQy+1-^M2{fR z5C>WakTF}3jjb`(*{EPqWQRs>EdRKf7W@!KEP4M#wAv@y_FCP=sROHl?6l<$sAj|)=Ez1xl zUDUeK0R{OLe{?PTQftLDlGOFx>(kK?`}+~?B=COgV$in~LEEewyrC8we#DG-igz+! zFp7U%*dNZZ8QLFJnCY4;pHBY?V^sxbMWo(ul;qo?7|1&Fv8@OlGO_)>Od6Qa|8NB< z=i5-zGZR0r&bESm66h50$#2v0!_Y8AL*=~a=Vd1kJl0OCe|QII$+$UVn%!+h&Xh=s zIKmxBhDe3@CZZ5#I1L0L2tC_*DfC6Y{J^J~z<3?g8YUv8MdigdJcjaExp`>_KWmC_ zI?~u02!z2uikaVwtR1fEc?DU(t@Xq}_K9rj_~De0x|}Df9Qn^v2V^$VD4r$jw807K zy0R3A*l9^NEHKG(TF4%~s@*#hW+X ziZU{ltFPb;q2QYFU(DW#<2h&!^8E#_yE;)G3ZjRF^|aWNT>AMqi3D)u4KdPC@%W^z zH(Yx-kIO9YW=AssSC{(*M0L+R&h9Y#_LIuSslb1NBk{H~PNK5}>-Bplbn;L%QG_$a zvoW@_F-2f0{(o=;you#4v(OKkI>A#`*S^NGmKQtRs@mXZG|5 z0iaw!uTn76Tq^hb!&{iO@mkh)S>;xng+IsNybi2y*z~WHdkg_5i_vc1ge4h2O0;4E zuGxFq(#ceFtdlX%?-D+b6%`f~7E~x?I-S*7Vv|d&;q4VU4sf9(t+f*|q(*j=kU@Ao z#0g02(ilL~Hq7{n<`7E`ehL*#C)_a~;y0;9%dR2=n;tM6%ubfP7yS$IB}c^`!sPCaLcEc&l^q3c*0fAEplk>X{Y&Q*;totFg1Q*zY??g|+fVkmb z2gLSYb|V*!3orb)bNxF|)#Qtdz)9XTqB!Jvym`US@gj-1>{1L4%v(_y2i@mbGVZsJ z3S%&L(1kd%>&m>$N%wC8uC=XDUrL}~q~Gx!wE}t?kAFqY%c9UTfT7@WzTde<{0Pu+>1g|!&wJc@*L*s@gj+N227c2gLJ-cq+#hRx} zsFca%-o9}GO*lA%epvMg|Ao?VTDN&Zo?=lYJ>1Gp_ZpQ-7jCD-)qpZtJj%A=yM|8Y zrX9Q+?vS4Apl6^{$Vs-#4mIp_iQ)N46VZ4VI+KBs-OO%01an>%g_Sl>J1AjQ7gYae z$=a`v=VSSp@&r-#i&_-`wy{;jO225s0Sgy6j88h6bhHh`#Az`Y~yI= zW?*MsXm=4hlMa(VbvSc~r;nrm4sh5jxG&ng%b~n@-1L@aaH3XT3BWP`g7eWP`@;_s zS4;hy9uNokT|l2WZ!q|L6P(KesKDoq>6GtF!?Bv;jCp+J$>B~?DdrkROb|$dq5B+5 zY#2*_0uqJ(8E#80RV_RQVEy7j~9nr#qT^IPK9 z-BDbCsRyGeZnHr;JX2zlx9xV#c@Els)NY-@yGXsE;h(y0q|6?atq*rff9p?%;B4VU&|vw{U2 Date: Tue, 3 Mar 2026 22:45:29 -0500 Subject: [PATCH 17/35] DIAG: add slug diagnostic logging to identify binary-not-found root cause Co-Authored-By: Claude Opus 4.6 --- toolchain/mfc/test/coverage.py | 31 +++++++++++++++++++++++++++---- toolchain/mfc/test/test.py | 14 ++++++++++++++ 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/toolchain/mfc/test/coverage.py b/toolchain/mfc/test/coverage.py index a9cfaaad4d..ac8d9cbbe9 100644 --- a/toolchain/mfc/test/coverage.py +++ b/toolchain/mfc/test/coverage.py @@ -239,9 +239,14 @@ def _collect_single_test_coverage( # pylint: disable=too-many-locals gcno_rel = rel[:-5] + ".gcno" gcno_src = os.path.join(root_dir, gcno_rel) if os.path.isfile(gcno_src): - # Copy .gcno alongside .gcda in the test's isolated dir + # Copy .gcno alongside .gcda in the test's isolated dir. + # Wrap in try/except for NFS TOCTOU races (file may vanish + # between isfile() and copy on networked filesystems). gcno_dst = os.path.join(dirpath, fname[:-5] + ".gcno") - shutil.copy2(gcno_src, gcno_dst) + try: + shutil.copy2(gcno_src, gcno_dst) + except OSError: + continue gcno_copies.append(gcno_dst) if not gcno_copies: @@ -336,7 +341,7 @@ def _run_single_test_direct(test_info: dict, gcda_dir: str, strip: str) -> tuple return uuid, test_gcda, failures -def _prepare_test(case, root_dir: str) -> dict: # pylint: disable=unused-argument +def _prepare_test(case, root_dir: str) -> dict: # pylint: disable=unused-argument,too-many-locals """ Prepare a test for direct execution: create directory, generate .inp files, and resolve binary paths. All Python/toolchain overhead happens @@ -405,6 +410,19 @@ def _prepare_test(case, root_dir: str) -> dict: # pylint: disable=unused-argume finally: cons.raw.file = orig_file + # Diagnostic: log slug info for first test + uuid = case.get_uuid() + if uuid == "B7A6CC79": + for tgt_name, bp in binaries: + slug = bp.split(os.sep)[-3] # extract slug from path + exists = os.path.isfile(bp) + cons.print(f"[dim]DIAG {uuid} {tgt_name}: slug={slug} exists={exists} path={bp}[/dim]") + # Also show what get_fpp produces + for target in [PRE_PROCESS, SIMULATION]: + fpp = input_file.get_fpp(target, False) + cons.print(f"[dim]DIAG {uuid} {target.name} FPP hash={hash(fpp)} len={len(fpp)}[/dim]") + cons.print(f"[dim]DIAG {uuid} cwd={os.getcwd()}[/dim]") + return { "uuid": case.get_uuid(), "dir": test_dir, @@ -518,7 +536,12 @@ def build_coverage_cache( # pylint: disable=too-many-locals,too-many-statements for uuid, test_gcda in test_results.items() } for future in as_completed(futures): - uuid, coverage = future.result() + try: + uuid, coverage = future.result() + except Exception as exc: # pylint: disable=broad-except + uuid = futures[future] + cons.print(f" [yellow]Warning: {uuid} coverage failed: {exc}[/yellow]") + coverage = [] cache[uuid] = coverage completed += 1 if completed % 50 == 0 or completed == len(cases): diff --git a/toolchain/mfc/test/test.py b/toolchain/mfc/test/test.py index 3d2366864d..b7839930ab 100644 --- a/toolchain/mfc/test/test.py +++ b/toolchain/mfc/test/test.py @@ -245,10 +245,24 @@ def test(): unique_builds = set() for case, code in itertools.product(all_cases, codes): slug = code.get_slug(case.to_input_file()) + # Diagnostic: log slug for test B7A6CC79 + if case.get_uuid() == "B7A6CC79": + cons.print(f"[dim]DIAG build-loop B7A6CC79 {code.name}: slug={slug}[/dim]") if slug not in unique_builds: build(code, case.to_input_file()) unique_builds.add(slug) + # Diagnostic: verify binaries exist after unique build loop + install_dir = os.path.join(os.getcwd(), "build", "install") + cons.print(f"[dim]Unique slugs built: {sorted(unique_builds)}[/dim]") + cons.print(f"[dim]install dir ({install_dir}): {sorted(os.listdir(install_dir))}[/dim]") + for slug in sorted(unique_builds): + bin_dir = os.path.join(install_dir, slug, "bin") + if os.path.isdir(bin_dir): + cons.print(f"[dim] {slug}/bin: {sorted(os.listdir(bin_dir))}[/dim]") + else: + cons.print(f"[yellow] {slug}/bin: MISSING[/yellow]") + build_coverage_cache(common.MFC_ROOT_DIR, all_cases, n_jobs=int(ARG("jobs"))) return From 8a47aa37393430d903c1657f676c90900672849a Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Wed, 4 Mar 2026 07:24:31 -0500 Subject: [PATCH 18/35] Fix OOM in coverage cache: reduce test parallelism to 16 and remove diagnostics The coverage cache Phase 2 was killed by OOM (exit code 137) when running 32 concurrent gcov-instrumented MPI processes. Reduce the cap from 32 to 16 workers. Also remove diagnostic logging added in the previous commit now that slug computation has been confirmed correct. Co-Authored-By: Claude Opus 4.6 --- toolchain/mfc/test/coverage.py | 19 +++---------------- toolchain/mfc/test/test.py | 14 -------------- 2 files changed, 3 insertions(+), 30 deletions(-) diff --git a/toolchain/mfc/test/coverage.py b/toolchain/mfc/test/coverage.py index ac8d9cbbe9..17a0d9e07e 100644 --- a/toolchain/mfc/test/coverage.py +++ b/toolchain/mfc/test/coverage.py @@ -410,19 +410,6 @@ def _prepare_test(case, root_dir: str) -> dict: # pylint: disable=unused-argume finally: cons.raw.file = orig_file - # Diagnostic: log slug info for first test - uuid = case.get_uuid() - if uuid == "B7A6CC79": - for tgt_name, bp in binaries: - slug = bp.split(os.sep)[-3] # extract slug from path - exists = os.path.isfile(bp) - cons.print(f"[dim]DIAG {uuid} {tgt_name}: slug={slug} exists={exists} path={bp}[/dim]") - # Also show what get_fpp produces - for target in [PRE_PROCESS, SIMULATION]: - fpp = input_file.get_fpp(target, False) - cons.print(f"[dim]DIAG {uuid} {target.name} FPP hash={hash(fpp)} len={len(fpp)}[/dim]") - cons.print(f"[dim]DIAG {uuid} cwd={os.getcwd()}[/dim]") - return { "uuid": case.get_uuid(), "dir": test_dir, @@ -455,9 +442,9 @@ def build_coverage_cache( # pylint: disable=too-many-locals,too-many-statements if n_jobs is None: n_jobs = max(os.cpu_count() or 1, 1) - # Cap Phase 1 parallelism: each test spawns MPI processes (~500MB each), - # so too many concurrent tests cause OOM on large nodes. - phase1_jobs = min(n_jobs, 32) + # Cap test parallelism: each test spawns gcov-instrumented MPI processes + # (~2-5 GB each under gcov). Too many concurrent tests cause OOM. + phase1_jobs = min(n_jobs, 16) cons.print(f"[bold]Building coverage cache for {len(cases)} tests " f"({phase1_jobs} test workers, {n_jobs} gcov workers)...[/bold]") cons.print(f"[dim]Using gcov binary: {gcov_bin}[/dim]") diff --git a/toolchain/mfc/test/test.py b/toolchain/mfc/test/test.py index b7839930ab..3d2366864d 100644 --- a/toolchain/mfc/test/test.py +++ b/toolchain/mfc/test/test.py @@ -245,24 +245,10 @@ def test(): unique_builds = set() for case, code in itertools.product(all_cases, codes): slug = code.get_slug(case.to_input_file()) - # Diagnostic: log slug for test B7A6CC79 - if case.get_uuid() == "B7A6CC79": - cons.print(f"[dim]DIAG build-loop B7A6CC79 {code.name}: slug={slug}[/dim]") if slug not in unique_builds: build(code, case.to_input_file()) unique_builds.add(slug) - # Diagnostic: verify binaries exist after unique build loop - install_dir = os.path.join(os.getcwd(), "build", "install") - cons.print(f"[dim]Unique slugs built: {sorted(unique_builds)}[/dim]") - cons.print(f"[dim]install dir ({install_dir}): {sorted(os.listdir(install_dir))}[/dim]") - for slug in sorted(unique_builds): - bin_dir = os.path.join(install_dir, slug, "bin") - if os.path.isdir(bin_dir): - cons.print(f"[dim] {slug}/bin: {sorted(os.listdir(bin_dir))}[/dim]") - else: - cons.print(f"[yellow] {slug}/bin: MISSING[/yellow]") - build_coverage_cache(common.MFC_ROOT_DIR, all_cases, n_jobs=int(ARG("jobs"))) return From 1d33a0a5e8497349ba9982b45fb2f09dc967eab4 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Wed, 4 Mar 2026 09:06:26 -0500 Subject: [PATCH 19/35] Restore ALWAYS_RUN_ALL entries and remove TEMP Fortran use statement Restore case.py and coverage.py to ALWAYS_RUN_ALL now that CI cache rebuild is confirmed working. Remove temporary 'use m_helper' from m_bubbles.fpp that was added to exercise coverage pruning. Fix unit tests to assert these files trigger the full suite. Co-Authored-By: Claude Opus 4.6 --- src/simulation/m_bubbles.fpp | 2 -- toolchain/mfc/test/coverage.py | 6 ++---- toolchain/mfc/test/test_coverage_unit.py | 12 ++++-------- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/simulation/m_bubbles.fpp b/src/simulation/m_bubbles.fpp index 5da422f3ce..0f17bd60c3 100644 --- a/src/simulation/m_bubbles.fpp +++ b/src/simulation/m_bubbles.fpp @@ -17,8 +17,6 @@ module m_bubbles use m_helper_basic !< Functions to compare floating point numbers - use m_helper !< TEMP: exercise coverage pruning in CI - implicit none real(wp) :: chi_vw !< Bubble wall properties (Ando 2010) diff --git a/toolchain/mfc/test/coverage.py b/toolchain/mfc/test/coverage.py index 17a0d9e07e..8e1c5147e6 100644 --- a/toolchain/mfc/test/coverage.py +++ b/toolchain/mfc/test/coverage.py @@ -40,10 +40,6 @@ # Changes to these files trigger the full test suite. # CPU coverage cannot tell us about GPU directive changes (macro files), and # toolchain files define or change the set of tests themselves. -# TEMP: case.py and coverage.py removed so CI exercises pruning on this PR. -# Restore before merge: -# "toolchain/mfc/test/case.py", -# "toolchain/mfc/test/coverage.py", ALWAYS_RUN_ALL = frozenset([ "src/common/include/parallel_macros.fpp", "src/common/include/acc_macros.fpp", @@ -51,7 +47,9 @@ "src/common/include/shared_parallel_macros.fpp", "src/common/include/macros.fpp", "src/common/include/case.fpp", + "toolchain/mfc/test/case.py", "toolchain/mfc/test/cases.py", + "toolchain/mfc/test/coverage.py", "toolchain/mfc/params/definitions.py", "toolchain/mfc/run/input.py", "toolchain/mfc/case_validator.py", diff --git a/toolchain/mfc/test/test_coverage_unit.py b/toolchain/mfc/test/test_coverage_unit.py index 5053eb25f2..bbe972e0ed 100644 --- a/toolchain/mfc/test/test_coverage_unit.py +++ b/toolchain/mfc/test/test_coverage_unit.py @@ -238,12 +238,10 @@ def test_cases_py_triggers_all(self): {"toolchain/mfc/test/cases.py"} ) is True - # TEMP: case.py removed from ALWAYS_RUN_ALL to exercise pruning in CI. - # Restore before merge. - def test_case_py_does_not_trigger_all(self): + def test_case_py_triggers_all(self): assert should_run_all_tests( {"toolchain/mfc/test/case.py"} - ) is False + ) is True def test_definitions_py_triggers_all(self): assert should_run_all_tests( @@ -268,12 +266,10 @@ def test_case_fpp_triggers_all(self): {"src/common/include/case.fpp"} ) is True - # TEMP: coverage.py removed from ALWAYS_RUN_ALL to exercise pruning in CI. - # Restore before merge. - def test_coverage_py_does_not_trigger_all(self): + def test_coverage_py_triggers_all(self): assert should_run_all_tests( {"toolchain/mfc/test/coverage.py"} - ) is False + ) is True def test_cmake_dir_triggers_all(self): assert should_run_all_tests( From 812000934e4a92a2930291f88352e06bd77819dc Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Wed, 4 Mar 2026 14:28:06 -0500 Subject: [PATCH 20/35] Skip 1D_qbmm example test: formatted I/O field overflow on gfortran 12 The qbmm example has x-coordinates from -1000 to 1000, which overflow the Fortran formatted write field width on gfortran 12.3.0, producing asterisks instead of numbers. This causes the packer to fail parsing the output. The test passes on newer gfortran (13+, 15) but fails on Phoenix GNR nodes with gfortran 12. Other qbmm tests still provide full coverage of the qbmm code paths. Co-Authored-By: Claude Opus 4.6 --- toolchain/mfc/test/cases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolchain/mfc/test/cases.py b/toolchain/mfc/test/cases.py index 7835981151..4c385f3b31 100644 --- a/toolchain/mfc/test/cases.py +++ b/toolchain/mfc/test/cases.py @@ -1071,7 +1071,7 @@ def foreach_example(): "2D_forward_facing_step", "1D_convergence", "3D_IGR_33jet", "1D_multispecies_diffusion", - "2D_ibm_stl_MFCCharacter"] + "2D_ibm_stl_MFCCharacter", "1D_qbmm"] if path in casesToSkip: continue name = f"{path.split('_')[0]} -> Example -> {'_'.join(path.split('_')[1:])}" From 160353b73c1045cf9a07998cb83124f477f1c02f Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Wed, 4 Mar 2026 15:30:52 -0500 Subject: [PATCH 21/35] Address PR review feedback: fix type annotation, dep detection, and error handling - Fix __filter return type annotation to match actual tuple return - Detect removed use/include statements in dep_changed (match ^- in addition to ^+) - Wrap git subprocess calls in try/except for TimeoutExpired/OSError Co-Authored-By: Claude Opus 4.6 --- .github/workflows/test.yml | 2 +- toolchain/mfc/test/coverage.py | 39 ++++++++++++++++++---------------- toolchain/mfc/test/test.py | 2 +- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a7ce7e981b..116d9efac4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -85,7 +85,7 @@ jobs: DIFF="" fi if echo "$DIFF" | \ - grep -qP '^\+\s*(use[\s,]+\w|#:include\s|include\s+['"'"'"])'; then + grep -qP '^[+-]\s*(use[\s,]+\w|#:include\s|include\s+['"'"'"])'; then echo "dep_changed=true" >> "$GITHUB_OUTPUT" echo "Fortran dependency change detected — will rebuild coverage cache." else diff --git a/toolchain/mfc/test/coverage.py b/toolchain/mfc/test/coverage.py index 8e1c5147e6..6cafe644eb 100644 --- a/toolchain/mfc/test/coverage.py +++ b/toolchain/mfc/test/coverage.py @@ -622,29 +622,32 @@ def get_changed_files(root_dir: str, compare_branch: str = "master") -> Optional Uses merge-base (not master tip) so that unrelated master advances don't appear as "your changes." """ - # Try local branch first, then origin/ remote ref (CI shallow clones). - for ref in [compare_branch, f"origin/{compare_branch}"]: - merge_base_result = subprocess.run( - ["git", "merge-base", ref, "HEAD"], + try: + # Try local branch first, then origin/ remote ref (CI shallow clones). + for ref in [compare_branch, f"origin/{compare_branch}"]: + merge_base_result = subprocess.run( + ["git", "merge-base", ref, "HEAD"], + capture_output=True, text=True, cwd=root_dir, timeout=30, check=False + ) + if merge_base_result.returncode == 0: + break + else: + return None + merge_base = merge_base_result.stdout.strip() + if not merge_base: + return None + + diff_result = subprocess.run( + ["git", "diff", merge_base, "HEAD", "--name-only", "--no-color"], capture_output=True, text=True, cwd=root_dir, timeout=30, check=False ) - if merge_base_result.returncode == 0: - break - else: - return None - merge_base = merge_base_result.stdout.strip() - if not merge_base: - return None + if diff_result.returncode != 0: + return None - diff_result = subprocess.run( - ["git", "diff", merge_base, "HEAD", "--name-only", "--no-color"], - capture_output=True, text=True, cwd=root_dir, timeout=30, check=False - ) - if diff_result.returncode != 0: + return _parse_diff_files(diff_result.stdout) + except (subprocess.TimeoutExpired, OSError): return None - return _parse_diff_files(diff_result.stdout) - def should_run_all_tests(changed_files: set) -> bool: """ diff --git a/toolchain/mfc/test/test.py b/toolchain/mfc/test/test.py index 3d2366864d..26e37c669e 100644 --- a/toolchain/mfc/test/test.py +++ b/toolchain/mfc/test/test.py @@ -77,7 +77,7 @@ def is_uuid(term): # pylint: disable=too-many-branches,too-many-locals,too-many-statements,trailing-whitespace -def __filter(cases_) -> typing.List[TestCase]: +def __filter(cases_) -> typing.Tuple[typing.List[TestCase], typing.List[TestCase]]: cases = cases_[:] selected_cases = [] skipped_cases = [] From c4db356bc1c2b2bf629e093a7318308f0ea4990e Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Wed, 4 Mar 2026 17:54:23 -0500 Subject: [PATCH 22/35] Filter out build/staging paths from coverage cache Auto-generated case-optimized files under build/staging/ never appear in PR diffs, so keeping them in the cache is noise. Filter at parse time and clean the existing cache. Co-Authored-By: Claude Opus 4.6 --- toolchain/mfc/test/coverage.py | 6 +++++- .../mfc/test/test_coverage_cache.json.gz | Bin 9585 -> 8827 bytes 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/toolchain/mfc/test/coverage.py b/toolchain/mfc/test/coverage.py index 6cafe644eb..0369d5144e 100644 --- a/toolchain/mfc/test/coverage.py +++ b/toolchain/mfc/test/coverage.py @@ -193,7 +193,11 @@ def _parse_gcov_json_output(raw_bytes: bytes, root_dir: str) -> set: rel_path = os.path.relpath(os.path.realpath(file_path), real_root) except ValueError: rel_path = file_path - result.add(rel_path) + # Only keep src/ paths — build/staging/ artifacts from + # case-optimized builds are auto-generated and never + # appear in PR diffs. + if rel_path.startswith("src/"): + result.add(rel_path) return result diff --git a/toolchain/mfc/test/test_coverage_cache.json.gz b/toolchain/mfc/test/test_coverage_cache.json.gz index ea29b5ebca7aee369995ff8905fafe2647c4bb28..0bb928250ae10924246c9c4845c6ecf959abadae 100644 GIT binary patch literal 8827 zcmYjW2{@GB_n%}-_AQZ+o$PC}D?*kmiJGymW2uaN>^mWQ*2un%2-R34gfJ*$>`V52 z-}%2oegD7bJ?44ddGFkN&gXpYIiLGtiX^^`%iG&)J9MmjZ)wx) zsAbp6%eVG?ZEJ0&&$`Cen{-nNLudcpDOTKxQ)<35?S+y3Vf+s5<7WB8eWWOvKl z%HHPG+U8VnX#Vlo@#1vB`QCO+Lp@Sv;zh=o!ui6$Am(hh+o#dXYkSt|IG(`QJ#EhE$n>mLKzesE?XY9Hw=Vg; z^!D`ec1L{e>e!fok9uv|Sc~JC=XP&%>q*JT{6wTc!^nGSp>u6n>GAhYkP~6!11iiq zf~nl+;B)49&i3jj0q?y%@2#!miFz$2kE!{McmiY3WI2xw_4UpIGU>I?2EFx?jbg{A zGbd-GS_Wy2XD3m|fBKHydVR?DYZpD8otpjn#KTDm@sYqq|{<(4X^0q5_-~K3O!j>^f1DT?MG=`(8sCSN51O~cS<;tLZJRaqJ zG+@9~A)y@Q+(s1*~Hk^+(@kCNSZb)#XB=OG}^P z_#au3Vkf2nx8H^af8`TT_CqX0V2#y9pBqXe<6;1N2pBpWsZ7y2N|ko`+dh|E1fqc1 zF7k40l|>X|ayJB@UFyHJ@~_O+8kAjlGE)hv6KvwBmUV0Gr@Gz*dh8jAFmJa^LkNk7 z{$1?+9CbYA<8^#6GD!PCgHPPZ>e~%CV4+J!-i7??!572SM!unH0)M52_lW|yTtc%A zsaGze>c9q!{IIMIc5q>d>=EVrcbi7r#;LM)5f3$_ofe|Om3NJ-e1oX`G27b-{t1z^ z2t>zKLC{sfLp~(`kO;JFOM$nB(NEwYK;R%6JiV%;SNzKB&gE=F%?g#9}2!ti={n?2mlmf^~q`ni1u1ILSY zax$fA#{y@>rz_!5#nuJwK4_SZvg)1aUAAeq=>@5(o|T};N};h3#DZGJO}`uEK|ywr zzji9F2!i6rvB-e+In+CrRQ_diiQltQv?v=3TnUp(xeu%|=`LNf-zBUST~2DM3a@x^ zFC(99F4-6BZn#J$fbt`7jygE3^dA!n5n?>+a4bo%JgGqDo7wSQLIilVwxpI>4$Bh0 zWR=t&Vw>8N^6VffpyF*vu;{^0Mf!lIu(k^FID^v1O`baTI`%N|mg3~?`dy^e2A(n2 z7>#5G^N@O99T~xWi4G>9x&!54>Z&N9^Z~%CI`z4%WXN_TjpHfVUaqR-LerTh=!C2E z*CV{%)(0CE*HrATH%tO{zh=1!mQ2KF{5Oqc^?Ed5TXCVw=%OTAy)h$_Uanj zU%vSp(ZJq{{hJ%bU>D>9Ne;qph3kt*52MYUk0Y7g<}FBl)N zj!HB!+}FP4Yz-`u-k@+m5lxDIR{e41XYflxP6cH&dkK~1{-I(R4i?Hky@b9&1zy#L zRKjCWa?&YSIm^Zt4ku?d^lqrU32uIXKSKSfpwU47QD+7`L5=NmC!us42ufvDy z zoz$|vQQOtE=!3*)dcC$MZkse0Js1nE9l41v(83dxfol%8bCl%wkzjXfpGyb7`AEX+ z8f{a2;|(y8u~k-|*JJOmi}`&F^;kRpu|xV>%+zuQMn2Y;(M%{QVCY(dR0XpqhXSeQ zPro03>%HT|C{FmN1#itXXrSXlh+`X$YUT~5pL;Sz%~p~?sBzYD^Q>x_Q1v_EOIZxa zY%gx4lYneGczoRHy+QowVd9^j@$U%e?6e+G*%2K68#AnR9wllt33AN0nz`j!H>PpY z3gK%Utb#bKf~ulHOcQob*lQQ1P-n!ONN5F;kfhwhj;(CXMEu|SQXFB zDq;c3o~{xAdD=Vuer~&VAMNxu+fMQo0`NRLS7kng=JYj)xarnztS!?NBFgY3hSh5c zSiH!Ed1H&Qi`$S9XCxY+RFPR0P+KybpP-v*H~`Z8`Ak@MVwfp1{A^AE%Q#A<1EICn zcMNe+gX!A}l0nWX*QA-y`g+KsR3hjsilJV$iQiOs$9bXcHx&UTCX2fKY-#ka3{1W@_IV*N1j*<^7=&# zrM1u42bzgeTr7n+fzobJ^16CybEoMZ(_@Na1qqlEVEcV2uu4xJ3k12GWbe#8-@pYAp{VaA@kfsQ#|{muJfpBFJ6rXY1g^uL}zKB?!enz(6!?F8%|jA?Z>)RAl9e=hWn zDx}?Yj>iz0df>w2%b+kb1NvD6c9J%rvx6ET1>BAtJ-6}=|B9DEB(A~VP% z`*r0(5YV5{dG6rs?Ap)!1ey@Ea*mLx?2Z;beKrMshxFkKxh-dJG6lB*8-t66B#S1? ztnO4!{e}0RlYhkanH4!aF8dW@aY@p6A!I199vtvZLA_7Up~FDV&^FbUM&c9 zafm$Z??v@ye#s9KItdtlm$w!5aQvB8v_^Ck;@o}V)UmPN`$S_iYWqid=_m;QvcDZ1 z4NDu7tm$4cdPyv8Di+srFjO+Z0f5^YG{cUk8nhe&tt0CX6{cYfh zS8_^9AeSNcWA0adGWc$!PJxFNty77ATx^W}lPiW0mubW5K%HOdbve-Mm;KUyKt7Ye zC-Tj|EQ`9i3^EAtG0>PKM-kcUndCI{65|vtLhQ5s{C9JQA}Jc0r9Joe&YF;jGJ$WF zdGK{wY$}UnQF~$u=wHqq)tp$Z$G*+BeX=;(g&x*38K4zaWEEAMQBfy+{@Li+QX+~l z3eGhzYO|6H-T!b;MIE%#Pbses;=59M<-E?}?xD7Z>5F7-d3;TL(gyQ(7o{iJIun<& z4J=yifC>2}gO2nUIC#C*_H{(%b6RA7CT)lI&9=upEqWnE#PPDeklHX=aU~UjY`>ip z;5y7PB}k623+QUiqM+wqj!P!hknV!NLcHRClI{nNprYpF($r2ru+P^Fd#g+SozM(d zD1c7kjeFGf;J~P&`}idlnZm@D@6g%HC-o4=ngL7;?33@57nsy^CLvxiE<8shMG|}c zPVF5g@qV3m@aPS*GC6A)1u)h3Uh|+uaZEnV#gy2uAv34!knZ~>Hl^P^d5-jU0L^lf z;}8FXDV(Q&6Z*A;?}U*0ts1IY;)c{qzVoBF2 zICPdOPp89n}B!x);paUI$U}=jJ7aHU`ChYw(a< z+yKXtImEd7h_Hk8NA;1L4%Sr}_07!W^|Xe+H6x+`0_6@ZNWtoQf0r_bVk>LcB2rP& z&1s#ibXnNmrxiId>Mhix>)rIYm`snUP`0l5DcC8ry-n`5&UbM~6g69~!8xek%IcynK#C(+xry}Up93n<0r-GXq0Ukbn608>tILeqwg*AKHI@JRs^>7`&IHWEnjxnKbikjmB!SACr}v zy-&B?(S-ix1wV5?iz|_9fDh8!o#Q+6I-m}P9;3jAJ*|0+!nfu*8B|(zw zmG*QN@tNUG>6(>K;kms*IQ`Z<%zEmZM{f4VZ)?)BEwWndE1=?M5`4k4=~N* zc24hl&X#l4Q-;<(Nt>WH^Pi6A2VGj}-M+UaoXb+A4-)6UsV-%S@TN4~URm6epS-Gf zr#KG7J1EW~h@<;BWP{Z{WzU&@ECCR!p)XT~P%a&@N}R=?Vy{tn%a0K!}%E9oxmUUM4p z84++OTy+Q4yB|T--ger?U=?g`;{wv;h&LzVE6u;t$_Tf$BFad_HUP2EnR_jWiD}At zL@Cu;LpVrDGAN!o2O!4?-2d~9ybH1vI049oH>Vat&En_*PEg`uf6-6-{DXzmRRCEw zdVjKyPc-)|uB(ayTCW9V;@i&|pxCCDitqK2z^mT@7XsgZ8(GoX*@S&=p!zEDmr5<0 zS$e@D<;+i@GTxex9lLF+qD-oGtpJc$Ea}naF!0OU!xFX`+9cQ`HuoCNPXA&#!YDk6 z7?=U4ygWc5d|Ex`?C|7>zVZEuYYp{8<-9AG3Gn9_I_=6;d{-`pq9}n9W?>erEg!9w z7|hc(ZRknKXEl4z;sK!hI~5aWhl=mkr)nGfrzQIrJW1t1de!9?mnmS9 zv?NAsV+JIn2HI^lYRTL_iy*XO@$y6Wt$8!ReVpP`?)#3P-p`$%&CRXiqCg}lwvLjz z73sj9m8+K`%j_3aodE6q3 zwQ|@p>=kzy;ea5?fS|_184&A`^4CfF=c1Q1YAm?eoH~M!^_NH4fv#Y1B zW&>L#6IG@93Ex0iU1$apNpf2<_qaY9rW3fJCbgCjj6HuW$lgYly}d(VD>-!4-A}GU zoy7Qnh*~AOL{jJBwuZ1e4hqzbVLsv?IZe{olS1WQghU5_JEOoE@j8WG1Sn&bVpeJ> z&hBj-%enO6*layvn>X$5!Zu;DO)^86Lr5BXAfPuaC1f*hsklbEqA?+Jn+%PcX+(sJ zyH9o>d2z!@?NLqcZwmTITMdSnrYyD~_v7qAXLx>cM^xt&!XNWrM19X0WV~mnA0Y~7 z7Ag{)C3TlAeyyU`O*<=cm3JX}?N85SoyA(|oLAL+m)5f%{=kclHR4{C#QV^as#2Mk zdSRP%TKFV=UDhalLi2hTASrZ@Yw;hKf8sSESQ%SL z9+1Ig`lN>XgO{!zUX9+FvLGLJQ1X%*D!fM&a3aoxSk~{G#&uilj;HESp+66;w(B|=l*(uI?;S5!B+E#hGWA@L-- zk|B#O4ZDg86zNbG{3zJMO;19uO=OMuuO~022P?3$e0kv^@sd5 z^bxEevJL7&%vTS;j5N)?ahKoB#Ldi)gHwrir3yefJ$zQzFStrFP+}o9s|T~Q_ ze*NBCU|6^pOXS&`fn*iBIr*u%P%KDTocv535|>}-$yO+Qt945|J+WOK1NL1|!$M@k zLTzUem8DrpBJl-gLI>pfVG#B8%+=1zABWncZai#T5xGEE(3zRw-PQ09QZ!;2P^@pn z&7tyNBu$@l;{_?uVU5i1pyngpeRPk9#V-N?3p2P=4qzt z?w`BJ<{a?gDp+W~mOtHrw9_5ceGtzp%4CVwbxktw#8xXIS z1`FWrNaD>pMH{S$vfmhfO2EFHMw?+M4Z_m~;i1YQe%z_s5F9Q)94>uOF^HfL=MR05 zUMSW1dV zXznOo`tzlXBR&_X3l8Aeg>T*Rx~T${;-c0dETT5Qmxt?pR1M1JRrgeXF~geWeCC*np8Fk{Q`hL@e! zdn~rOG{v7JX|O6=WN%E+9#D*;>iEa^@m~o;S#q8!^xc_$_(NT+3BtdYo51m&D(a~`BtTDsqkIb%6%LCSE z%2H1{t~{_j@#S6a z99|6lUp^;GJQFW?@CybG5MigHF0fE6bm!L-{q@s8`zI- zb)tQcz458R-lrkH?D!Y;10_+c3B2cMcTC;P=n?`NjqQyB@JY#Y8t1poH4jwj4Y^h0 z@pa=k&_4X=!3rq>O!&hG%;@~_&BMUtWHR4u+REYTN~;DMktN)T4U&ljHLgVXs^-vb zPd*|n6|ijZ%r1HP;raC}5K$I2ITvMV`J=4hpHGa2qgm)bu%yzjclJ!`Bz8pn^wfj5 zS;N~Vck@tpr1GGw!C1s?bQWf&pVLkSPR^+ugg709EYMJ{=IGgGKKIAwL!YAFHz^8U z(HC$>`__J2mmMAJ`6By}_TBAFO-^Vf<%O0S3xc2MW%oi1W9{)hvj{x5UJ*l#<+6$T zQXkkdd;nswB``K50;l%#a2?fe-h96av@Fw)5=ETL*h>6vk1QRsxEGtLs|tB(NbKiq zNQ@CrZYAf*yq_Xk{e6h+SAb%rW+mwq+cf9MB)S$ljG6?v1xJ$j_@2yi_RPt}q}^NQ zV5{4D%ltvjs9mv{WY45gcEuKb)}P}m+t|>y&g2LY%r6)yD{Cw~8LRE~Np0-v?j8x} zlBdpEQH-uv&hcK}j*I)nRcnIX%`@xE_PXK92WDUVujIE2p7!;&#*8XR<<9IDRv{yA z)mz~blyB0PZ+h;s#>+;&_!2}}&K-#IR@}eF@{K;Fs_^cIJ8a~XgEW*w96vZpSlZn9 z619@5m}Z1B%PPP7>uStt{HTa9vtbW?NoXvt(LOQYKxNFVLN*pUhpQ`(A{n~{kE4qL z+v${u>*&m0NTH)fK>Ko4a{ng2)UrieSFB#zC-u3A>zNATf9hu_Ko56;OLP^cMKxyt z+c82Ps~T94CS(GMOZjR#7xFf%Sbe)YG2!dAYj3)BDchG5y~cvxKE+pYx=k6Mp2LeI z1>ebom3@WnS(}S016KZ0Jxm=uZ&maRz{GhdY=E~OqO<%WFI&gIiSQ@bj5wTWq{Aes z|7WtoH1=(-K5^wtcA0yF&8lq**5^q9X?KsqjT>;tazySkSoF77;jHi$3FzaF0WC4D z``upf&+NP>p^f;|i+bSXydnA#koW@6 zHCQxc$L137)VN!DOwF}{jG#2uS?ZWDYrHoDHT`uYJ9DIID2soE4l(h%lp6|p!~ksM z!SDU|Z)20A1RfTi-=g#V8ES;GGa~nd4@iIi#{#lB_Hxpv3DW1pWRi-%>8Iimf`z`zGcdc@i z(Eg07X60H-{*)0ZzfXeah%q(LmjobI>DlG9&ETM%Z4 zUPNud5ST16w8ZAH{sK1~I7Z)=q2*)jeU-D(n);OY;{Ck@jH%Z_SXiGWN|JbNn+jcf z+Hy+pqTS_1!Q^IfM2~{k?^XK6ygMY=!(2IojjlI|IKyl6gL3aymbM9C|jcb2h@w`^>oc;TLC%k;cO-&_dwXwVLhS^!%$f0t< zDF$9XP+Krxpqgo%t}fKU6|kc)9J##+XZSu?;@&VuZ>jMv2sVF1+!0LpeEGxZ!gVHr z8S1(6eOIsJvlSJ*`SOFEk%`kT-*JZ;-=i_#gst+%_@(=&8SdNrjpqU&5a|5;DoFm2 L?8!DR5$OK_J|8wp literal 9585 zcmYjWcRbbq_peA~Q&vW1$cpS$Zg#ety+_E%xP++e?3F#Ua^=e26xn-TGi8PAQf4ml zd%rJzzQ4!)L+^L5*Lj`sJm);m>&+B_k6-UwWPjz%!^Yji%*w^n#?9iXjhU5&m7NWr zgS(6KrOkOY_xbL39N+!U!mpRvS{9lRu*o#z2vSzA&-Wv&0v>AXQRK;Mad6e&WH?>8 z#$6(vw8E{zV(}|}NS#}bXJ=>W?cs8d``@K(f6Tg9c_K%M#OW5P>?sO!xZj%5-1=g7 z@n`z%m>#u@+38COk8-Ba8iaq#3*W!1rxEQ#+X!c8jc+DqnM}^k{xY)7;wOJs8ogSN zvtRv1iXwl?W}BIqnl_*I?IsufMCysQ?hLQ=Oivuw@~fB0W}O_ho@}`PEF=AS=2_Q* zDo2KjZj&IrnvRw?_f{8A^q3l6dbuLcR@>M2JJzSu+Xpb;XGud*=@Dg2e;0!>eRrOB z>F}$&XlNk5EwuQ)@h3@<=67U@zk@` z@0FjY&!OZ=HUIKJ$I7v2>&dZ3*~;{hOY7-&>!g{~?k)cPx39{JoujL8bp30%E& z>1iNNh7CJgqB2%kAgzUMpa7-i7%6n+Qt0`b$~S_i#(MLb^`Di`RhVA<^Gbhvvfn-j zkN^SRt^RmdTqHhewQYPM7T97%)9zDlTll{f>q++b^^w?9A+)9aM*Z(oj~za=2LQI> z38esL*$I{$7nG@hAitIAdw&`sKRh>8%t|DHU`nz7gMUPR$nspx{dbE#5ZHZsO@ z8D`B3xfrS)-Oj}w;Nj@tK7tKQv^|L00&^ju7CwRsfa{YZadI`oGT-N`pN;u5X7*n{ZM5`Q^r zbgF486o8;AU5yqr+O)W9k=jSua1XOHHnj0$xuc^mITY1CyW2icbK9g@tm(_dZ+DpM z0fXfibV}Z2dV6SldU`uLFZma;_0HL9xoVCKm1eS3Bvkl`<*jQSA{B3)5~sg^fD_K^ z-^{xSDlP_pKawT?mASS!Jgvk25i4DfRO0l5R6gW@F;qkt#3f*lh$aW0MXI6NcWjWY zblOGI=^D6;#Osd2u|aO@r=nEWNeh{A)4@VHx1ZyhkY9Gu#5t@02>?3F)04bux#-{{?ShHq3BN+F%K8p znfQ$<-Zm_Povz3YMd#i*vR!eX;e~TZ(W^HI4k`RCGi+RBS%v7&jw|ZsEaK0MZ=6{YT%&~@&ARS zP?yS_UqR(4KD%(teyIb@5M(YigT#fmLqH8AIt*sMmji#=4A zM@8B{%4axQv!+wI(xZDv#-ZhR3z0wY6~(l7barNom-SQ}-LgxK*ugzC%+HW~t;3!5 z_PBs%z?!xZ1bwNyft8-5NE>|Sf`V&Rz_z5g&!NxhS7glk9=ZglGNps( zM(&!wkX~N{9al1*d^|Z&g};|B&V1g^n`9c|<1atJsfM}$s+C~%m8aJN_Z+*YZOYg2=wi|mLVV5*S+hvP39sIM^qN#SXxDPJRr* zcy6F?R=|BSHpFt3y5AGo{AKR_C@YUDd0DsXH?o#$ugi}XARZyO0T$aknqo8i+)(Kb z`^ZcN=4BS@dR^2ezIgQh7idZjARRdzm}fG}(!e278c{C>1U`c=4&Bp#V8jm*zPFKn z8vvZ`5&x&G&kD!-8&d2`W4VBv-#**(h_<3tFFV&S(zYoBi?-ShJSiIJiGPe=Sco%f!?{e8J9KUiLoMcrZ35VG31CRJ`x* z4J8cdf-w@QdlW3QhZg89Mi+`yUw)?FPB-}}r25Y18RItNP(18&WYh5SA6=?w4{vpM zPn>=m>ysZEzO3P%ga$B!Ro8VbK5Rit4<^ljUhM2_U`|M=!`LnTY89Mjy;QlPvq3k}vN}ip z(J|+Z@srkg*xd%WWYsj`S5?R7(1P`(>f{V;S@TdGwj0tm7ccyG?ihpV)Z@LlVd|7Z`6&8l~5+mu)7+C>$wBBmwd$l8XJnijjqgG4%ep%8R-dz&WIN*`w63={6&q0AAF+5p#j)v~ zgF$2muswL`fGp}@>_d#7&c+~``GOkPr1T}orG(!5k8FI296=P%6u2tBJNHDTe8_Js&A@f8q)}&nLkOr+Cmvgd(~2++FyuGZJv3at^jG!U4@OK7g3NZA^k~UF;X$^0@in2WFPHEHUSB54 z4n0dQh{}aoUIrc?TJ+Zcp=!5El1W6kpXB;JKBOu4h*@W>{D|(td@AzqRug0jeYZn* zx-9JN6mcX5frozNz93}NcefMk843%`K$;z=v=q=8Y-XMU;m!_u5%480o|U~@OxrD= z&GJ;g+-0+CwC?^_74iswmZY%#BN2Pif`%XxEKoD9tP8*R^CvFE-=RVMM6+9`n>9%; z3;2DDA@;_%NGW2DP0VLw=TcVYcJVJ*9*!Oxr49c25UN&D>>#6MGpVt-vVQbq`QX>V zDni5ZXig`8 zaLykULfd5+NUUgg#}%N$6mbA!iTA3RLQe7SjNTY1g*NM(n~H42Y8kUrY12b0P&G`kL+>-A{v5kT?xN8P>R z!Cjtu<(<3#8^FM8nBc|A=e&}(*Ky?PNR`g~&Ev8QI+fl}T9$0968!%%c|p%9l`AjL zNKv%h!vnQexU82DjiFhU;7?r6;ZYDVh!nilpR=|@FF8*LifxfkWz||L#xqjdGc*X*q=EEDQZ6MgaYy4Gyzc|5(2 z@g0i-`|br60e~ivc|3{+*|o?T@6DYLN=3l$8m+v9yaeUGzvFvT2e-AK(fJ>5l&=-V z7R4U^>Rql!LG01;`RAp?F{j_S$eWS6G8!~-?i97Lr_-}?uy|taM;QHl@3XXH%zp=^{rF9qg?usy`=Na3K@_X$l zwFy`%b`F^Hb=zKd4{g^?Y8X+x2Byp-*ZeAZrg@3Kd{O^~Ud|x2ha`-yfrAR?^?gx0 z=bW@vyaD2!89kLJCX(aOrEiqiR)j00zc&|E9zONL>>Q;gXj?z3kp9&=EzF6>T~?G;*?`M(MB}&S?_X~nW9Vj$ zk;)2eAPb?t9qoyRXQxdei#y`t3*y;l5_#SMzgKpjob6`{9|gh;xTZ#iPVZBo*|Z#1 za}B9>#X!`EOXV$47C#A*SREgnk?U3vsaS79dY>K9Jae3#8ZXQbyi8<1b$92aPgKUU z!k=~l(FLW558C@grVgcFbfn_Pe~2%K4i=lKHBUx4Y(0bcx&qW#Je*}#r@m{VOsiy^ z?q?4m)c$a6s}}h&l5ZKo5XI^d+;@`?Uf;HIJ-O)sW?Hi0<3Pp0LZ682E`&VFe_2Q= z8{*2ST?WAox{aIL#%@zH6&JrsmcWpD4(!zC=&s2o9q7GAp~qm^QIx7UAW~4HgZtdP z1~7H2-QTt-aU|l*-#5Hx75a!M`JOMCSMPK(EJhC=QUn|@v`Sn-?WP>}D z;JB#`b7Im4*e0+BO(-@~nSm6s>Q5-gicQ?@1~!$@h5>qHYYgG?@p0kUsXI{ zH83Wu-hA`RyT%~MAw&D^ZCqyjhn&ej?1scE=Eaik-EsuibYJ(myC$pAbgK(i(QZ9R zM)Ua+UM4xqkk0y1dT0ZQi5gZHJ_%Zbs-f=pExJzB8**3^1_NTNtpviwCvzW*Y%c;6 ziO-+-Y`1TI}rHsF@NQkPgK1|oH0s-E4+z%WhczaMTw(iieBLcT(?Ix znkfu?@fx`jT@k)f^q`~0?@rv!e$?#7H89+-U}H}FOn)q!Q_5+?HwDRjW8mX-dvS`+ zJxlSM?%b=oo1tt@g${#Dj_hn;me4wJYRim`gbc)E$T-LX$bxR)7YuUL$k$KXuql?* zp}UV7KbG)c?O#YPw}k5-?x-){cDWrJ?HB%Gev*RIqCCcrA}OevTHa~F zqx^0M*E<0`w%eC`xF%db3)lGB>5%e0e?w^e{R&d+^Sd;}cBg!L!y;2fY%nm4Oqz~F z*N307KzcMET%m}y{;%pIXNHvi2{*BT4$_J$h^Ev278AOKwtQ&`A}vmg(Vms%BW^Tt;- zfeC}N6iAG_cnxpC1xTTJT37sk1uj@gwi&((pVT{g{_|Md_@#442fd0JH`3Q+Tb`=I z!z9P9)^@i(^;oQqogH$SK5*~vbL*7W&k0YF7ZDzQEKZ1IU%us>rz&N9>4 zpP~TR|JA!s?-+?3U*gSlO zOne8in`uG6#sy-htarOEdMWyey1RY)Rjl9HxNf5*30Xs$mIt46+y+dJzfl*BgMibf zIXgeYhBrRn|3-cvcn1;-JLB8a9~JwVg>9i>Y%FNHxU^!2F6YN!kiiqs*R}NwNon(lxdkQphx+E z+afL^?{!If30JLCVi7p&_@Qi#ja;?^N5DIX(vEq!+*G1wQqupX#nSEejF~x3zBW?- zU+yzD#hopx!VL}vNhz#g31t!Aa+`zE=W%z6CA&KRhxzMuh2AQz+!WMxpTRUhDNQmA zp2k}~qoM>~gz7z*@fI`B+R=qw1)v9dG~e$j&xE^aOnzk5A9b5=&8D)y_pK!SjyGPP zPnQj5W@ht|>`$kaM9$Y|M!;0n`$mm;uU`1Sp0rG3LhMio8=lKOCUbtVL5zsbE{UkzCL?t2dP#dY@T2|5z`&1#5!FAa zg@jkTQ|y{dYCWzdn=Lm~3E8CxNCPMsV-L0{i2Rt2JHt6ayps}+#=Uk2$BzIHDR;A? z8vc7cArOHeW+DrGNXx5qc}|lBJ@DT&?*$44YFsQF2O;;bNz7dWW64bA6vc~~W_!6G(n+KWU8r7+g83z|N>aqOFyxa`i~iKx zKBTeB>aq-de}IoBpJA<1w2O(ovVF}VyYGJf6P%_SoLA*Euj7s`X}l>PM7{6klT*E( zc%|S0m*(}Ltx3ItlKL#gH96{r;b1YUM_7i#Lc8K}7HCQ7Ek)TFQ{07&Sj& zk9%J4F5BzHuF|N^drs0^Id_C7Bgb&0I4>90FY(O{2OIun%&N#5(`e1|mp=8u?AJKQleRZim412pPb25nEm1Wq6O?ig;0r)y2a9^-pDh8> z{o$F=AAd%Ev*PaSUa>ceO}kNoNWC8W{~}ssJwqZsp6RXv#5gb~=vuztLzMyDvXUDt z=T(3}sWCwJ9fHKl!%wb9>*7cL;qY>`?t~bh)ivFm3tw(0XhM?Pj9%!}TJEhBia3xx$>A#m^CD3Ia z{iP;*d1?Z6Td@)YK#MF<`*ziV+8c{OwdO}Q$^g_Z5I09Rk60E7>Sb^`haBRDZ`lpW z;08v#9PSeC5;;wSn@TaVS4o_z_jHPL&us%by?m6aPMUu)wg&YGiz4S*j-Pyy(#_?i zgwQ*Q&$v~s*!7JhX?kXBQgzMkzx&-Im#T^P<*7FN%cGqGjovVg7piJwkN0Uh9>KMu z98{uIq83{9ZE2G-9I(lTEeq6-*V;yF&db6zjo+KUsfR2-td={>dj?Qd<*X=$Im9%* zwdkD~51IaGVM!jLZ%Gx)_^SW9m0p+eR~|43mQ)j2uN|P*Ps!r_-N1w1(^N6a8lLJ; zjQLd37!Ct+7%57}G+IZT`3E&7Yfn}KpKcFrd93aV-6?oI$~@D~GBtQQE;@e&7+OHe z`n?h&AnF?J-V3l-T;@8$fMph>S06M=l=9QyObfBU&+3$0xp79{H5bSqgiGZBhp#X!`*2NON4)sgZ%RwVwh- zS?0JpQn>M3infBg(akw(!X2?u9JU-nsYf}sbd5tbT{0^#-a_X!O8dKI|Nb8Gr_+>a zC7UAJO@TNVzX6AVgVG)n1?L*+L48r4Q_Y6&+M*!kCBEY}Mg?JTk5+???Tcq14Fbl6r|C^&iC}7 zluKd4AY1PBZyUpl`sEj1?iEZ3)R0m4%j+wBv$Eu<_2|zGIvHKLcXjjVCO+6!zO_5t zd~X6x)`imm10=rOFvC;8p(H{OJGgfZMM7%7^=dmGNu$s2-^(i#`$qsl^jQ;5g3?(k zCHaH?8pSOp*x~v#8LXYgp}UjPIG*w~pOMO96Mx*$GTp5#oD&T2jK=LhM2C)pCbA+? zZ&*P1GaYb7IXU_+dy%CHs@>~Idv&tJbQbggh>?n&Y&APC4mi%@OQaoR){#sLCZ*jk zcj%2{R#_eh9iXN5FkIk=D8FaQ+To)dM`EbB;!=nt$@$1q&-}WjNPwfBE>w&-dKw=v z9xec<-)f333|Gy1W>pJT9chW!$?W(0P?@=-md^!Ye_%6dJ5{Tt^5k*)Us;my+2x#a z9t(M>={zOx9Vd0riZC4aIdSO7jkASTY{R13$wk8)#3q z-xaDIkcyd(I!8s-Ym^e^${796arW~RX0)Dl=O#t){)XCNqOlhDLvUAiZ-p2=k#EGGm;1;m`KIG{(eqZqD?THREnd z(UT8PUfc-PgKVM-{M1RrzsXSn)91$}wvrw{#+iEZOtG)bB(7l)UoqWkr9!=k`N9Swa`*-wIzb zj|%^X>Uu$cmxj(3PcYAYd|#CR4PBVX-TEv3^%cp$8Nm-y*luXUsUJfZN6nelsW1N7 zzU&d($Usj+_vt?W*~aq^gQgkKADU4XA27KNW*H2FgQ3sAA(y6)yme{^p5sHk4&E~i zHCl+dG?q(5@ne3kyrB7uYHpEe#XR3Rsy!?^JYL_HJ*_=E-P$`@@W+^OogQYxF^9ho ee~jJP$v)f5J|n$!>Ff;W(s#tHAQ_tQ(*FS;*(eYI From 6d10fb73423ff772f00591b6cc00332dde5a8ab8 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Wed, 4 Mar 2026 18:05:11 -0500 Subject: [PATCH 23/35] Harden cache push target and add safety comments Co-Authored-By: Claude Opus 4.6 --- .github/workflows/test.yml | 2 +- toolchain/mfc/test/coverage.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 116d9efac4..f6ccb68efe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -144,7 +144,7 @@ jobs: echo "Coverage cache unchanged." else git commit -m "Regenerate gcov coverage cache [skip ci]" - git push + git push origin HEAD:refs/heads/master fi github: diff --git a/toolchain/mfc/test/coverage.py b/toolchain/mfc/test/coverage.py index 0369d5144e..2a1d77418c 100644 --- a/toolchain/mfc/test/coverage.py +++ b/toolchain/mfc/test/coverage.py @@ -400,6 +400,7 @@ def _prepare_test(case, root_dir: str) -> dict: # pylint: disable=unused-argume # Suppress console output from get_inp() to avoid 555×4 messages. targets = [SYSCHECK, PRE_PROCESS, SIMULATION, POST_PROCESS] binaries = [] + # NOTE: not thread-safe — Phase 1 must remain single-threaded. orig_file = cons.raw.file cons.raw.file = io.StringIO() try: @@ -677,7 +678,9 @@ def filter_tests_by_coverage( Conservative behavior: - Test not in cache (newly added) -> include it - - No changed .fpp files -> skip all tests + - No changed .fpp files -> skip all tests (safe because should_run_all_tests() + is called first in the workflow, catching toolchain/infra changes via + ALWAYS_RUN_ALL before this function is ever reached) - Test has incomplete coverage (no simulation files recorded but simulation files changed) -> include it (cache build likely failed for this test) """ From 66c792a23ab4709c45979432efda463ad24e541a Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Thu, 5 Mar 2026 05:03:58 -0500 Subject: [PATCH 24/35] Reduce benchmark steps and switch Frontier bench to batch/normal QOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Cut benchmark time steps from 60-70 to 20 (GPU) / 10 (CPU) — still sufficient for grind time measurement - Unify Frontier SLURM config: bench now uses CFD154/batch/normal like tests instead of ENG160/extended (2hr wall time vs 6hr) - Reduce CI timeout from 8hr to 4hr Co-Authored-By: Claude Opus 4.6 --- .github/workflows/bench.yml | 2 +- .github/workflows/frontier/submit.sh | 15 ++++----------- benchmarks/5eq_rk3_weno3_hllc/case.py | 4 ++-- benchmarks/hypo_hll/case.py | 4 ++-- benchmarks/ibm/case.py | 4 ++-- benchmarks/igr/case.py | 4 ++-- benchmarks/viscous_weno5_sgb_acoustic/case.py | 4 ++-- 7 files changed, 15 insertions(+), 22 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index b45fc45e40..5cf9681e33 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -88,7 +88,7 @@ jobs: runs-on: group: ${{ matrix.group }} labels: ${{ matrix.labels }} - timeout-minutes: 480 + timeout-minutes: 240 steps: - name: Clone - PR uses: actions/checkout@v4 diff --git a/.github/workflows/frontier/submit.sh b/.github/workflows/frontier/submit.sh index eeec87c487..743d70d95f 100644 --- a/.github/workflows/frontier/submit.sh +++ b/.github/workflows/frontier/submit.sh @@ -44,17 +44,10 @@ else fi # Select SBATCH params based on job type -if [ "$job_type" = "bench" ]; then - sbatch_account="#SBATCH -A ENG160" - sbatch_time="#SBATCH -t 05:59:00" - sbatch_partition="#SBATCH -p extended" - sbatch_extra="" -else - sbatch_account="#SBATCH -A CFD154" - sbatch_time="#SBATCH -t 01:59:00" - sbatch_partition="#SBATCH -p batch" - sbatch_extra="#SBATCH --qos=normal" -fi +sbatch_account="#SBATCH -A CFD154" +sbatch_time="#SBATCH -t 01:59:00" +sbatch_partition="#SBATCH -p batch" +sbatch_extra="#SBATCH --qos=normal" shard_suffix="" if [ -n "$4" ]; then diff --git a/benchmarks/5eq_rk3_weno3_hllc/case.py b/benchmarks/5eq_rk3_weno3_hllc/case.py index 5ecc327e8f..fa09426ffe 100644 --- a/benchmarks/5eq_rk3_weno3_hllc/case.py +++ b/benchmarks/5eq_rk3_weno3_hllc/case.py @@ -191,8 +191,8 @@ "cyl_coord": "F", "dt": dt, "t_step_start": 0, - "t_step_stop": ARGS["steps"] if ARGS["steps"] is not None else int(7 * (5 * size + 5)), - "t_step_save": ARGS["steps"] if ARGS["steps"] is not None else int(7 * (5 * size + 5)), + "t_step_stop": ARGS["steps"] if ARGS["steps"] is not None else int(2 * (5 * size + 5)), + "t_step_save": ARGS["steps"] if ARGS["steps"] is not None else int(2 * (5 * size + 5)), # Simulation Algorithm Parameters "num_patches": 3, "model_eqns": 2, diff --git a/benchmarks/hypo_hll/case.py b/benchmarks/hypo_hll/case.py index 1663a507aa..f8d0928a01 100644 --- a/benchmarks/hypo_hll/case.py +++ b/benchmarks/hypo_hll/case.py @@ -44,8 +44,8 @@ "p": Nz, "dt": 1e-8, "t_step_start": 0, - "t_step_stop": ARGS["steps"] if ARGS["steps"] is not None else int(7 * (5 * size + 5)), - "t_step_save": ARGS["steps"] if ARGS["steps"] is not None else int(7 * (5 * size + 5)), + "t_step_stop": ARGS["steps"] if ARGS["steps"] is not None else int(2 * (5 * size + 5)), + "t_step_save": ARGS["steps"] if ARGS["steps"] is not None else int(2 * (5 * size + 5)), # Simulation Algorithm Parameters "num_patches": 2, "model_eqns": 2, diff --git a/benchmarks/ibm/case.py b/benchmarks/ibm/case.py index e16cb620b7..303cf7fcaf 100644 --- a/benchmarks/ibm/case.py +++ b/benchmarks/ibm/case.py @@ -48,8 +48,8 @@ "p": Nz, "dt": mydt, "t_step_start": 0, - "t_step_stop": ARGS["steps"] if ARGS["steps"] is not None else int(7 * (5 * size + 5)), - "t_step_save": ARGS["steps"] if ARGS["steps"] is not None else int(7 * (5 * size + 5)), + "t_step_stop": ARGS["steps"] if ARGS["steps"] is not None else int(2 * (5 * size + 5)), + "t_step_save": ARGS["steps"] if ARGS["steps"] is not None else int(2 * (5 * size + 5)), # Simulation Algorithm Parameters "num_patches": 1, "model_eqns": 2, diff --git a/benchmarks/igr/case.py b/benchmarks/igr/case.py index 469bff1fa9..4ceed76257 100644 --- a/benchmarks/igr/case.py +++ b/benchmarks/igr/case.py @@ -63,8 +63,8 @@ "cyl_coord": "F", "dt": dt, "t_step_start": 0, - "t_step_stop": ARGS["steps"] if ARGS["steps"] is not None else int(7 * (5 * size + 5)), - "t_step_save": ARGS["steps"] if ARGS["steps"] is not None else int(7 * (5 * size + 5)), + "t_step_stop": ARGS["steps"] if ARGS["steps"] is not None else int(2 * (5 * size + 5)), + "t_step_save": ARGS["steps"] if ARGS["steps"] is not None else int(2 * (5 * size + 5)), # Simulation Algorithm Parameters "num_patches": 1, "model_eqns": 2, diff --git a/benchmarks/viscous_weno5_sgb_acoustic/case.py b/benchmarks/viscous_weno5_sgb_acoustic/case.py index 9f1351b0c1..83bdc43e9c 100644 --- a/benchmarks/viscous_weno5_sgb_acoustic/case.py +++ b/benchmarks/viscous_weno5_sgb_acoustic/case.py @@ -94,8 +94,8 @@ "p": Nz, "dt": dt, "t_step_start": 0, - "t_step_stop": ARGS["steps"] if ARGS["steps"] is not None else int(6 * (5 * size + 5)), - "t_step_save": ARGS["steps"] if ARGS["steps"] is not None else int(6 * (5 * size + 5)), + "t_step_stop": ARGS["steps"] if ARGS["steps"] is not None else int(2 * (5 * size + 5)), + "t_step_save": ARGS["steps"] if ARGS["steps"] is not None else int(2 * (5 * size + 5)), # Simulation Algorithm Parameters "num_patches": 2, "model_eqns": 2, From b50e6ea4bc381ca11c8332303dd10b2137be86c5 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Thu, 5 Mar 2026 11:21:11 -0500 Subject: [PATCH 25/35] Use retry_build in coverage cache rebuild for NFS resilience --- .github/workflows/phoenix/rebuild-cache.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/phoenix/rebuild-cache.sh b/.github/workflows/phoenix/rebuild-cache.sh index 14db7c83e5..23b3fa05c0 100644 --- a/.github/workflows/phoenix/rebuild-cache.sh +++ b/.github/workflows/phoenix/rebuild-cache.sh @@ -10,9 +10,11 @@ if [ "$NJOBS" -gt 64 ]; then NJOBS=64; fi # GPU build (e.g. --gpu mp) whose CMake flags are incompatible with gcov. ./mfc.sh clean +# Source retry_build() for NFS stale file handle resilience (3 attempts). +source .github/scripts/retry-build.sh + # Build MFC with gcov coverage instrumentation (CPU-only, gfortran). -# -j 8 for compilation (memory-heavy, more cores doesn't help much). -./mfc.sh build --gcov -j 8 +retry_build ./mfc.sh build --gcov -j 8 # Run all tests in parallel, collecting per-test coverage data. # Each test gets an isolated GCOV_PREFIX directory so .gcda files From 1bac876a65c0480ab4201bf795a07b00fd84148b Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Thu, 5 Mar 2026 12:28:49 -0500 Subject: [PATCH 26/35] Address PR review feedback: fix type annotation, dep detection, and error handling --- .github/scripts/retry-build.sh | 3 +- .github/scripts/run-tests-with-retry.sh | 2 +- .github/workflows/phoenix/rebuild-cache.sh | 2 +- .github/workflows/phoenix/submit.sh | 4 +- .github/workflows/phoenix/test.sh | 2 +- .github/workflows/test.yml | 14 +++- toolchain/mfc/test/coverage.py | 78 ++++++++++++++++------ 7 files changed, 76 insertions(+), 29 deletions(-) diff --git a/.github/scripts/retry-build.sh b/.github/scripts/retry-build.sh index b82a2e5d8d..d32b78f920 100755 --- a/.github/scripts/retry-build.sh +++ b/.github/scripts/retry-build.sh @@ -8,7 +8,8 @@ # Try normal cleanup; if it fails, escalate to cache nuke. _retry_clean() { local clean_cmd="$1" - if eval "$clean_cmd" 2>/dev/null; then + # shellcheck disable=SC2086 # word splitting is intentional here + if $clean_cmd 2>/dev/null; then return 0 fi echo " Normal cleanup failed." diff --git a/.github/scripts/run-tests-with-retry.sh b/.github/scripts/run-tests-with-retry.sh index 18f1d05d0b..d6e4106dc3 100755 --- a/.github/scripts/run-tests-with-retry.sh +++ b/.github/scripts/run-tests-with-retry.sh @@ -8,7 +8,7 @@ PASSTHROUGH="" for arg in "$@"; do case "$arg" in - --test-all) PASSTHROUGH="$PASSTHROUGH --test-all" ;; + --test-all|--single|--debug|--gcov) PASSTHROUGH="$PASSTHROUGH $arg" ;; esac done diff --git a/.github/workflows/phoenix/rebuild-cache.sh b/.github/workflows/phoenix/rebuild-cache.sh index 23b3fa05c0..4ef2a09522 100644 --- a/.github/workflows/phoenix/rebuild-cache.sh +++ b/.github/workflows/phoenix/rebuild-cache.sh @@ -2,7 +2,7 @@ set -e # Number of parallel jobs: use SLURM allocation or default to 24. -# Cap at 64 to avoid overwhelming MPI's ORTE daemons with concurrent launches. +# Cap at 64 to avoid overwhelming OpenMPI daemons and OS process limits with concurrent launches. NJOBS="${SLURM_CPUS_ON_NODE:-24}" if [ "$NJOBS" -gt 64 ]; then NJOBS=64; fi diff --git a/.github/workflows/phoenix/submit.sh b/.github/workflows/phoenix/submit.sh index b2fc79132c..2ab1c0349e 100755 --- a/.github/workflows/phoenix/submit.sh +++ b/.github/workflows/phoenix/submit.sh @@ -32,14 +32,14 @@ sbatch_cpu_opts="\ if [ "$job_type" = "bench" ]; then sbatch_gpu_opts="\ #SBATCH -CL40S -#SBATCH --ntasks-per-node=4 # Number of cores per node required +#SBATCH --ntasks-per-node=4 # Number of MPI tasks per node required #SBATCH -G2\ " sbatch_time="#SBATCH -t 04:00:00" else sbatch_gpu_opts="\ #SBATCH -p gpu-v100,gpu-a100,gpu-h100,gpu-l40s -#SBATCH --ntasks-per-node=4 # Number of cores per node required +#SBATCH --ntasks-per-node=4 # Number of MPI tasks per node required #SBATCH -G2\ " sbatch_time="#SBATCH -t 03:00:00" diff --git a/.github/workflows/phoenix/test.sh b/.github/workflows/phoenix/test.sh index ddf1b958d5..4422adb054 100644 --- a/.github/workflows/phoenix/test.sh +++ b/.github/workflows/phoenix/test.sh @@ -13,7 +13,7 @@ RETRY_VALIDATE_CMD='syscheck_bin=$(find build/install -name syscheck -type f 2>/ retry_build ./mfc.sh test -v --dry-run -j 8 $build_opts || exit 1 # Use up to 64 parallel test threads on CPU (GNR nodes have 192 cores). -# Cap at 64 to avoid overwhelming MPI's ORTE daemons with concurrent launches. +# Cap at 64 to avoid overwhelming OpenMPI daemons and OS process limits with concurrent launches. n_test_threads=$(( SLURM_CPUS_ON_NODE > 64 ? 64 : ${SLURM_CPUS_ON_NODE:-8} )) if [ "$job_device" = "gpu" ]; then diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f6ccb68efe..9810184a60 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -77,10 +77,18 @@ jobs: run: | # Detect added/removed use/include statements that change the # Fortran dependency graph, which would make the coverage cache stale. + PR_NUMBER="${{ github.event.pull_request.number }}" + BEFORE="${{ github.event.before }}" + AFTER="${{ github.event.after }}" if [ "${{ github.event_name }}" = "pull_request" ]; then - DIFF=$(gh pr diff ${{ github.event.pull_request.number }}) + # Default to dep_changed=true if gh pr diff fails (safe fallback). + DIFF=$(gh pr diff "$PR_NUMBER" 2>/dev/null) || { + echo "gh pr diff failed — defaulting to dep_changed=true for safety." + echo "dep_changed=true" >> "$GITHUB_OUTPUT" + exit 0 + } elif [ "${{ github.event_name }}" = "push" ]; then - DIFF=$(git diff ${{ github.event.before }}..${{ github.event.after }} 2>/dev/null || echo "") + DIFF=$(git diff "$BEFORE".."$AFTER" 2>/dev/null || echo "") else DIFF="" fi @@ -135,7 +143,7 @@ jobs: retention-days: 1 - name: Commit Cache to Master - if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' + if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/master' run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" diff --git a/toolchain/mfc/test/coverage.py b/toolchain/mfc/test/coverage.py index 2a1d77418c..2b85add7ed 100644 --- a/toolchain/mfc/test/coverage.py +++ b/toolchain/mfc/test/coverage.py @@ -165,6 +165,8 @@ def _parse_gcov_json_output(raw_bytes: bytes, root_dir: str) -> set: try: text = raw_bytes.decode("utf-8", errors="replace") except (UnicodeDecodeError, ValueError): + cons.print("[yellow]Warning: gcov output is not valid UTF-8 or gzip — " + "no coverage recorded for this test.[/yellow]") return set() result = set() @@ -261,7 +263,8 @@ def _collect_single_test_coverage( # pylint: disable=too-many-locals proc = subprocess.run( cmd, capture_output=True, cwd=root_dir, timeout=120, check=False ) - except (subprocess.TimeoutExpired, subprocess.SubprocessError, OSError): + except (subprocess.TimeoutExpired, subprocess.SubprocessError, OSError) as exc: + cons.print(f"[yellow]Warning: gcov failed for {uuid}: {exc}[/yellow]") return uuid, [] finally: for g in gcno_copies: @@ -271,6 +274,8 @@ def _collect_single_test_coverage( # pylint: disable=too-many-locals pass if proc.returncode != 0 or not proc.stdout: + if proc.returncode != 0: + cons.print(f"[yellow]Warning: gcov exited {proc.returncode} for {uuid}[/yellow]") return uuid, [] coverage = _parse_gcov_json_output(proc.stdout, root_dir) @@ -337,7 +342,7 @@ def _run_single_test_direct(test_info: dict, gcda_dir: str, strip: str) -> tuple failures.append((target_name, result.returncode, tail)) except subprocess.TimeoutExpired: failures.append((target_name, "timeout", "")) - except Exception as exc: + except (subprocess.SubprocessError, OSError) as exc: failures.append((target_name, str(exc), "")) return uuid, test_gcda, failures @@ -385,7 +390,7 @@ def _prepare_test(case, root_dir: str) -> dict: # pylint: disable=unused-argume # Heavy 3D tests: remove vorticity output (omega_wrt + fd_order) for # 3D QBMM tests. Normal test execution never runs post_process (only - # PRE_PROCESS + SIMULATION; see test.py line ~469), so post_process on + # PRE_PROCESS + SIMULATION, never POST_PROCESS), so post_process on # heavy 3D configs is untested. Vorticity FD computation on large grids # with many QBMM variables causes post_process to crash (exit code 2). if (int(case.params.get('p', 0)) > 0 and @@ -397,7 +402,7 @@ def _prepare_test(case, root_dir: str) -> dict: # pylint: disable=unused-argume input_file = case.to_input_file() # Write .inp files directly (no subprocess, no Mako templates). - # Suppress console output from get_inp() to avoid 555×4 messages. + # Suppress console output from get_inp() to avoid one message per (test, target) pair. targets = [SYSCHECK, PRE_PROCESS, SIMULATION, POST_PROCESS] binaries = [] # NOTE: not thread-safe — Phase 1 must remain single-threaded. @@ -434,8 +439,9 @@ def build_coverage_cache( # pylint: disable=too-many-locals,too-many-statements directly (no ``./mfc.sh run``, no shell scripts). Each test's GCOV_PREFIX points to an isolated directory so .gcda files don't collide. - Phase 3 — For each test, copy its .gcda tree into the real build directory, - run gcov to collect which .fpp files had coverage, then remove the .gcda files. + Phase 3 — For each test, temporarily copy .gcno files from the real build tree + into the test's isolated .gcda directory, run gcov to collect which .fpp files + had coverage, then remove the .gcno copies. Requires a prior ``--gcov`` build: ``./mfc.sh build --gcov -j 8`` """ @@ -455,11 +461,14 @@ def build_coverage_cache( # pylint: disable=too-many-locals,too-many-statements cons.print(f"[dim]GCOV_PREFIX_STRIP={strip}[/dim]") cons.print() - # Phase 1: Prepare all tests (single-threaded, ~30s for 555 tests). + # Phase 1: Prepare all tests (single-threaded; scales linearly with test count). cons.print("[bold]Phase 1/3: Preparing tests...[/bold]") test_infos = [] for i, case in enumerate(cases): - test_infos.append(_prepare_test(case, root_dir)) + try: + test_infos.append(_prepare_test(case, root_dir)) + except Exception as exc: # pylint: disable=broad-except + cons.print(f" [yellow]Warning: skipping {case.get_uuid()} — prep failed: {exc}[/yellow]") if (i + 1) % 100 == 0 or (i + 1) == len(cases): cons.print(f" [{i+1:3d}/{len(cases):3d}] prepared") cons.print() @@ -476,7 +485,12 @@ def build_coverage_cache( # pylint: disable=too-many-locals,too-many-statements for info in test_infos } for i, future in enumerate(as_completed(futures)): - uuid, test_gcda, failures = future.result() + try: + uuid, test_gcda, failures = future.result() + except Exception as exc: # pylint: disable=broad-except + info = futures[future] + cons.print(f" [yellow]Warning: {info['uuid']} failed to run: {exc}[/yellow]") + continue test_results[uuid] = test_gcda if failures: all_failures[uuid] = failures @@ -491,7 +505,7 @@ def build_coverage_cache( # pylint: disable=too-many-locals,too-many-statements cons.print(f" [yellow]{uuid}[/yellow]: {fail_str}") for target_name, _rc, tail in fails: if tail: - cons.print(f" {target_name} output (last 5 lines):") + cons.print(f" {target_name} output (last 15 lines):") for line in tail.splitlines(): cons.print(f" {line}") @@ -551,7 +565,12 @@ def build_coverage_cache( # pylint: disable=too-many-locals,too-many-statements f"have coverage data. Cache may be incomplete.[/bold yellow]") cases_py_path = Path(root_dir) / "toolchain/mfc/test/cases.py" - cases_hash = hashlib.sha256(cases_py_path.read_bytes()).hexdigest() + try: + cases_hash = hashlib.sha256(cases_py_path.read_bytes()).hexdigest() + except OSError as exc: + raise MFCException( + f"Failed to read {cases_py_path} for cache metadata: {exc}" + ) from exc gcov_version = _get_gcov_version(gcov_bin) cache["_meta"] = { @@ -560,8 +579,14 @@ def build_coverage_cache( # pylint: disable=too-many-locals,too-many-statements "gcov_version": gcov_version, } - with gzip.open(COVERAGE_CACHE_PATH, "wt", encoding="utf-8") as f: - json.dump(cache, f, indent=2) + try: + with gzip.open(COVERAGE_CACHE_PATH, "wt", encoding="utf-8") as f: + json.dump(cache, f, indent=2) + except OSError as exc: + raise MFCException( + f"Failed to write coverage cache to {COVERAGE_CACHE_PATH}: {exc}\n" + "Check disk space and filesystem permissions." + ) from exc cons.print() cons.print(f"[bold green]Coverage cache written to {COVERAGE_CACHE_PATH}[/bold green]") @@ -574,10 +599,19 @@ def _normalize_cache(cache: dict) -> dict: Old format: {uuid: {file: [lines], ...}, ...} New format: {uuid: [file, ...], ...} """ - return { - k: (sorted(v.keys()) if k != "_meta" and isinstance(v, dict) else v) - for k, v in cache.items() - } + result = {} + for k, v in cache.items(): + if k == "_meta": + result[k] = v + elif isinstance(v, dict): + result[k] = sorted(v.keys()) + elif isinstance(v, list): + result[k] = v + else: + cons.print(f"[yellow]Warning: unexpected cache value type for {k}: " + f"{type(v).__name__} — treating as empty.[/yellow]") + result[k] = [] + return result def load_coverage_cache(root_dir: str) -> Optional[dict]: @@ -597,6 +631,10 @@ def load_coverage_cache(root_dir: str) -> Optional[dict]: cons.print("[yellow]Warning: Coverage cache is unreadable or corrupt.[/yellow]") return None + if not isinstance(cache, dict): + cons.print("[yellow]Warning: Coverage cache has unexpected format.[/yellow]") + return None + cases_py = Path(root_dir) / "toolchain/mfc/test/cases.py" try: current_hash = hashlib.sha256(cases_py.read_bytes()).hexdigest() @@ -678,9 +716,9 @@ def filter_tests_by_coverage( Conservative behavior: - Test not in cache (newly added) -> include it - - No changed .fpp files -> skip all tests (safe because should_run_all_tests() - is called first in the workflow, catching toolchain/infra changes via - ALWAYS_RUN_ALL before this function is ever reached) + - No changed .fpp files -> skip all tests (this branch is unreachable from + test.py, which handles the no-changed-fpp case before calling this function; + retained as a safe fallback for direct callers) - Test has incomplete coverage (no simulation files recorded but simulation files changed) -> include it (cache build likely failed for this test) """ From 9719ea3d11d899abf841433e65223d07926ba67f Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Fri, 6 Mar 2026 03:41:13 -0500 Subject: [PATCH 27/35] Fix coverage cache: remove SYSCHECK, short-circuit on failure, fix progress counters - Remove SYSCHECK from cache-build targets (never built in this context, caused 459 spurious warnings per cache rebuild) - Break binary execution loop on missing binary or failed target to prevent downstream targets from producing init-only gcda files with garbage coverage - Warn with byte offset when gcov JSON parse truncates mid-stream - Fix Phase 2/3 progress counters to use len(test_infos)/len(test_results) instead of len(cases) so the final line prints correctly when tests are skipped - Include the actual exception in load_coverage_cache corrupt-cache warning - Add --only-changes to retry PASSTHROUGH so PR runs retry with same scope - Rename phase1_jobs -> phase2_jobs (Phase 1 is serial; cap is for Phase 2) Co-Authored-By: Claude Sonnet 4.6 --- .github/scripts/run-tests-with-retry.sh | 2 +- toolchain/mfc/test/coverage.py | 59 ++++++++++++++++--------- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/.github/scripts/run-tests-with-retry.sh b/.github/scripts/run-tests-with-retry.sh index d6e4106dc3..a625b4ae9d 100755 --- a/.github/scripts/run-tests-with-retry.sh +++ b/.github/scripts/run-tests-with-retry.sh @@ -8,7 +8,7 @@ PASSTHROUGH="" for arg in "$@"; do case "$arg" in - --test-all|--single|--debug|--gcov) PASSTHROUGH="$PASSTHROUGH $arg" ;; + --test-all|--single|--debug|--gcov|--only-changes) PASSTHROUGH="$PASSTHROUGH $arg" ;; esac done diff --git a/toolchain/mfc/test/coverage.py b/toolchain/mfc/test/coverage.py index 2b85add7ed..a77d5353d0 100644 --- a/toolchain/mfc/test/coverage.py +++ b/toolchain/mfc/test/coverage.py @@ -30,7 +30,7 @@ from ..printer import cons from .. import common from ..common import MFCException -from ..build import PRE_PROCESS, SIMULATION, POST_PROCESS, SYSCHECK +from ..build import PRE_PROCESS, SIMULATION, POST_PROCESS from .case import (input_bubbles_lagrange, get_post_process_mods, POST_PROCESS_3D_PARAMS) @@ -184,6 +184,11 @@ def _parse_gcov_json_output(raw_bytes: bytes, root_dir: str) -> set: data, end_pos = decoder.raw_decode(text, pos) pos = end_pos except json.JSONDecodeError: + remaining = len(text) - pos + if remaining > 0: + cons.print(f"[yellow]Warning: gcov JSON parse error at offset " + f"{pos} ({remaining} bytes remaining) — partial " + f"coverage recorded for this test.[/yellow]") break for file_entry in data.get("files", []): @@ -319,9 +324,13 @@ def _run_single_test_direct(test_info: dict, gcda_dir: str, strip: str) -> tuple failures = [] for target_name, bin_path in binaries: if not os.path.isfile(bin_path): - cons.print(f"[yellow]Warning: binary {target_name} not found " - f"at {bin_path} for test {uuid}[/yellow]") - continue + # Record missing binary as a failure and stop: downstream targets + # depend on outputs from earlier ones (e.g. simulation needs the + # grid from pre_process), so running them without a predecessor + # produces misleading init-only gcda files. + failures.append((target_name, "missing-binary", + f"binary not found: {bin_path}")) + break # Verify .inp file exists before running (diagnostic for transient # filesystem issues where the file goes missing between phases). @@ -329,7 +338,7 @@ def _run_single_test_direct(test_info: dict, gcda_dir: str, strip: str) -> tuple if not os.path.isfile(inp_file): failures.append((target_name, "missing-inp", f"{inp_file} not found before launch")) - continue + break cmd = mpi_cmd + [bin_path] try: @@ -337,13 +346,18 @@ def _run_single_test_direct(test_info: dict, gcda_dir: str, strip: str) -> tuple stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env, cwd=test_dir, timeout=600) if result.returncode != 0: - # Save last lines of output for debugging. + # Save last lines of output for debugging. Stop here: a + # failed pre_process/simulation leaves no valid outputs for + # the next target, and running it produces spurious coverage. tail = "\n".join(result.stdout.strip().splitlines()[-15:]) failures.append((target_name, result.returncode, tail)) + break except subprocess.TimeoutExpired: failures.append((target_name, "timeout", "")) + break except (subprocess.SubprocessError, OSError) as exc: failures.append((target_name, str(exc), "")) + break return uuid, test_gcda, failures @@ -375,9 +389,9 @@ def _prepare_test(case, root_dir: str) -> dict: # pylint: disable=unused-argume case.params.update(get_post_process_mods(case.params)) # Run only one timestep: we only need to know which source files are - # *touched*, not verify correctness. A single step exercises the same - # code paths (init, RHS, time-stepper, output) while preventing heavy - # 3D tests from timing out under gcov instrumentation (~10x slowdown). + # *touched*, not verify correctness. A single step exercises the key + # code paths across all three executables while preventing heavy 3D tests + # from timing out under gcov instrumentation (~10x slowdown). case.params['t_step_stop'] = 1 # Adaptive-dt tests: post_process computes n_save = int(t_stop/t_save)+1 @@ -403,7 +417,9 @@ def _prepare_test(case, root_dir: str) -> dict: # pylint: disable=unused-argume # Write .inp files directly (no subprocess, no Mako templates). # Suppress console output from get_inp() to avoid one message per (test, target) pair. - targets = [SYSCHECK, PRE_PROCESS, SIMULATION, POST_PROCESS] + # Run all three executables to capture coverage across the full pipeline + # (pre_process: grid/IC generation; simulation: RHS/time-stepper; post_process: field I/O). + targets = [PRE_PROCESS, SIMULATION, POST_PROCESS] binaries = [] # NOTE: not thread-safe — Phase 1 must remain single-threaded. orig_file = cons.raw.file @@ -451,11 +467,12 @@ def build_coverage_cache( # pylint: disable=too-many-locals,too-many-statements if n_jobs is None: n_jobs = max(os.cpu_count() or 1, 1) - # Cap test parallelism: each test spawns gcov-instrumented MPI processes - # (~2-5 GB each under gcov). Too many concurrent tests cause OOM. - phase1_jobs = min(n_jobs, 16) + # Cap Phase 2 test parallelism: each test spawns gcov-instrumented MPI + # processes (~2-5 GB each under gcov). Too many concurrent tests cause OOM. + # Phase 3 gcov workers run at full n_jobs (gcov is lightweight by comparison). + phase2_jobs = min(n_jobs, 16) cons.print(f"[bold]Building coverage cache for {len(cases)} tests " - f"({phase1_jobs} test workers, {n_jobs} gcov workers)...[/bold]") + f"({phase2_jobs} test workers, {n_jobs} gcov workers)...[/bold]") cons.print(f"[dim]Using gcov binary: {gcov_bin}[/dim]") cons.print(f"[dim]Found {len(gcno_files)} .gcno files[/dim]") cons.print(f"[dim]GCOV_PREFIX_STRIP={strip}[/dim]") @@ -479,7 +496,7 @@ def build_coverage_cache( # pylint: disable=too-many-locals,too-many-statements cons.print("[bold]Phase 2/3: Running tests...[/bold]") test_results: dict = {} all_failures: dict = {} - with ThreadPoolExecutor(max_workers=phase1_jobs) as pool: + with ThreadPoolExecutor(max_workers=phase2_jobs) as pool: futures = { pool.submit(_run_single_test_direct, info, gcda_dir, strip): info for info in test_infos @@ -494,8 +511,8 @@ def build_coverage_cache( # pylint: disable=too-many-locals,too-many-statements test_results[uuid] = test_gcda if failures: all_failures[uuid] = failures - if (i + 1) % 50 == 0 or (i + 1) == len(cases): - cons.print(f" [{i+1:3d}/{len(cases):3d}] tests completed") + if (i + 1) % 50 == 0 or (i + 1) == len(test_infos): + cons.print(f" [{i+1:3d}/{len(test_infos):3d}] tests completed") if all_failures: cons.print() @@ -548,8 +565,8 @@ def build_coverage_cache( # pylint: disable=too-many-locals,too-many-statements coverage = [] cache[uuid] = coverage completed += 1 - if completed % 50 == 0 or completed == len(cases): - cons.print(f" [{completed:3d}/{len(cases):3d}] tests processed") + if completed % 50 == 0 or completed == len(test_results): + cons.print(f" [{completed:3d}/{len(test_results):3d}] tests processed") finally: shutil.rmtree(gcda_dir, ignore_errors=True) @@ -627,8 +644,8 @@ def load_coverage_cache(root_dir: str) -> Optional[dict]: try: with gzip.open(COVERAGE_CACHE_PATH, "rt", encoding="utf-8") as f: cache = json.load(f) - except (OSError, gzip.BadGzipFile, json.JSONDecodeError, UnicodeDecodeError): - cons.print("[yellow]Warning: Coverage cache is unreadable or corrupt.[/yellow]") + except (OSError, gzip.BadGzipFile, json.JSONDecodeError, UnicodeDecodeError) as exc: + cons.print(f"[yellow]Warning: Coverage cache is unreadable or corrupt: {exc}[/yellow]") return None if not isinstance(cache, dict): From df97a8adcf214b90567b57adda37d36e78176c74 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Fri, 6 Mar 2026 11:51:23 -0500 Subject: [PATCH 28/35] Clean up test output dirs after cache build to reduce NFS pressure The cache rebuild runs 459 tests which each write grid files, restart files, and simulation output to the shared scratch NFS filesystem. Previously these were never cleaned up, leaving several GB of data on NFS between the rebuild job finishing and the subsequent test jobs starting. This I/O pressure can destabilize the NFS mount and cause transient failures (missing silo files) in the main test jobs. Co-Authored-By: Claude Sonnet 4.6 --- toolchain/mfc/test/coverage.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/toolchain/mfc/test/coverage.py b/toolchain/mfc/test/coverage.py index a77d5353d0..5bb534489b 100644 --- a/toolchain/mfc/test/coverage.py +++ b/toolchain/mfc/test/coverage.py @@ -609,6 +609,17 @@ def build_coverage_cache( # pylint: disable=too-many-locals,too-many-statements cons.print(f"[bold green]Coverage cache written to {COVERAGE_CACHE_PATH}[/bold green]") cons.print(f"[dim]Cache has {len(cases)} test entries.[/dim]") + # Clean up test output directories from Phase 1/2 (grid files, restart files, + # silo output, etc.). These live on NFS scratch and can total several GB for + # the full test suite. Leaving them behind creates I/O pressure for subsequent + # test jobs that share the same scratch filesystem. + cons.print("[dim]Cleaning up test output directories...[/dim]") + for case in cases: + try: + case.delete_output() + except OSError: + pass # Best-effort; NFS errors are non-fatal here + def _normalize_cache(cache: dict) -> dict: """Convert old line-level cache format to file-level if needed. From 44e15ef2fbdd0a4c95d8414c343746e52add5c85 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Fri, 6 Mar 2026 11:39:45 -0500 Subject: [PATCH 29/35] Remove persistent build cache for self-hosted test runners Replace setup-build-cache.sh symlink mechanism with rm -rf build before each test run on Phoenix and Frontier. Benchmark jobs unaffected. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/frontier/build.sh | 3 +-- .github/workflows/phoenix/test.sh | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/frontier/build.sh b/.github/workflows/frontier/build.sh index 88446ad2a0..6abb0cff8a 100644 --- a/.github/workflows/frontier/build.sh +++ b/.github/workflows/frontier/build.sh @@ -20,9 +20,8 @@ build_opts="$gpu_opts" . ./mfc.sh load -c $compiler_flag -m $([ "$job_device" = "gpu" ] && echo "g" || echo "c") -# Only set up build cache for test suite, not benchmarks if [ "$run_bench" != "bench" ]; then - source .github/scripts/setup-build-cache.sh "$cluster_name" "$job_device" "$job_interface" + rm -rf build fi source .github/scripts/retry-build.sh diff --git a/.github/workflows/phoenix/test.sh b/.github/workflows/phoenix/test.sh index 4422adb054..5c2d57d27f 100644 --- a/.github/workflows/phoenix/test.sh +++ b/.github/workflows/phoenix/test.sh @@ -3,8 +3,7 @@ source .github/scripts/gpu-opts.sh build_opts="$gpu_opts" -# Set up persistent build cache -source .github/scripts/setup-build-cache.sh phoenix "$job_device" "$job_interface" +rm -rf build # Build with retry; smoke-test cached binaries to catch architecture mismatches # (SIGILL from binaries compiled on a different compute node). From a010a9a34d2afecb397ba0e432d6e203d28a6b59 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Fri, 6 Mar 2026 11:50:47 -0500 Subject: [PATCH 30/35] Remove build cache from benchmark jobs on Phoenix and Frontier --- .github/workflows/frontier/build.sh | 4 +--- .github/workflows/phoenix/bench.sh | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/frontier/build.sh b/.github/workflows/frontier/build.sh index 6abb0cff8a..d21b1ddac4 100644 --- a/.github/workflows/frontier/build.sh +++ b/.github/workflows/frontier/build.sh @@ -20,9 +20,7 @@ build_opts="$gpu_opts" . ./mfc.sh load -c $compiler_flag -m $([ "$job_device" = "gpu" ] && echo "g" || echo "c") -if [ "$run_bench" != "bench" ]; then - rm -rf build -fi +rm -rf build source .github/scripts/retry-build.sh if [ "$run_bench" == "bench" ]; then diff --git a/.github/workflows/phoenix/bench.sh b/.github/workflows/phoenix/bench.sh index 89406942f7..9a661cb924 100644 --- a/.github/workflows/phoenix/bench.sh +++ b/.github/workflows/phoenix/bench.sh @@ -19,6 +19,8 @@ else bench_opts="--mem 1" fi +rm -rf build + source .github/scripts/retry-build.sh RETRY_CLEAN_CMD="./mfc.sh clean" retry_build ./mfc.sh build -j $n_jobs $build_opts || exit 1 From 2b52d57172a69320a46ac8e07b2c3ab7787ba7f1 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Fri, 6 Mar 2026 14:10:18 -0500 Subject: [PATCH 31/35] Fix submit.sh to survive monitor SIGKILL by re-checking SLURM state When the runner process is killed (exit 137) before the SLURM job completes, sacct is used to verify the job's final state. If the SLURM job completed with exit 0:0, the CI step passes regardless of the monitor's exit code. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/phoenix/submit.sh | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/phoenix/submit.sh b/.github/workflows/phoenix/submit.sh index 2ab1c0349e..4d3716daac 100755 --- a/.github/workflows/phoenix/submit.sh +++ b/.github/workflows/phoenix/submit.sh @@ -97,4 +97,20 @@ echo "Submitted batch job $job_id" # Use resilient monitoring instead of sbatch -W SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -bash "$SCRIPT_DIR/../../scripts/monitor_slurm_job.sh" "$job_id" "$output_file" +monitor_exit=0 +bash "$SCRIPT_DIR/../../scripts/monitor_slurm_job.sh" "$job_id" "$output_file" || monitor_exit=$? + +if [ "$monitor_exit" -ne 0 ]; then + echo "Monitor exited with code $monitor_exit; re-checking SLURM job $job_id final state..." + # Give the SLURM epilog time to finalize if the job just finished + sleep 30 + final_state=$(sacct -j "$job_id" -n -X -P -o State 2>/dev/null | head -n1 | cut -d'|' -f1 | tr -d ' ' || echo "UNKNOWN") + final_exit=$(sacct -j "$job_id" --format=ExitCode --noheader --parsable2 2>/dev/null | head -n1 | tr -d ' ' || echo "") + echo "Final SLURM state=$final_state exit=$final_exit" + if [ "$final_state" = "COMPLETED" ] && [ "$final_exit" = "0:0" ]; then + echo "SLURM job $job_id completed successfully despite monitor failure — continuing." + else + echo "ERROR: SLURM job $job_id did not complete successfully (state=$final_state exit=$final_exit)" + exit 1 + fi +fi From 2caf95fcdbf5969b21bda7689b8cbccea2de5e6b Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Fri, 6 Mar 2026 14:28:40 -0500 Subject: [PATCH 32/35] Extract monitor SIGKILL recovery into shared run_monitored_slurm_job.sh All three submit.sh scripts (phoenix, frontier, frontier_amd symlink) now call a single helper that wraps monitor_slurm_job.sh with sacct fallback: if the monitor is killed before the SLURM job completes, the helper re-checks the job's final state and exits 0 if it succeeded. Co-Authored-By: Claude Sonnet 4.6 --- .github/scripts/run_monitored_slurm_job.sh | 37 ++++++++++++++++++++++ .github/workflows/frontier/submit.sh | 3 +- .github/workflows/phoenix/submit.sh | 19 +---------- 3 files changed, 39 insertions(+), 20 deletions(-) create mode 100644 .github/scripts/run_monitored_slurm_job.sh diff --git a/.github/scripts/run_monitored_slurm_job.sh b/.github/scripts/run_monitored_slurm_job.sh new file mode 100644 index 0000000000..905520c45e --- /dev/null +++ b/.github/scripts/run_monitored_slurm_job.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Run monitor_slurm_job.sh and recover if the monitor is killed (e.g. SIGKILL +# from the runner OS) before the SLURM job completes. When the monitor exits +# non-zero, sacct is used to verify the job's actual final state; if the SLURM +# job succeeded we exit 0 so the CI step is not falsely marked as failed. +# +# Usage: run_monitored_slurm_job.sh + +set -euo pipefail + +if [ $# -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi + +job_id="$1" +output_file="$2" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +monitor_exit=0 +bash "$SCRIPT_DIR/monitor_slurm_job.sh" "$job_id" "$output_file" || monitor_exit=$? + +if [ "$monitor_exit" -ne 0 ]; then + echo "Monitor exited with code $monitor_exit; re-checking SLURM job $job_id final state..." + # Give the SLURM epilog time to finalize if the job just finished + sleep 30 + final_state=$(sacct -j "$job_id" -n -X -P -o State 2>/dev/null | head -n1 | cut -d'|' -f1 | tr -d ' ' || echo "UNKNOWN") + final_exit=$(sacct -j "$job_id" --format=ExitCode --noheader --parsable2 2>/dev/null | head -n1 | tr -d ' ' || echo "") + echo "Final SLURM state=$final_state exit=$final_exit" + if [ "$final_state" = "COMPLETED" ] && [ "$final_exit" = "0:0" ]; then + echo "SLURM job $job_id completed successfully despite monitor failure — continuing." + else + echo "ERROR: SLURM job $job_id did not complete successfully (state=$final_state exit=$final_exit)" + exit 1 + fi +fi diff --git a/.github/workflows/frontier/submit.sh b/.github/workflows/frontier/submit.sh index 743d70d95f..37157cf934 100644 --- a/.github/workflows/frontier/submit.sh +++ b/.github/workflows/frontier/submit.sh @@ -96,5 +96,4 @@ fi echo "Submitted batch job $job_id" -# Use resilient monitoring instead of sbatch -W -bash "$SCRIPT_DIR/../../scripts/monitor_slurm_job.sh" "$job_id" "$output_file" +bash "$SCRIPT_DIR/../../scripts/run_monitored_slurm_job.sh" "$job_id" "$output_file" diff --git a/.github/workflows/phoenix/submit.sh b/.github/workflows/phoenix/submit.sh index 4d3716daac..61ea80635d 100755 --- a/.github/workflows/phoenix/submit.sh +++ b/.github/workflows/phoenix/submit.sh @@ -95,22 +95,5 @@ fi echo "Submitted batch job $job_id" -# Use resilient monitoring instead of sbatch -W SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -monitor_exit=0 -bash "$SCRIPT_DIR/../../scripts/monitor_slurm_job.sh" "$job_id" "$output_file" || monitor_exit=$? - -if [ "$monitor_exit" -ne 0 ]; then - echo "Monitor exited with code $monitor_exit; re-checking SLURM job $job_id final state..." - # Give the SLURM epilog time to finalize if the job just finished - sleep 30 - final_state=$(sacct -j "$job_id" -n -X -P -o State 2>/dev/null | head -n1 | cut -d'|' -f1 | tr -d ' ' || echo "UNKNOWN") - final_exit=$(sacct -j "$job_id" --format=ExitCode --noheader --parsable2 2>/dev/null | head -n1 | tr -d ' ' || echo "") - echo "Final SLURM state=$final_state exit=$final_exit" - if [ "$final_state" = "COMPLETED" ] && [ "$final_exit" = "0:0" ]; then - echo "SLURM job $job_id completed successfully despite monitor failure — continuing." - else - echo "ERROR: SLURM job $job_id did not complete successfully (state=$final_state exit=$final_exit)" - exit 1 - fi -fi +bash "$SCRIPT_DIR/../../scripts/run_monitored_slurm_job.sh" "$job_id" "$output_file" From baa49bf4b8cca8b9d9102e4d547d5fc2d5be9f3c Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Fri, 6 Mar 2026 15:14:25 -0500 Subject: [PATCH 33/35] Fix bench: use PR's submit.sh for master job to get SIGKILL recovery When benchmarking master vs PR, submit_and_monitor_bench.sh was using the master directory's submit.sh for the master bench job. Master's submit.sh calls monitor_slurm_job.sh directly without SIGKILL recovery. When the monitor was killed (exit 137), the master bench YAML was never found. Fix: always use the PR's submit.sh (which calls run_monitored_slurm_job.sh with sacct fallback) for both master and PR bench submissions. Co-Authored-By: Claude Sonnet 4.6 --- .github/scripts/submit_and_monitor_bench.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/scripts/submit_and_monitor_bench.sh b/.github/scripts/submit_and_monitor_bench.sh index c081c8692a..9eae6b9ff7 100755 --- a/.github/scripts/submit_and_monitor_bench.sh +++ b/.github/scripts/submit_and_monitor_bench.sh @@ -17,9 +17,13 @@ cluster="$4" echo "[$dir] Submitting benchmark for $device-$interface on $cluster..." cd "$dir" -# Submit and monitor job (submit.sh auto-detects bench mode from script name) -bash .github/workflows/$cluster/submit.sh \ - .github/workflows/$cluster/bench.sh "$device" "$interface" +# Always use the PR's submit.sh so both master and PR builds benefit from the +# run_monitored_slurm_job.sh SIGKILL recovery wrapper. The bench script is +# still resolved relative to the current directory (master/ or pr/) so the +# correct branch code is benchmarked. SLURM_SUBMIT_DIR ensures the job runs +# in the right directory regardless of which submit.sh is invoked. +PR_SUBMIT="${SCRIPT_DIR}/../workflows/${cluster}/submit.sh" +bash "$PR_SUBMIT" .github/workflows/$cluster/bench.sh "$device" "$interface" # Verify the YAML output file was created job_slug="bench-$device-$interface" From a84cabca59df4e7d9b5b071e17fde18af642e059 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Fri, 6 Mar 2026 15:21:10 -0500 Subject: [PATCH 34/35] Fix submit_and_monitor_bench.sh: define SCRIPT_DIR before use Co-Authored-By: Claude Sonnet 4.6 --- .github/scripts/submit_and_monitor_bench.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/scripts/submit_and_monitor_bench.sh b/.github/scripts/submit_and_monitor_bench.sh index 9eae6b9ff7..e0a6eb7384 100755 --- a/.github/scripts/submit_and_monitor_bench.sh +++ b/.github/scripts/submit_and_monitor_bench.sh @@ -14,6 +14,8 @@ device="$2" interface="$3" cluster="$4" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + echo "[$dir] Submitting benchmark for $device-$interface on $cluster..." cd "$dir" From a7cfc5c8b0cf07a434f9f9c548d5f26564965513 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sat, 7 Mar 2026 03:43:34 -0500 Subject: [PATCH 35/35] bench: update Phoenix tmpbuild path to project storage --- .github/workflows/phoenix/bench.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/phoenix/bench.sh b/.github/workflows/phoenix/bench.sh index 9a661cb924..218cf68a5f 100644 --- a/.github/workflows/phoenix/bench.sh +++ b/.github/workflows/phoenix/bench.sh @@ -6,7 +6,7 @@ source .github/scripts/bench-preamble.sh # (GNR nodes have 192 cores but nproc is too aggressive for build/bench). n_jobs=$(( $(nproc) > 64 ? 64 : $(nproc) )) -tmpbuild=/storage/scratch1/6/sbryngelson3/mytmp_build +tmpbuild=/storage/project/r-sbryngelson3-0/sbryngelson3/mytmp_build currentdir=$tmpbuild/run-$(( RANDOM % 900 )) mkdir -p $tmpbuild mkdir -p $currentdir