test.yml

name: Python CI

on:
  pull_request:
    paths-ignore:
      - "**.md"
      - "LICENSE"
  push:
    branches:
      - main
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  lint:
    name: Lint and Format
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v6

      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true

      - name: Pin Python version
        run: uv python pin 3.12

      - name: Install dependencies
        run: uv sync --extra dev

      - name: Lint with Ruff
        run: uv run ruff check src/

      - name: Check formatting
        run: uv run ruff format --check src/

  type-check:
    name: Type Check (${{ matrix.type-checker }})
    runs-on: ubuntu-latest
    timeout-minutes: 10
    strategy:
      matrix:
        type-checker: [mypy, pyright]
    steps:
      - uses: actions/checkout@v6

      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true

      - name: Pin Python version
        run: uv python pin 3.12

      - name: Install dependencies
        run: uv sync --extra dev

      - name: Type check with mypy
        if: matrix.type-checker == 'mypy'
        run: uv run mypy src/promptfoo/

      - name: Type check with pyright
        if: matrix.type-checker == 'pyright'
        run: uv run pyright src/promptfoo/

  test:
    name: Test (py${{ matrix.python-version }}, ${{ matrix.os }})
    runs-on: ${{ matrix.os }}
    timeout-minutes: 30
    strategy:
      matrix:
        # Temporarily excluding macos-latest due to GitHub Actions runner resource constraints
        # causing BlockingIOError [Errno 35] when spawning subprocess
        os: [ubuntu-latest, windows-latest]
        # Test only min and max supported Python versions for efficiency
        python-version: ["3.9", "3.13"]
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-node@v6
        with:
          node-version: "24"

      - name: Configure npm on Windows
        if: matrix.os == 'windows-latest'
        shell: pwsh
        run: |
          # Configure cache location (applies immediately to this step)
          $cacheDir = Join-Path $env:RUNNER_TEMP "npm-cache"
          New-Item -ItemType Directory -Force -Path $cacheDir | Out-Null
          npm config set cache $cacheDir --location=user

          # Configure prefix location (applies immediately to this step)
          $globalPrefix = npm config get prefix
          if (-not $globalPrefix -or $globalPrefix -eq "undefined") {
            $globalPrefix = Join-Path $env:APPDATA "npm"
          }
          $globalPrefix = $globalPrefix.Trim()
          npm config set prefix $globalPrefix --location=user

          # NOW clean and verify cache (cleans the correctly-configured cache)
          npm cache clean --force
          npm cache verify

          # Export settings for future steps
          "NPM_CONFIG_CACHE=$cacheDir" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
          "NPM_CONFIG_PREFIX=$globalPrefix" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
          "npm_config_prefix=$globalPrefix" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append

          # Add global bin directories to PATH
          $binPaths = @($globalPrefix, (Join-Path $globalPrefix "bin")) | Where-Object { Test-Path $_ }
          foreach ($binPath in $binPaths) {
            $binPath | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
          }

          Write-Host "npm cache: $cacheDir"
          Write-Host "npm prefix: $globalPrefix"

      - name: Install promptfoo globally
        run: npm install -g promptfoo@latest
        env:
          NODE_OPTIONS: --max-old-space-size=4096

      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true

      - name: Pin Python version
        run: uv python pin ${{ matrix.python-version }}

      - name: Install package with dev dependencies
        run: uv sync --extra dev

      - name: Run unit tests
        run: uv run pytest tests/ -v -m 'not smoke'

      - name: Run smoke tests
        run: uv run pytest tests/smoke/ -v

      - name: Test CLI can be invoked
        run: uv run promptfoo --version

      - name: Test Node.js detection
        run: uv run python -c "from promptfoo.cli import check_node_installed, check_npx_installed; assert check_node_installed(); assert check_npx_installed()"

  test-npx-fallback:
    name: Test npx fallback (py${{ matrix.python-version }}, ${{ matrix.os }})
    runs-on: ${{ matrix.os }}
    timeout-minutes: 30
    strategy:
      matrix:
        # Test npx fallback (without global install)
        # Temporarily excluding macos-latest due to GitHub Actions runner resource constraints
        os: [ubuntu-latest, windows-latest]
        # Use middle-version Python for this test
        python-version: ["3.12"]
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-node@v6
        with:
          node-version: "24"

      - name: Configure npm on Windows
        if: matrix.os == 'windows-latest'
        shell: pwsh
        run: |
          # Configure cache location (applies immediately to this step)
          $cacheDir = Join-Path $env:RUNNER_TEMP "npm-cache"
          New-Item -ItemType Directory -Force -Path $cacheDir | Out-Null
          npm config set cache $cacheDir --location=user

          # Configure prefix location (applies immediately to this step)
          $globalPrefix = npm config get prefix
          if (-not $globalPrefix -or $globalPrefix -eq "undefined") {
            $globalPrefix = Join-Path $env:APPDATA "npm"
          }
          $globalPrefix = $globalPrefix.Trim()
          npm config set prefix $globalPrefix --location=user

          # NOW clean and verify cache (cleans the correctly-configured cache)
          npm cache clean --force
          npm cache verify

          # Export settings for future steps
          "NPM_CONFIG_CACHE=$cacheDir" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
          "NPM_CONFIG_PREFIX=$globalPrefix" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
          "npm_config_prefix=$globalPrefix" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append

          Write-Host "npm cache: $cacheDir"
          Write-Host "npm prefix: $globalPrefix"

      # Intentionally skip installing promptfoo globally
      # This tests the npx fallback path

      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true

      - name: Pin Python version
        run: uv python pin ${{ matrix.python-version }}

      - name: Install package with dev dependencies
        run: uv sync --extra dev

      - name: Run unit tests
        run: uv run pytest tests/ -v -m 'not smoke'

      - name: Run smoke tests (with npx fallback)
        run: uv run pytest tests/smoke/ -v

      - name: Test CLI fallback to npx (no global install)
        run: uv run promptfoo --version

      - name: Test Node.js detection
        run: uv run python -c "from promptfoo.cli import check_node_installed, check_npx_installed; assert check_node_installed(); assert check_npx_installed()"

  build:
    name: Build Package
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v6

      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true

      - name: Pin Python version
        run: uv python pin 3.12

      - name: Build package
        run: uv build

      - name: Upload artifacts
        uses: actions/upload-artifact@v7
        with:
          name: dist
          path: dist/

  ci-success:
    name: CI Success
    needs: [lint, type-check, test, test-npx-fallback, build]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Check if all jobs succeeded
        run: |
          LINT_RESULT="${{ needs.lint.result }}"
          TYPE_CHECK_RESULT="${{ needs.type-check.result }}"
          TEST_RESULT="${{ needs.test.result }}"
          TEST_NPX_FALLBACK_RESULT="${{ needs.test-npx-fallback.result }}"
          BUILD_RESULT="${{ needs.build.result }}"

          echo "Job results:"
          echo "  lint: $LINT_RESULT"
          echo "  type-check: $TYPE_CHECK_RESULT"
          echo "  test: $TEST_RESULT"
          echo "  test-npx-fallback: $TEST_NPX_FALLBACK_RESULT"
          echo "  build: $BUILD_RESULT"

          if [[ "$LINT_RESULT" == "failure" || "$LINT_RESULT" == "cancelled" ||
                "$TYPE_CHECK_RESULT" == "failure" || "$TYPE_CHECK_RESULT" == "cancelled" ||
                "$TEST_RESULT" == "failure" || "$TEST_RESULT" == "cancelled" ||
                "$TEST_NPX_FALLBACK_RESULT" == "failure" || "$TEST_NPX_FALLBACK_RESULT" == "cancelled" ||
                "$BUILD_RESULT" == "failure" || "$BUILD_RESULT" == "cancelled" ]]; then
            echo "Some CI checks failed!"
            exit 1
          else
            echo "All CI checks passed!"
            exit 0
          fi