diff --git a/src/promptfoo/cli.py b/src/promptfoo/cli.py index 2d35896..c93dd80 100644 --- a/src/promptfoo/cli.py +++ b/src/promptfoo/cli.py @@ -15,6 +15,7 @@ _WRAPPER_ENV = "PROMPTFOO_PY_WRAPPER" _WINDOWS_SHELL_EXTENSIONS = (".bat", ".cmd") +_WINDOWS_STATUS_SIGN_BIT = 1 << 31 _VERSION_ENV = "PROMPTFOO_VERSION" @@ -173,6 +174,18 @@ def _run_command(cmd: list[str], env: Optional[dict[str, str]] = None) -> subpro return subprocess.run(cmd, env=env) +def _normalize_exit_code(returncode: int) -> int: + """Normalize subprocess return codes into portable process exit codes.""" + if os.name == "nt": + if returncode < 0 or returncode & _WINDOWS_STATUS_SIGN_BIT: + return 1 + return returncode + + if returncode < 0: + return 128 + min(abs(returncode), 127) + return returncode + + def main() -> NoReturn: """ Main entry point for the promptfoo CLI wrapper. @@ -207,7 +220,7 @@ def main() -> NoReturn: print("Or ensure Node.js is properly installed.", file=sys.stderr) sys.exit(1) - sys.exit(result.returncode) + sys.exit(_normalize_exit_code(result.returncode)) except KeyboardInterrupt: sys.exit(130) diff --git a/tests/test_cli.py b/tests/test_cli.py index 0e4a1c0..dded5bc 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -22,6 +22,7 @@ _WRAPPER_ENV, _find_external_promptfoo, _find_windows_promptfoo, + _normalize_exit_code, _normalize_path, _requires_shell, _resolve_argv0, @@ -358,6 +359,34 @@ def test_run_command_passes_environment(self, monkeypatch: pytest.MonkeyPatch) - assert call_args.kwargs.get("env") == env +class TestExitCodeNormalization: + """Test subprocess exit code normalization.""" + + @pytest.mark.parametrize("returncode", [0, 1, 100, 255]) + def test_normalize_exit_code_preserves_standard_codes( + self, returncode: int, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Standard shell exit codes pass through unchanged.""" + monkeypatch.setattr(os, "name", "nt") + + assert _normalize_exit_code(returncode) == returncode + + @pytest.mark.parametrize("returncode", [4294967295, 3221226505, -1]) + def test_normalize_exit_code_maps_windows_error_statuses( + self, returncode: int, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Windows unsigned and NTSTATUS failure values map to exit code 1.""" + monkeypatch.setattr(os, "name", "nt") + + assert _normalize_exit_code(returncode) == 1 + + def test_normalize_exit_code_maps_unix_signal_status(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Unix signal-style negative return codes map to 128 + signal number.""" + monkeypatch.setattr(os, "name", "posix") + + assert _normalize_exit_code(-15) == 143 + + # ============================================================================= # Integration Tests for main() # ============================================================================= @@ -533,6 +562,30 @@ def test_main_returns_subprocess_exit_code(self, monkeypatch: pytest.MonkeyPatch assert exc_info.value.code == 42 + @pytest.mark.parametrize("raw_returncode", [4294967295, 3221226505]) + def test_main_normalizes_windows_error_statuses(self, raw_returncode: int, monkeypatch: pytest.MonkeyPatch) -> None: + """Converts Windows-specific subprocess statuses into a stable exit code.""" + monkeypatch.setattr(os, "name", "nt") + monkeypatch.setattr(sys, "argv", ["promptfoo", "eval", "-c", "missing.yaml"]) + monkeypatch.setattr( + "shutil.which", + lambda cmd, path=None: { + "node": "C:\\Program Files\\nodejs\\node.exe", + "npx": "C:\\Program Files\\nodejs\\npx.cmd", + }.get(cmd), + ) + monkeypatch.setattr("promptfoo.cli.record_wrapper_used", lambda mode: None) + monkeypatch.setattr( + subprocess, + "run", + MagicMock(return_value=subprocess.CompletedProcess([], raw_returncode)), + ) + + with pytest.raises(SystemExit) as exc_info: + main() + + assert exc_info.value.code == 1 + # ============================================================================= # Platform-Specific Tests