From 807669e454a0c2f635bc207f8e05a9feb2ec5970 Mon Sep 17 00:00:00 2001 From: ali Date: Sat, 9 May 2026 23:07:46 +0300 Subject: [PATCH] fix stdout comparator when it's empty --- codeflash/verification/equivalence.py | 10 ++--- tests/test_comparator.py | 56 ++++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/codeflash/verification/equivalence.py b/codeflash/verification/equivalence.py index f660e35ea..0fbf6f2fb 100644 --- a/codeflash/verification/equivalence.py +++ b/codeflash/verification/equivalence.py @@ -129,16 +129,12 @@ def compare_test_results( ) except Exception as e: logger.error(e) - elif ( - not pass_fail_only - and (original_test_result.stdout and cdd_test_result.stdout) - and not comparator(original_test_result.stdout, cdd_test_result.stdout) - ): + elif not pass_fail_only and not comparator(original_test_result.stdout or "", cdd_test_result.stdout or ""): test_diffs.append( TestDiff( scope=TestDiffScope.STDOUT, - original_value=str(original_test_result.stdout), - candidate_value=str(cdd_test_result.stdout), + original_value=original_test_result.stdout or "", + candidate_value=cdd_test_result.stdout or "", test_src_code=original_test_result.id.get_src_code(original_test_result.file_name), candidate_pytest_error=cdd_pytest_error, original_pass=original_test_result.did_pass, diff --git a/tests/test_comparator.py b/tests/test_comparator.py index 3c8190317..52d5192a4 100644 --- a/tests/test_comparator.py +++ b/tests/test_comparator.py @@ -16,7 +16,7 @@ import pytest from codeflash.either import Failure, Success -from codeflash.models.models import FunctionTestInvocation, InvocationId, TestResults, TestType +from codeflash.models.models import FunctionTestInvocation, InvocationId, TestDiffScope, TestResults, TestType from codeflash.verification.comparator import ( PYTEST_TEMP_PATH_PATTERN, PYTHON_TEMPFILE_PATTERN, @@ -2999,6 +2999,60 @@ def test_compare_results_fn(): assert not match +def test_compare_results_detects_stdout_mismatch_when_candidate_stdout_is_empty() -> None: + original_results = TestResults() + original_results.add( + FunctionTestInvocation( + id=InvocationId( + test_module_path="test_module_path", + test_class_name="test_class_name", + test_function_name="test_function_name", + function_getting_tested="function_getting_tested", + iteration_id="0", + ), + file_name=Path("file_name"), + did_pass=True, + runtime=5, + test_framework="pytest", + test_type=TestType.EXISTING_UNIT_TEST, + return_value=[0, 1, 2], + timed_out=False, + loop_index=1, + stdout="codeflash stdout: Sorting list\nresult: [0, 1, 2]\n", + ) + ) + + candidate_results = TestResults() + candidate_results.add( + FunctionTestInvocation( + id=InvocationId( + test_module_path="test_module_path", + test_class_name="test_class_name", + test_function_name="test_function_name", + function_getting_tested="function_getting_tested", + iteration_id="0", + ), + file_name=Path("file_name"), + did_pass=True, + runtime=5, + test_framework="pytest", + test_type=TestType.EXISTING_UNIT_TEST, + return_value=[0, 1, 2], + timed_out=False, + loop_index=1, + stdout="", + ) + ) + + match, diffs = compare_test_results(original_results, candidate_results) + + assert not match + assert len(diffs) == 1 + assert diffs[0].scope == TestDiffScope.STDOUT + assert diffs[0].original_value == "codeflash stdout: Sorting list\nresult: [0, 1, 2]\n" + assert diffs[0].candidate_value == "" + + def test_exceptions(): type_error = TypeError("This is a type error")