diff --git a/changelog-entries/735.md b/changelog-entries/735.md new file mode 100644 index 000000000..e86208f00 --- /dev/null +++ b/changelog-entries/735.md @@ -0,0 +1 @@ +- Add optional `timeout` field to `tests.yaml` entries for per-test timeout configuration. Overridable via `PRECICE_SYSTEMTEST_TIMEOUT` environment variable. diff --git a/tools/tests/README.md b/tools/tests/README.md index 5675c47b6..a841d3968 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -331,8 +331,11 @@ test_suites: - fluid-openfoam - solid-fenics reference_result: ./flow-over-heated-plate/reference-results/fluid-openfoam_solid-fenics.tar.gz + timeout: 1200 ``` +The optional `timeout` field (in seconds) sets the maximum time for the solver run and fieldcompare phases of that specific case. If omitted, it defaults to `GLOBAL_TIMEOUT` (currently 900s, overridable via the `PRECICE_SYSTEMTEST_TIMEOUT` environment variable). + This defines two test suites, namely `openfoam_adapter_pr` and `openfoam_adapter_release`. Each of them defines which case combinations of which tutorials to run. ### Generate Reference Results diff --git a/tools/tests/generate_reference_results.py b/tools/tests/generate_reference_results.py index 055e7b31c..204dc2401 100644 --- a/tools/tests/generate_reference_results.py +++ b/tools/tests/generate_reference_results.py @@ -2,7 +2,7 @@ from metadata_parser.metdata import Tutorials, ReferenceResult from systemtests.TestSuite import TestSuites from systemtests.SystemtestArguments import SystemtestArguments -from systemtests.Systemtest import Systemtest +from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT from pathlib import Path from typing import List from paths import PRECICE_TESTS_DIR, PRECICE_TUTORIAL_DIR @@ -108,10 +108,12 @@ def main(): for test_suite in test_suites: tutorials = test_suite.cases_of_tutorial.keys() for tutorial in tutorials: - for case, reference_result in zip( - test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial]): + timeouts = test_suite.timeouts.get(tutorial, []) + for i, (case, reference_result) in enumerate(zip( + test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial])): + timeout = timeouts[i] if i < len(timeouts) and timeouts[i] is not None else GLOBAL_TIMEOUT systemtests_to_run.add( - Systemtest(tutorial, build_args, case, reference_result)) + Systemtest(tutorial, build_args, case, reference_result, timeout=timeout)) reference_result_per_tutorial = {} current_time_string = datetime.now().strftime('%Y-%m-%d %H:%M:%S') diff --git a/tools/tests/systemtests.py b/tools/tests/systemtests.py index 8a37670eb..36ffbf3c6 100644 --- a/tools/tests/systemtests.py +++ b/tools/tests/systemtests.py @@ -2,7 +2,7 @@ import argparse from pathlib import Path from systemtests.SystemtestArguments import SystemtestArguments -from systemtests.Systemtest import Systemtest, display_systemtestresults_as_table +from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT, display_systemtestresults_as_table from systemtests.TestSuite import TestSuites from metadata_parser.metdata import Tutorials, Case import logging @@ -58,10 +58,12 @@ def main(): for test_suite in test_suites_to_execute: tutorials = test_suite.cases_of_tutorial.keys() for tutorial in tutorials: - for case, reference_result in zip( - test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial]): + timeouts = test_suite.timeouts.get(tutorial, []) + for i, (case, reference_result) in enumerate(zip( + test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial])): + timeout = timeouts[i] if i < len(timeouts) and timeouts[i] is not None else GLOBAL_TIMEOUT systemtests_to_run.append( - Systemtest(tutorial, build_args, case, reference_result)) + Systemtest(tutorial, build_args, case, reference_result, timeout=timeout)) if not systemtests_to_run: raise RuntimeError("Did not find any Systemtests to execute.") diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index 6abc5a029..552f95d3d 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -19,7 +19,7 @@ import os -GLOBAL_TIMEOUT = 900 +GLOBAL_TIMEOUT = int(os.environ.get("PRECICE_SYSTEMTEST_TIMEOUT", 900)) SHORT_TIMEOUT = 10 @@ -134,6 +134,7 @@ class Systemtest: arguments: SystemtestArguments case_combination: CaseCombination reference_result: ReferenceResult + timeout: int = GLOBAL_TIMEOUT params_to_use: Dict[str, str] = field(init=False) env: Dict[str, str] = field(init=False) @@ -394,7 +395,7 @@ def _run_field_compare(self): cwd=self.system_test_dir) try: - stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT) + stdout, stderr = process.communicate(timeout=self.timeout) except KeyboardInterrupt as k: process.kill() raise KeyboardInterrupt from k @@ -483,7 +484,7 @@ def _run_tutorial(self): cwd=self.system_test_dir) try: - stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT) + stdout, stderr = process.communicate(timeout=self.timeout) except KeyboardInterrupt as k: process.kill() # process.send_signal(9) diff --git a/tools/tests/systemtests/TestSuite.py b/tools/tests/systemtests/TestSuite.py index 9d8c2ac72..c2790eddd 100644 --- a/tools/tests/systemtests/TestSuite.py +++ b/tools/tests/systemtests/TestSuite.py @@ -10,6 +10,7 @@ class TestSuite: name: str cases_of_tutorial: Dict[Tutorial, List[CaseCombination]] reference_results: Dict[Tutorial, List[ReferenceResult]] + timeouts: Dict[Tutorial, List] = field(default_factory=dict) def __repr__(self) -> str: return_string = f"Test suite: {self.name} contains:" @@ -48,6 +49,7 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): for test_suite_name in test_suites_raw: case_combinations_of_tutorial = {} reference_results_of_tutorial = {} + timeouts_of_tutorial = {} # iterate over tutorials: for tutorial_case in test_suites_raw[test_suite_name]['tutorials']: tutorial = parsed_tutorials.get_by_path(tutorial_case['path']) @@ -57,6 +59,7 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): if tutorial not in case_combinations_of_tutorial: case_combinations_of_tutorial[tutorial] = [] reference_results_of_tutorial[tutorial] = [] + timeouts_of_tutorial[tutorial] = [] all_case_combinations = tutorial.case_combinations case_combination_requested = CaseCombination.from_string_list( @@ -65,12 +68,13 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): case_combinations_of_tutorial[tutorial].append(case_combination_requested) reference_results_of_tutorial[tutorial].append(ReferenceResult( tutorial_case['reference_result'], case_combination_requested)) + timeouts_of_tutorial[tutorial].append(tutorial_case.get('timeout', None)) else: raise Exception( f"Could not find the following cases {tutorial_case['case-combination']} in the current metadata of tutorial {tutorial.name}") testsuites.append(TestSuite(test_suite_name, case_combinations_of_tutorial, - reference_results_of_tutorial)) + reference_results_of_tutorial, timeouts_of_tutorial)) return cls(testsuites) diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index fce38ea38..5b23d3e5d 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -32,6 +32,7 @@ test_suites: - fluid-openfoam - solid-nutils reference_result: ./flow-over-heated-plate/reference-results/fluid-openfoam_solid-nutils.tar.gz + timeout: 1200 calculix_test: tutorials: - path: perpendicular-flap