systemtests: add configurable timeout per test case

changelog-entries/735.md

@@ -0,0 +1 @@
+- Add optional `timeout` field to `tests.yaml` entries for per-test timeout configuration. Overridable via `PRECICE_SYSTEMTEST_TIMEOUT` environment variable.

tools/tests/README.md

@@ -331,8 +331,11 @@ test_suites:
           - fluid-openfoam
           - solid-fenics
         reference_result: ./flow-over-heated-plate/reference-results/fluid-openfoam_solid-fenics.tar.gz
+        timeout: 1200
 ```
 
+The optional `timeout` field (in seconds) sets the maximum time for the solver run and fieldcompare phases of that specific case. If omitted, it defaults to `GLOBAL_TIMEOUT` (currently 900s, overridable via the `PRECICE_SYSTEMTEST_TIMEOUT` environment variable).
+
 This defines two test suites, namely `openfoam_adapter_pr` and `openfoam_adapter_release`. Each of them defines which case combinations of which tutorials to run.
 
 ### Generate Reference Results

tools/tests/generate_reference_results.py

@@ -2,7 +2,7 @@
 from metadata_parser.metdata import Tutorials, ReferenceResult
 from systemtests.TestSuite import TestSuites
 from systemtests.SystemtestArguments import SystemtestArguments
-from systemtests.Systemtest import Systemtest
+from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT
 from pathlib import Path
 from typing import List
 from paths import PRECICE_TESTS_DIR, PRECICE_TUTORIAL_DIR
@@ -108,10 +108,12 @@ def main():
     for test_suite in test_suites:
         tutorials = test_suite.cases_of_tutorial.keys()
         for tutorial in tutorials:
-            for case, reference_result in zip(
-                    test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial]):
+            timeouts = test_suite.timeouts.get(tutorial, [])
+            for i, (case, reference_result) in enumerate(zip(
+                    test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial])):
+                timeout = timeouts[i] if i < len(timeouts) and timeouts[i] is not None else GLOBAL_TIMEOUT
                 systemtests_to_run.add(
-                    Systemtest(tutorial, build_args, case, reference_result))
+                    Systemtest(tutorial, build_args, case, reference_result, timeout=timeout))
 
     reference_result_per_tutorial = {}
     current_time_string = datetime.now().strftime('%Y-%m-%d %H:%M:%S')

tools/tests/systemtests.py

@@ -2,7 +2,7 @@
 import argparse
 from pathlib import Path
 from systemtests.SystemtestArguments import SystemtestArguments
-from systemtests.Systemtest import Systemtest, display_systemtestresults_as_table
+from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT, display_systemtestresults_as_table
 from systemtests.TestSuite import TestSuites
 from metadata_parser.metdata import Tutorials, Case
 import logging
@@ -58,10 +58,12 @@ def main():
         for test_suite in test_suites_to_execute:
             tutorials = test_suite.cases_of_tutorial.keys()
             for tutorial in tutorials:
-                for case, reference_result in zip(
-                        test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial]):
+                timeouts = test_suite.timeouts.get(tutorial, [])
+                for i, (case, reference_result) in enumerate(zip(
+                        test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial])):
+                    timeout = timeouts[i] if i < len(timeouts) and timeouts[i] is not None else GLOBAL_TIMEOUT
                     systemtests_to_run.append(
-                        Systemtest(tutorial, build_args, case, reference_result))
+                        Systemtest(tutorial, build_args, case, reference_result, timeout=timeout))
 
     if not systemtests_to_run:
         raise RuntimeError("Did not find any Systemtests to execute.")

tools/tests/systemtests/Systemtest.py

@@ -19,7 +19,7 @@
 import os
 
 
-GLOBAL_TIMEOUT = 900
+GLOBAL_TIMEOUT = int(os.environ.get("PRECICE_SYSTEMTEST_TIMEOUT", 900))
 SHORT_TIMEOUT = 10
 
 
@@ -134,6 +134,7 @@ class Systemtest:
     arguments: SystemtestArguments
     case_combination: CaseCombination
     reference_result: ReferenceResult
+    timeout: int = GLOBAL_TIMEOUT
     params_to_use: Dict[str, str] = field(init=False)
     env: Dict[str, str] = field(init=False)
 
@@ -394,7 +395,7 @@ def _run_field_compare(self):
                                        cwd=self.system_test_dir)
 
             try:
-                stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT)
+                stdout, stderr = process.communicate(timeout=self.timeout)
             except KeyboardInterrupt as k:
                 process.kill()
                 raise KeyboardInterrupt from k
@@ -483,7 +484,7 @@ def _run_tutorial(self):
                                        cwd=self.system_test_dir)
 
             try:
-                stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT)
+                stdout, stderr = process.communicate(timeout=self.timeout)
             except KeyboardInterrupt as k:
                 process.kill()
                 # process.send_signal(9)

tools/tests/systemtests/TestSuite.py

@@ -10,6 +10,7 @@ class TestSuite:
     name: str
     cases_of_tutorial: Dict[Tutorial, List[CaseCombination]]
     reference_results: Dict[Tutorial, List[ReferenceResult]]
+    timeouts: Dict[Tutorial, List] = field(default_factory=dict)
 
     def __repr__(self) -> str:
         return_string = f"Test suite: {self.name} contains:"
@@ -48,6 +49,7 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials):
             for test_suite_name in test_suites_raw:
                 case_combinations_of_tutorial = {}
                 reference_results_of_tutorial = {}
+                timeouts_of_tutorial = {}
                 # iterate over tutorials:
                 for tutorial_case in test_suites_raw[test_suite_name]['tutorials']:
                     tutorial = parsed_tutorials.get_by_path(tutorial_case['path'])
@@ -57,6 +59,7 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials):
                     if tutorial not in case_combinations_of_tutorial:
                         case_combinations_of_tutorial[tutorial] = []
                         reference_results_of_tutorial[tutorial] = []
+                        timeouts_of_tutorial[tutorial] = []
 
                     all_case_combinations = tutorial.case_combinations
                     case_combination_requested = CaseCombination.from_string_list(
@@ -65,12 +68,13 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials):
                         case_combinations_of_tutorial[tutorial].append(case_combination_requested)
                         reference_results_of_tutorial[tutorial].append(ReferenceResult(
                             tutorial_case['reference_result'], case_combination_requested))
+                        timeouts_of_tutorial[tutorial].append(tutorial_case.get('timeout', None))
                     else:
                         raise Exception(
                             f"Could not find the following cases {tutorial_case['case-combination']} in the current metadata of tutorial {tutorial.name}")
 
                 testsuites.append(TestSuite(test_suite_name, case_combinations_of_tutorial,
-                                            reference_results_of_tutorial))
+                                            reference_results_of_tutorial, timeouts_of_tutorial))
 
         return cls(testsuites)
 

tools/tests/tests.yaml

@@ -32,6 +32,7 @@ test_suites:
           - fluid-openfoam
           - solid-nutils
         reference_result: ./flow-over-heated-plate/reference-results/fluid-openfoam_solid-nutils.tar.gz
+        timeout: 1200
   calculix_test:
     tutorials:
       - path: perpendicular-flap