fix(tests): make sure multiple e2e tests run concurrently (#1861)

Author: rubenfonseca

.github/workflows/run-e2e-tests.yml

@@ -26,6 +26,7 @@ jobs:
       id-token: write # needed to request JWT with GitHub's OIDC Token endpoint. docs: https://bit.ly/3MNgQO9
       contents: read
     strategy:
+      fail-fast: false # needed so if a version fails, the others will still be able to complete and cleanup
       matrix:
         version: ["3.7", "3.8", "3.9"]
     if: ${{ github.actor != 'dependabot[bot]' }}

parallel_run_e2e.py

@@ -8,7 +8,7 @@ def main():
     features = Path("tests/e2e").rglob("infrastructure.py")
     workers = len(list(features)) - 1
 
-    command = f"poetry run pytest -n {workers} --dist loadfile -o log_cli=true tests/e2e"
+    command = f"poetry run pytest -n {workers} -o log_cli=true tests/e2e"
     result = subprocess.run(command.split(), shell=False)
     sys.exit(result.returncode)
 

tests/e2e/event_handler/conftest.py

@@ -3,7 +3,7 @@
 from tests.e2e.event_handler.infrastructure import EventHandlerStack
 
 
-@pytest.fixture(autouse=True, scope="module")
+@pytest.fixture(autouse=True, scope="package")
 def infrastructure():
     """Setup and teardown logic for E2E test infrastructure
 

tests/e2e/event_handler/test_header_serializer.py

@@ -36,6 +36,7 @@ def lambda_function_url_endpoint(infrastructure: dict) -> str:
     return infrastructure.get("LambdaFunctionUrl", "")
 
 
+@pytest.mark.xdist_group(name="event_handler")
 def test_alb_headers_serializer(alb_basic_listener_endpoint):
     # GIVEN
     url = f"{alb_basic_listener_endpoint}/todos"
@@ -74,6 +75,7 @@ def test_alb_headers_serializer(alb_basic_listener_endpoint):
     assert response.cookies.get(last_cookie.name) == last_cookie.value
 
 
+@pytest.mark.xdist_group(name="event_handler")
 def test_alb_multi_value_headers_serializer(alb_multi_value_header_listener_endpoint):
     # GIVEN
     url = f"{alb_multi_value_header_listener_endpoint}/todos"
@@ -112,6 +114,7 @@ def test_alb_multi_value_headers_serializer(alb_multi_value_header_listener_endp
         assert response.cookies.get(cookie.name) == cookie.value
 
 
+@pytest.mark.xdist_group(name="event_handler")
 def test_api_gateway_rest_headers_serializer(apigw_rest_endpoint):
     # GIVEN
     url = f"{apigw_rest_endpoint}todos"
@@ -147,6 +150,7 @@ def test_api_gateway_rest_headers_serializer(apigw_rest_endpoint):
         assert response.cookies.get(cookie.name) == cookie.value
 
 
+@pytest.mark.xdist_group(name="event_handler")
 def test_api_gateway_http_headers_serializer(apigw_http_endpoint):
     # GIVEN
     url = f"{apigw_http_endpoint}todos"
@@ -182,6 +186,7 @@ def test_api_gateway_http_headers_serializer(apigw_http_endpoint):
         assert response.cookies.get(cookie.name) == cookie.value
 
 
+@pytest.mark.xdist_group(name="event_handler")
 def test_lambda_function_url_headers_serializer(lambda_function_url_endpoint):
     # GIVEN
     url = f"{lambda_function_url_endpoint}todos"  # the function url endpoint already has the trailing /

tests/e2e/event_handler/test_paths_ending_with_slash.py

@@ -33,6 +33,7 @@ def lambda_function_url_endpoint(infrastructure: dict) -> str:
     return infrastructure.get("LambdaFunctionUrl", "")
 
 
+@pytest.mark.xdist_group(name="event_handler")
 def test_api_gateway_rest_trailing_slash(apigw_rest_endpoint):
     # GIVEN API URL ends in a trailing slash
     url = f"{apigw_rest_endpoint}todos/"
@@ -51,6 +52,7 @@ def test_api_gateway_rest_trailing_slash(apigw_rest_endpoint):
     assert response.status_code == 200
 
 
+@pytest.mark.xdist_group(name="event_handler")
 def test_api_gateway_http_trailing_slash(apigw_http_endpoint):
     # GIVEN the URL for the API ends in a trailing slash API gateway should return a 404
     url = f"{apigw_http_endpoint}todos/"
@@ -67,6 +69,7 @@ def test_api_gateway_http_trailing_slash(apigw_http_endpoint):
         )
 
 
+@pytest.mark.xdist_group(name="event_handler")
 def test_lambda_function_url_trailing_slash(lambda_function_url_endpoint):
     # GIVEN the URL for the API ends in a trailing slash it should behave as if there was not one
     url = f"{lambda_function_url_endpoint}todos/"  # the function url endpoint already has the trailing /
@@ -83,6 +86,7 @@ def test_lambda_function_url_trailing_slash(lambda_function_url_endpoint):
         )
 
 
+@pytest.mark.xdist_group(name="event_handler")
 def test_alb_url_trailing_slash(alb_multi_value_header_listener_endpoint):
     # GIVEN url has a trailing slash - it should behave as if there was not one
     url = f"{alb_multi_value_header_listener_endpoint}/todos/"

tests/e2e/idempotency/conftest.py

@@ -3,8 +3,8 @@
 from tests.e2e.idempotency.infrastructure import IdempotencyDynamoDBStack
 
 
-@pytest.fixture(autouse=True, scope="module")
-def infrastructure(tmp_path_factory, worker_id):
+@pytest.fixture(autouse=True, scope="package")
+def infrastructure():
     """Setup and teardown logic for E2E test infrastructure
 
     Yields

tests/e2e/idempotency/test_idempotency_dynamodb.py

@@ -27,6 +27,7 @@ def idempotency_table_name(infrastructure: dict) -> str:
     return infrastructure.get("DynamoDBTable", "")
 
 
+@pytest.mark.xdist_group(name="idempotency")
 def test_ttl_caching_expiration_idempotency(ttl_cache_expiration_handler_fn_arn: str):
     # GIVEN
     payload = json.dumps({"message": "Lambda Powertools - TTL 5s"})
@@ -56,6 +57,7 @@ def test_ttl_caching_expiration_idempotency(ttl_cache_expiration_handler_fn_arn:
     assert third_execution_response != second_execution_response
 
 
+@pytest.mark.xdist_group(name="idempotency")
 def test_ttl_caching_timeout_idempotency(ttl_cache_timeout_handler_fn_arn: str):
     # GIVEN
     payload_timeout_execution = json.dumps({"sleep": 5, "message": "Lambda Powertools - TTL 1s"})
@@ -79,6 +81,7 @@ def test_ttl_caching_timeout_idempotency(ttl_cache_timeout_handler_fn_arn: str):
     assert payload_working_execution == execution_working_response
 
 
+@pytest.mark.xdist_group(name="idempotency")
 def test_parallel_execution_idempotency(parallel_execution_handler_fn_arn: str):
     # GIVEN
     arguments = json.dumps({"message": "Lambda Powertools - Parallel execution"})

tests/e2e/logger/conftest.py

@@ -3,8 +3,8 @@
 from tests.e2e.logger.infrastructure import LoggerStack
 
 
-@pytest.fixture(autouse=True, scope="module")
-def infrastructure(tmp_path_factory, worker_id):
+@pytest.fixture(autouse=True, scope="package")
+def infrastructure():
     """Setup and teardown logic for E2E test infrastructure
 
     Yields

tests/e2e/logger/test_logger.py

@@ -17,6 +17,7 @@ def basic_handler_fn_arn(infrastructure: dict) -> str:
     return infrastructure.get("BasicHandlerArn", "")
 
 
+@pytest.mark.xdist_group(name="logger")
 def test_basic_lambda_logs_visible(basic_handler_fn, basic_handler_fn_arn):
     # GIVEN
     message = "logs should be visible with default settings"

tests/e2e/metrics/conftest.py

@@ -3,8 +3,8 @@
 from tests.e2e.metrics.infrastructure import MetricsStack
 
 
-@pytest.fixture(autouse=True, scope="module")
-def infrastructure(tmp_path_factory, worker_id):
+@pytest.fixture(autouse=True, scope="package")
+def infrastructure():
     """Setup and teardown logic for E2E test infrastructure
 
     Yields