diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index e3a2b8c1aff9d3..f3d91ac2e78330 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -200,13 +200,159 @@ jobs: run: | python3 .github/workflows/scripts/calculate-backend-test-shards.py + split-tiers: + if: >- + always() && + !cancelled() && + needs.files-changed.outputs.backend == 'true' && + needs.select-tests.outputs.has-selected-tests != 'true' + needs: [files-changed, select-tests] + name: split tests into tiers + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + actions: read + outputs: + has-tiers: ${{ steps.split.outputs.has-tiers }} + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Download classification report + id: download + env: + GH_TOKEN: ${{ github.token }} + run: | + # Filter by conclusion=success via jq — gh CLI's --status flag is unreliable for this. + RUN_ID=$(gh run list --workflow=classify-services.yml --limit=10 --json databaseId,conclusion --jq '[.[] | select(.conclusion == "success")][0].databaseId') + if [ -z "$RUN_ID" ]; then + echo "No classify-services run found, skipping tiers" + echo "has-tiers=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + if ! gh run download "$RUN_ID" --name test-service-classification --dir /tmp/classification; then + echo "Classification artifact unavailable (may be expired), skipping tiers" + echo "has-tiers=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "has-tiers=true" >> "$GITHUB_OUTPUT" + + - name: Split tests by tier + id: split + if: steps.download.outputs.has-tiers == 'true' + run: | + python3 .github/workflows/scripts/split-tests-by-tier.py \ + --classification /tmp/classification/test-service-classification.json \ + --tier tier1 --output /tmp/backend-light-tests.txt + python3 .github/workflows/scripts/split-tests-by-tier.py \ + --classification /tmp/classification/test-service-classification.json \ + --tier tier2 --output /tmp/backend-tests.txt + echo "has-tiers=true" >> "$GITHUB_OUTPUT" + + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: steps.split.outputs.has-tiers == 'true' + with: + name: tier-file-lists + path: | + /tmp/backend-light-tests.txt + /tmp/backend-tests.txt + retention-days: 1 + + backend-light: + if: >- + always() && + !cancelled() && + needs.files-changed.outputs.backend == 'true' && + needs.split-tiers.outputs.has-tiers == 'true' + needs: [files-changed, split-tiers] + name: 'backend-light (${{ matrix.instance }})' + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + id-token: write + services: + redis-cluster: + image: ghcr.io/getsentry/docker-redis-cluster:7.0.10 + ports: + ['7000:7000', '7001:7001', '7002:7002', '7003:7003', '7004:7004', '7005:7005'] + env: + IP: 0.0.0.0 + zookeeper: + image: ghcr.io/getsentry/image-mirror-confluentinc-cp-zookeeper:6.2.0 + env: + ZOOKEEPER_CLIENT_PORT: 2181 + kafka: + image: ghcr.io/getsentry/image-mirror-confluentinc-cp-kafka:6.2.0 + env: + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: 1 + ports: ['9092:9092'] + strategy: + fail-fast: false + matrix: + instance: [0, 1, 2, 3, 4] + env: + MATRIX_INSTANCE_TOTAL: 5 + TEST_GROUP_STRATEGY: roundrobin + PYTHONHASHSEED: '0' + SENTRY_SKIP_SELENIUM_PLUGIN: '1' + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: tier-file-lists + path: /tmp/ + + - name: Setup sentry env + uses: ./.github/actions/setup-sentry + id: setup + with: + mode: migrations + + - name: Run backend-light tests + env: + SELECTED_TESTS_FILE: /tmp/backend-light-tests.txt + run: | + python3 -b -m pytest tests \ + --reuse-db \ + -n 4 \ + --dist=loadfile \ + --ignore tests/acceptance \ + --ignore tests/apidocs \ + --ignore tests/js \ + --ignore tests/tools \ + --json-report \ + --json-report-file=".artifacts/pytest.json" \ + --json-report-omit=log \ + --junit-xml=.artifacts/pytest.junit.xml \ + -o junit_suite_name=pytest-backend-light + + - name: Inspect failure + if: failure() + run: devservices logs 2>/dev/null || true + + - name: Collect test data + uses: ./.github/actions/collect-test-data + if: ${{ !cancelled() }} + with: + artifact_path: .artifacts/pytest.json + gcs_bucket: ${{ secrets.COLLECT_TEST_DATA_GCS_BUCKET }} + gcp_project_id: ${{ secrets.COLLECT_TEST_DATA_GCP_PROJECT_ID }} + workload_identity_provider: ${{ secrets.SENTRY_GCP_DEV_WORKLOAD_IDENTITY_POOL }} + service_account_email: ${{ secrets.COLLECT_TEST_DATA_SERVICE_ACCOUNT_EMAIL }} + matrix_instance_number: ${{ steps.setup.outputs.matrix-instance-number }} + backend-test: # Use always() so this job runs even when select-tests is skipped (master) if: >- always() && !cancelled() && needs.files-changed.outputs.backend == 'true' && needs.calculate-shards.outputs.shard-count != '0' - needs: [files-changed, select-tests, calculate-shards] + needs: [files-changed, select-tests, calculate-shards, split-tiers] name: backend test runs-on: ubuntu-24.04 timeout-minutes: 60 @@ -220,12 +366,11 @@ jobs: # and reducing the risk that one of many runs would turn red again (read: intermittent tests) fail-fast: false matrix: - # Dynamic matrix from calculate-shards - instance: ${{ fromJSON(needs.calculate-shards.outputs.shard-indices) }} + # With tiers: 17 shards for tier2; 5 more run as backend-light (22 total). + instance: ${{ needs.split-tiers.outputs.has-tiers == 'true' && fromJSON('[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]') || fromJSON(needs.calculate-shards.outputs.shard-indices) }} env: - # Dynamic total from calculate-shards - MATRIX_INSTANCE_TOTAL: ${{ needs.calculate-shards.outputs.shard-count }} + MATRIX_INSTANCE_TOTAL: ${{ needs.split-tiers.outputs.has-tiers == 'true' && '17' || needs.calculate-shards.outputs.shard-count }} TEST_GROUP_STRATEGY: roundrobin PYTHONHASHSEED: '0' XDIST_PER_WORKER_SNUBA: '1' @@ -258,12 +403,21 @@ jobs: name: selected-tests-${{ github.run_id }} path: .artifacts/ + - name: Download backend test list + if: needs.split-tiers.outputs.has-tiers == 'true' + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: tier-file-lists + path: /tmp/ + - name: Run backend test (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }}) env: - SELECTED_TESTS_FILE: ${{ needs.select-tests.outputs.has-selected-tests == 'true' && '.artifacts/selected-tests.txt' || '' }} + SELECTED_TESTS_FILE: ${{ needs.select-tests.outputs.has-selected-tests == 'true' && '.artifacts/selected-tests.txt' || (needs.split-tiers.outputs.has-tiers == 'true' && '/tmp/backend-tests.txt' || '') }} + # tier2 uses --dist=load; snuba-heavy tests have high per-test variance that load-balances better than loadfile. + XDIST_DIST: ${{ needs.split-tiers.outputs.has-tiers == 'true' && 'load' || 'loadfile' }} run: | if [ -n "${XDIST_WORKERS}" ]; then - export PYTEST_ADDOPTS="$PYTEST_ADDOPTS -n ${XDIST_WORKERS} --dist=loadfile" + export PYTEST_ADDOPTS="$PYTEST_ADDOPTS -n ${XDIST_WORKERS} --dist=${XDIST_DIST}" timeout 1200 make test-python-ci || { rc=$? if [ "$rc" -eq 124 ]; then @@ -620,6 +774,8 @@ jobs: [ api-docs, backend-test, + backend-light, + split-tiers, backend-migration-tests, calculate-shards, cli, diff --git a/.github/workflows/classify-services.yml b/.github/workflows/classify-services.yml new file mode 100644 index 00000000000000..b049962716b054 --- /dev/null +++ b/.github/workflows/classify-services.yml @@ -0,0 +1,90 @@ +name: classify test services + +on: + workflow_dispatch: + +jobs: + classify: + name: classify (${{ matrix.instance }}) + runs-on: ubuntu-24.04 + timeout-minutes: 60 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + instance: + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21] + + env: + MATRIX_INSTANCE_TOTAL: 22 + TEST_GROUP_STRATEGY: roundrobin + + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Setup sentry env + uses: ./.github/actions/setup-sentry + with: + mode: backend-ci + + - name: Run tests with classification + env: + PYTEST_ADDOPTS: '${{ env.PYTEST_ADDOPTS }} --classify-services --classification-output=test-service-classification.json' + run: | + python3 -b -m pytest tests \ + --ignore tests/acceptance \ + --ignore tests/apidocs \ + --ignore tests/js \ + --ignore tests/tools \ + || true + + - name: Upload classification report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: classification-shard-${{ matrix.instance }} + path: test-service-classification.json + retention-days: 90 + + merge-reports: + name: merge classification reports + needs: classify + if: always() + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: classification-shard-* + path: shards/ + + - name: Merge shard reports + run: | + python3 - <<'SCRIPT' + import json + from collections import defaultdict + from pathlib import Path + + merged = defaultdict(set) + shard_dirs = sorted(Path("shards").iterdir()) + + for shard_dir in shard_dirs: + report = shard_dir / "test-service-classification.json" + if report.exists(): + for tid, svcs in json.loads(report.read_text()).get("tests", {}).items(): + merged[tid].update(svcs) + + Path("test-service-classification.json").write_text(json.dumps( + {"total_tests": len(merged), "tests": {k: sorted(v) for k, v in sorted(merged.items())}}, + indent=2, + ) + "\n") + print(f"Merged {len(merged)} tests from {len(shard_dirs)} shards") + SCRIPT + + - name: Upload merged report + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: test-service-classification + path: test-service-classification.json + retention-days: 90 diff --git a/.github/workflows/scripts/split-tests-by-tier.py b/.github/workflows/scripts/split-tests-by-tier.py new file mode 100644 index 00000000000000..feb7e18e5c44aa --- /dev/null +++ b/.github/workflows/scripts/split-tests-by-tier.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import sys +from collections import defaultdict +from pathlib import Path + +TIER2_SERVICES = {"snuba", "kafka", "symbolicator", "objectstore", "bigtable"} + + +def split(classification: dict) -> dict[str, set[str]]: + file_services: dict[str, set[str]] = defaultdict(set) + for test_id, services in classification.get("tests", {}).items(): + file_services[test_id.split("::")[0]].update(services) + + tier1: set[str] = set() + tier2: set[str] = set() + for path, services in file_services.items(): + (tier2 if services & TIER2_SERVICES else tier1).add(path) + + return {"tier1": tier1, "tier2": tier2} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--classification", required=True) + parser.add_argument("--tier", choices=["tier1", "tier2"], required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + + with open(args.classification) as f: + classification = json.load(f) + + if not classification.get("tests"): + print("Error: classification JSON has no tests", file=sys.stderr) + return 1 + + tiers = split(classification) + if not tiers["tier1"] and not tiers["tier2"]: + print("Error: classification produced 0 files in both tiers", file=sys.stderr) + return 1 + + if not tiers["tier1"] or not tiers["tier2"]: + print( + f"Warning: tier1={len(tiers['tier1'])} tier2={len(tiers['tier2'])} — one tier is empty", + file=sys.stderr, + ) + + scopes = sorted(tiers[args.tier]) + Path(args.output).write_text("\n".join(scopes) + "\n") + print( + f"tier1={len(tiers['tier1'])} tier2={len(tiers['tier2'])} -> wrote {len(scopes)} to {args.output}", + file=sys.stderr, + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/sentry/testutils/pytest/__init__.py b/src/sentry/testutils/pytest/__init__.py index 231eee6a140712..9fa6342b5259d1 100644 --- a/src/sentry/testutils/pytest/__init__.py +++ b/src/sentry/testutils/pytest/__init__.py @@ -12,6 +12,7 @@ "sentry.testutils.pytest.json_report_reruns", "sentry.testutils.pytest.show_flaky_failures", "sentry.testutils.thread_leaks.pytest", + "sentry.testutils.pytest.service_classifier", ] if os.environ.get("SENTRY_SKIP_SELENIUM_PLUGIN") != "1": diff --git a/src/sentry/testutils/pytest/service_classifier.py b/src/sentry/testutils/pytest/service_classifier.py new file mode 100644 index 00000000000000..18c705e8935af1 --- /dev/null +++ b/src/sentry/testutils/pytest/service_classifier.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +import socket +import time +from collections import defaultdict +from pathlib import Path +from typing import Any + +import pytest + +from sentry.utils import json + +SERVICE_PORTS: dict[int, str] = { + 1218: "snuba", + 3021: "symbolicator", + 8086: "bigtable", + 8888: "objectstore", +} + +FIXTURE_SERVICE_MAP: dict[str, str] = { + "_requires_snuba": "snuba", + "_requires_kafka": "kafka", + "_requires_symbolicator": "symbolicator", + "_requires_objectstore": "objectstore", +} + +_original_send: Any = None +_original_sendall: Any = None +_current_test: str | None = None +_test_services: dict[str, set[str]] = defaultdict(set) +_enabled: bool = False + + +def _classify_socket(sock: socket.socket) -> None: + if not _current_test: + return + try: + service = SERVICE_PORTS.get(sock.getpeername()[1]) + if service: + _test_services[_current_test].add(service) + except (OSError, AttributeError, IndexError): + pass + + +def _patched_send(self: socket.socket, *args: Any, **kwargs: Any) -> Any: + _classify_socket(self) + return _original_send(self, *args, **kwargs) + + +def _patched_sendall(self: socket.socket, *args: Any, **kwargs: Any) -> Any: + _classify_socket(self) + return _original_sendall(self, *args, **kwargs) + + +def _install_socket_patches() -> None: + global _original_send, _original_sendall + _original_send = socket.socket.send + _original_sendall = socket.socket.sendall + socket.socket.send = _patched_send # type: ignore[assignment,method-assign] + socket.socket.sendall = _patched_sendall # type: ignore[assignment,method-assign] + + +def _uninstall_socket_patches() -> None: + if _original_send is not None: + socket.socket.send = _original_send # type: ignore[method-assign] + if _original_sendall is not None: + socket.socket.sendall = _original_sendall # type: ignore[method-assign] + + +def _detect_static_services(item: pytest.Item) -> set[str]: + services: set[str] = set() + + if getattr(item, "cls", None) is not None: + services.add("postgres") + elif hasattr(item, "fixturenames"): + if {"db", "transactional_db", "django_db_reset_sequences"} & set(item.fixturenames): + services.add("postgres") + + if hasattr(item, "fixturenames"): + for fixture, service in FIXTURE_SERVICE_MAP.items(): + if fixture in item.fixturenames: + services.add(service) + + for marker in item.iter_markers("usefixtures"): + for name in marker.args: + if name in FIXTURE_SERVICE_MAP: + services.add(FIXTURE_SERVICE_MAP[name]) + + return services + + +def pytest_addoption(parser: pytest.Parser) -> None: + group = parser.getgroup("service-classifier") + group.addoption("--classify-services", action="store_true", default=False) + group.addoption("--classification-output", default="test-service-classification.json") + + +def pytest_configure(config: pytest.Config) -> None: + global _enabled + _enabled = config.getoption("--classify-services", default=False) + if _enabled: + _install_socket_patches() + + +def pytest_unconfigure(config: pytest.Config) -> None: + if _enabled: + _uninstall_socket_patches() + + +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + if _enabled: + for item in items: + _test_services[item.nodeid].update(_detect_static_services(item)) + + +@pytest.hookimpl(tryfirst=True) +def pytest_runtest_setup(item: pytest.Item) -> None: + global _current_test + if _enabled: + _current_test = item.nodeid + + +@pytest.hookimpl(trylast=True) +def pytest_runtest_teardown(item: pytest.Item, nextitem: pytest.Item | None) -> None: + global _current_test + if _enabled: + _current_test = None + + +def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: + if _enabled: + report = { + "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "total_tests": len(_test_services), + "tests": {nid: sorted(svcs) for nid, svcs in sorted(_test_services.items())}, + } + Path(session.config.getoption("--classification-output")).write_text( + json.dumps(report) + "\n" + )