feat(ci): two-tier test split with service classification

.github/workflows/backend.yml

@@ -200,13 +200,159 @@ jobs:
         run: |
           python3 .github/workflows/scripts/calculate-backend-test-shards.py
 
+  split-tiers:
+    if: >-
+      always() &&
+      !cancelled() &&
+      needs.files-changed.outputs.backend == 'true' &&
+      needs.select-tests.outputs.has-selected-tests != 'true'
+    needs: [files-changed, select-tests]
+    name: split tests into tiers
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      actions: read
+    outputs:
+      has-tiers: ${{ steps.split.outputs.has-tiers }}
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Download classification report
+        id: download
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          # Filter by conclusion=success via jq — gh CLI's --status flag is unreliable for this.
+          RUN_ID=$(gh run list --workflow=classify-services.yml --limit=10 --json databaseId,conclusion --jq '[.[] | select(.conclusion == "success")][0].databaseId')
+          if [ -z "$RUN_ID" ]; then
+            echo "No classify-services run found, skipping tiers"
+            echo "has-tiers=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          if ! gh run download "$RUN_ID" --name test-service-classification --dir /tmp/classification; then
+            echo "Classification artifact unavailable (may be expired), skipping tiers"
+            echo "has-tiers=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          echo "has-tiers=true" >> "$GITHUB_OUTPUT"
+
+      - name: Split tests by tier
+        id: split
+        if: steps.download.outputs.has-tiers == 'true'
+        run: |
+          python3 .github/workflows/scripts/split-tests-by-tier.py \
+            --classification /tmp/classification/test-service-classification.json \
+            --tier tier1 --output /tmp/backend-light-tests.txt
+          python3 .github/workflows/scripts/split-tests-by-tier.py \
+            --classification /tmp/classification/test-service-classification.json \
+            --tier tier2 --output /tmp/backend-tests.txt
+          echo "has-tiers=true" >> "$GITHUB_OUTPUT"
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        if: steps.split.outputs.has-tiers == 'true'
+        with:
+          name: tier-file-lists
+          path: |
+            /tmp/backend-light-tests.txt
+            /tmp/backend-tests.txt
+          retention-days: 1
+
+  backend-light:
+    if: >-
+      always() &&
+      !cancelled() &&
+      needs.files-changed.outputs.backend == 'true' &&
+      needs.split-tiers.outputs.has-tiers == 'true'
+    needs: [files-changed, split-tiers]
+    name: 'backend-light (${{ matrix.instance }})'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      id-token: write
+    services:
+      redis-cluster:
+        image: ghcr.io/getsentry/docker-redis-cluster:7.0.10
+        ports:
+          ['7000:7000', '7001:7001', '7002:7002', '7003:7003', '7004:7004', '7005:7005']
+        env:
+          IP: 0.0.0.0
+      zookeeper:
+        image: ghcr.io/getsentry/image-mirror-confluentinc-cp-zookeeper:6.2.0
+        env:
+          ZOOKEEPER_CLIENT_PORT: 2181
+      kafka:
+        image: ghcr.io/getsentry/image-mirror-confluentinc-cp-kafka:6.2.0
+        env:
+          KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092
+          KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+          KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: 1
+        ports: ['9092:9092']
+    strategy:
+      fail-fast: false
+      matrix:
+        instance: [0, 1, 2, 3, 4]
+    env:
+      MATRIX_INSTANCE_TOTAL: 5
+      TEST_GROUP_STRATEGY: roundrobin
+      PYTHONHASHSEED: '0'
+      SENTRY_SKIP_SELENIUM_PLUGIN: '1'
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          name: tier-file-lists
+          path: /tmp/
+
+      - name: Setup sentry env
+        uses: ./.github/actions/setup-sentry
+        id: setup
+        with:
+          mode: migrations
+
+      - name: Run backend-light tests
+        env:
+          SELECTED_TESTS_FILE: /tmp/backend-light-tests.txt
+        run: |
+          python3 -b -m pytest tests \
+            --reuse-db \
+            -n 4 \
+            --dist=loadfile \
+            --ignore tests/acceptance \
+            --ignore tests/apidocs \
+            --ignore tests/js \
+            --ignore tests/tools \
+            --json-report \
+            --json-report-file=".artifacts/pytest.json" \
+            --json-report-omit=log \
+            --junit-xml=.artifacts/pytest.junit.xml \
+            -o junit_suite_name=pytest-backend-light
+
+      - name: Inspect failure
+        if: failure()
+        run: devservices logs 2>/dev/null || true
+
+      - name: Collect test data
+        uses: ./.github/actions/collect-test-data
+        if: ${{ !cancelled() }}
+        with:
+          artifact_path: .artifacts/pytest.json
+          gcs_bucket: ${{ secrets.COLLECT_TEST_DATA_GCS_BUCKET }}
+          gcp_project_id: ${{ secrets.COLLECT_TEST_DATA_GCP_PROJECT_ID }}
+          workload_identity_provider: ${{ secrets.SENTRY_GCP_DEV_WORKLOAD_IDENTITY_POOL }}
+          service_account_email: ${{ secrets.COLLECT_TEST_DATA_SERVICE_ACCOUNT_EMAIL }}
+          matrix_instance_number: ${{ steps.setup.outputs.matrix-instance-number }}
+
   backend-test:
     # Use always() so this job runs even when select-tests is skipped (master)
     if: >-
       always() &&
       !cancelled() &&
       needs.files-changed.outputs.backend == 'true' && needs.calculate-shards.outputs.shard-count != '0'
-    needs: [files-changed, select-tests, calculate-shards]
+    needs: [files-changed, select-tests, calculate-shards, split-tiers]
     name: backend test
     runs-on: ubuntu-24.04
     timeout-minutes: 60
@@ -220,12 +366,11 @@ jobs:
       # and reducing the risk that one of many runs would turn red again (read: intermittent tests)
       fail-fast: false
       matrix:
-        # Dynamic matrix from calculate-shards
-        instance: ${{ fromJSON(needs.calculate-shards.outputs.shard-indices) }}
+        # With tiers: 17 shards for tier2; 5 more run as backend-light (22 total).
+        instance: ${{ needs.split-tiers.outputs.has-tiers == 'true' && fromJSON('[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]') || fromJSON(needs.calculate-shards.outputs.shard-indices) }}
 
     env:
-      # Dynamic total from calculate-shards
-      MATRIX_INSTANCE_TOTAL: ${{ needs.calculate-shards.outputs.shard-count }}
+      MATRIX_INSTANCE_TOTAL: ${{ needs.split-tiers.outputs.has-tiers == 'true' && '17' || needs.calculate-shards.outputs.shard-count }}
       TEST_GROUP_STRATEGY: roundrobin
       PYTHONHASHSEED: '0'
       XDIST_PER_WORKER_SNUBA: '1'
@@ -258,12 +403,21 @@ jobs:
           name: selected-tests-${{ github.run_id }}
           path: .artifacts/
 
+      - name: Download backend test list
+        if: needs.split-tiers.outputs.has-tiers == 'true'
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          name: tier-file-lists
+          path: /tmp/
+
       - name: Run backend test (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }})
         env:
-          SELECTED_TESTS_FILE: ${{ needs.select-tests.outputs.has-selected-tests == 'true' && '.artifacts/selected-tests.txt' || '' }}
+          SELECTED_TESTS_FILE: ${{ needs.select-tests.outputs.has-selected-tests == 'true' && '.artifacts/selected-tests.txt' || (needs.split-tiers.outputs.has-tiers == 'true' && '/tmp/backend-tests.txt' || '') }}
+          # tier2 uses --dist=load; snuba-heavy tests have high per-test variance that load-balances better than loadfile.
+          XDIST_DIST: ${{ needs.split-tiers.outputs.has-tiers == 'true' && 'load' || 'loadfile' }}
         run: |
           if [ -n "${XDIST_WORKERS}" ]; then
-            export PYTEST_ADDOPTS="$PYTEST_ADDOPTS -n ${XDIST_WORKERS} --dist=loadfile"
+            export PYTEST_ADDOPTS="$PYTEST_ADDOPTS -n ${XDIST_WORKERS} --dist=${XDIST_DIST}"
             timeout 1200 make test-python-ci || {
               rc=$?
               if [ "$rc" -eq 124 ]; then
@@ -620,6 +774,8 @@ jobs:
       [
         api-docs,
         backend-test,
+        backend-light,
+        split-tiers,
         backend-migration-tests,
         calculate-shards,
         cli,

.github/workflows/classify-services.yml

@@ -0,0 +1,90 @@
+name: classify test services
+
+on:
+  workflow_dispatch:
+
+jobs:
+  classify:
+    name: classify (${{ matrix.instance }})
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        instance:
+          [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]
+
+    env:
+      MATRIX_INSTANCE_TOTAL: 22
+      TEST_GROUP_STRATEGY: roundrobin
+
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Setup sentry env
+        uses: ./.github/actions/setup-sentry
+        with:
+          mode: backend-ci
+
+      - name: Run tests with classification
+        env:
+          PYTEST_ADDOPTS: '${{ env.PYTEST_ADDOPTS }} --classify-services --classification-output=test-service-classification.json'
+        run: |
+          python3 -b -m pytest tests \
+            --ignore tests/acceptance \
+            --ignore tests/apidocs \
+            --ignore tests/js \
+            --ignore tests/tools \
+            || true
+
+      - name: Upload classification report
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: classification-shard-${{ matrix.instance }}
+          path: test-service-classification.json
+          retention-days: 90
+
+  merge-reports:
+    name: merge classification reports
+    needs: classify
+    if: always()
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    steps:
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          pattern: classification-shard-*
+          path: shards/
+
+      - name: Merge shard reports
+        run: |
+          python3 - <<'SCRIPT'
+          import json
+          from collections import defaultdict
+          from pathlib import Path
+
+          merged = defaultdict(set)
+          shard_dirs = sorted(Path("shards").iterdir())
+
+          for shard_dir in shard_dirs:
+              report = shard_dir / "test-service-classification.json"
+              if report.exists():
+                  for tid, svcs in json.loads(report.read_text()).get("tests", {}).items():
+                      merged[tid].update(svcs)
+
+          Path("test-service-classification.json").write_text(json.dumps(
+              {"total_tests": len(merged), "tests": {k: sorted(v) for k, v in sorted(merged.items())}},
+              indent=2,
+          ) + "\n")
+          print(f"Merged {len(merged)} tests from {len(shard_dirs)} shards")
+          SCRIPT
+
+      - name: Upload merged report
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: test-service-classification
+          path: test-service-classification.json
+          retention-days: 90

.github/workflows/scripts/split-tests-by-tier.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+TIER2_SERVICES = {"snuba", "kafka", "symbolicator", "objectstore", "bigtable"}
+
+
+def split(classification: dict) -> dict[str, set[str]]:
+    file_services: dict[str, set[str]] = defaultdict(set)
+    for test_id, services in classification.get("tests", {}).items():
+        file_services[test_id.split("::")[0]].update(services)
+
+    tier1: set[str] = set()
+    tier2: set[str] = set()
+    for path, services in file_services.items():
+        (tier2 if services & TIER2_SERVICES else tier1).add(path)
+
+    return {"tier1": tier1, "tier2": tier2}
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--classification", required=True)
+    parser.add_argument("--tier", choices=["tier1", "tier2"], required=True)
+    parser.add_argument("--output", required=True)
+    args = parser.parse_args()
+
+    with open(args.classification) as f:
+        classification = json.load(f)
+
+    if not classification.get("tests"):
+        print("Error: classification JSON has no tests", file=sys.stderr)
+        return 1
+
+    tiers = split(classification)
+    if not tiers["tier1"] and not tiers["tier2"]:
+        print("Error: classification produced 0 files in both tiers", file=sys.stderr)
+        return 1
+
+    if not tiers["tier1"] or not tiers["tier2"]:
+        print(
+            f"Warning: tier1={len(tiers['tier1'])} tier2={len(tiers['tier2'])} — one tier is empty",
+            file=sys.stderr,
+        )
+
+    scopes = sorted(tiers[args.tier])
+    Path(args.output).write_text("\n".join(scopes) + "\n")
+    print(
+        f"tier1={len(tiers['tier1'])} tier2={len(tiers['tier2'])} -> wrote {len(scopes)} to {args.output}",
+        file=sys.stderr,
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/sentry/testutils/pytest/__init__.py

@@ -12,6 +12,7 @@
     "sentry.testutils.pytest.json_report_reruns",
     "sentry.testutils.pytest.show_flaky_failures",
     "sentry.testutils.thread_leaks.pytest",
+    "sentry.testutils.pytest.service_classifier",
 ]
 
 if os.environ.get("SENTRY_SKIP_SELENIUM_PLUGIN") != "1":