From 45131b72ead47d74aeadf7efcce695821db9708e Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Mon, 11 May 2026 14:10:23 -0400 Subject: [PATCH 1/5] Run diagnose in CI --- .github/workflows/find-flaky-tests.yml | 43 ++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/find-flaky-tests.yml diff --git a/.github/workflows/find-flaky-tests.yml b/.github/workflows/find-flaky-tests.yml new file mode 100644 index 00000000000..c8ac737ef30 --- /dev/null +++ b/.github/workflows/find-flaky-tests.yml @@ -0,0 +1,43 @@ +name: Find Flaky Tests + +on: + schedule: + - cron: "0 0 * * *" + workflow_dispatch: + inputs: + dir: + description: The directory to find flaky tests in (core/services/...) + required: true + type: string + default: "core/..." + iterations: + description: The number of iterations to run the tests for + required: true + type: number + default: 10 + parallel_iterations: + description: The number of iterations to run in parallel + required: false + type: number + default: 1 + +jobs: + find-flaky-tests: + name: Find Flaky Tests + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + persist-credentials: false + repository: smartcontractkit/chainlink + ref: ${{ github.sha }} + - name: Run `diagnose` + run: | + go -C tools/test run . diagnose --iterations ${{ inputs.iterations }} --parallel-iterations ${{ inputs.parallel_iterations }} -- ${{ inputs.dir }} + - name: Upload Results + uses: actions/upload-artifact@v7 + with: + name: flaky-tests-results + path: ./diagnose-* + if-no-files-found: error From f8c15bb06b65dc6366df24b5011ae66b043e83fd Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Mon, 11 May 2026 14:12:21 -0400 Subject: [PATCH 2/5] Equal power --- .github/workflows/find-flaky-tests.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/find-flaky-tests.yml b/.github/workflows/find-flaky-tests.yml index c8ac737ef30..31b96378f9c 100644 --- a/.github/workflows/find-flaky-tests.yml +++ b/.github/workflows/find-flaky-tests.yml @@ -24,7 +24,8 @@ on: jobs: find-flaky-tests: name: Find Flaky Tests - runs-on: ubuntu-latest + runs-on: runs-on=${{ github.run_id + }}-diagnose/cpu=48/ram=96/family=c6id+c5ad/spot=false/image=ubuntu24-full-x64/extras=s3-cache steps: - name: Checkout uses: actions/checkout@v6 @@ -32,6 +33,10 @@ jobs: persist-credentials: false repository: smartcontractkit/chainlink ref: ${{ github.sha }} + - name: Enable S3 Cache for Self-Hosted Runners + uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2.1.0 + with: + metrics: cpu,network,memory,disk - name: Run `diagnose` run: | go -C tools/test run . diagnose --iterations ${{ inputs.iterations }} --parallel-iterations ${{ inputs.parallel_iterations }} -- ${{ inputs.dir }} From 52702d7b1918f8fcfef12fb76756b664237ca5ac Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Mon, 11 May 2026 14:24:04 -0400 Subject: [PATCH 3/5] env vars --- .github/workflows/find-flaky-tests.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/find-flaky-tests.yml b/.github/workflows/find-flaky-tests.yml index 31b96378f9c..ab9d1738d75 100644 --- a/.github/workflows/find-flaky-tests.yml +++ b/.github/workflows/find-flaky-tests.yml @@ -9,23 +9,32 @@ on: description: The directory to find flaky tests in (core/services/...) required: true type: string - default: "core/..." + default: "./core/..." iterations: description: The number of iterations to run the tests for required: true type: number - default: 10 + default: 20 # Should take 3.3 hours max parallel_iterations: description: The number of iterations to run in parallel required: false type: number default: 1 +permissions: + contents: read + +env: + ITERATIONS: ${{ inputs.iterations || 20 }} + PARALLEL_ITERATIONS: ${{ inputs.parallel_iterations || 1 }} + DIR: ${{ inputs.dir || './core/...' }} + jobs: find-flaky-tests: name: Find Flaky Tests runs-on: runs-on=${{ github.run_id }}-diagnose/cpu=48/ram=96/family=c6id+c5ad/spot=false/image=ubuntu24-full-x64/extras=s3-cache + timeout-minutes: 270 # 4.5 hours steps: - name: Checkout uses: actions/checkout@v6 @@ -39,7 +48,7 @@ jobs: metrics: cpu,network,memory,disk - name: Run `diagnose` run: | - go -C tools/test run . diagnose --iterations ${{ inputs.iterations }} --parallel-iterations ${{ inputs.parallel_iterations }} -- ${{ inputs.dir }} + go -C tools/test run . diagnose --iterations ${{ env.ITERATIONS }} --parallel-iterations ${{ env.PARALLEL_ITERATIONS }} -- ${{ env.DIR }} - name: Upload Results uses: actions/upload-artifact@v7 with: From 73e2f7d800b7cae1e4e7cf3f139916be05121ac1 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Mon, 11 May 2026 14:25:09 -0400 Subject: [PATCH 4/5] update ver --- .github/workflows/find-flaky-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/find-flaky-tests.yml b/.github/workflows/find-flaky-tests.yml index ab9d1738d75..1232898548e 100644 --- a/.github/workflows/find-flaky-tests.yml +++ b/.github/workflows/find-flaky-tests.yml @@ -43,7 +43,7 @@ jobs: repository: smartcontractkit/chainlink ref: ${{ github.sha }} - name: Enable S3 Cache for Self-Hosted Runners - uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2.1.0 + uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2 with: metrics: cpu,network,memory,disk - name: Run `diagnose` From 7b588bce63bc81d03d656e5f51ced0a7ca4c0a5e Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Mon, 11 May 2026 15:24:32 -0400 Subject: [PATCH 5/5] valid bash --- .github/workflows/find-flaky-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/find-flaky-tests.yml b/.github/workflows/find-flaky-tests.yml index 1232898548e..adf89cf504f 100644 --- a/.github/workflows/find-flaky-tests.yml +++ b/.github/workflows/find-flaky-tests.yml @@ -48,7 +48,7 @@ jobs: metrics: cpu,network,memory,disk - name: Run `diagnose` run: | - go -C tools/test run . diagnose --iterations ${{ env.ITERATIONS }} --parallel-iterations ${{ env.PARALLEL_ITERATIONS }} -- ${{ env.DIR }} + go -C tools/test run . diagnose --iterations "${ITERATIONS}" --parallel-iterations "${PARALLEL_ITERATIONS}" -- "${DIR}" - name: Upload Results uses: actions/upload-artifact@v7 with: