diff --git a/docs/src/advanced.md b/docs/src/advanced.md index 5051775..2ac2e93 100644 --- a/docs/src/advanced.md +++ b/docs/src/advanced.md @@ -127,6 +127,53 @@ end # hide ``` The `init_worker_code` is evaluated once per worker, so all definitions can be imported for use by the test module. +## Serial Tests + +Some tests cannot safely run in parallel with other tests — for example, tests that allocate very large arrays and would exhaust memory if multiple ran simultaneously. +The `serial` keyword argument to [`runtests`](@ref) lets you designate specific tests to run one at a time, while the remaining tests still run in parallel. + +```@example mypackage +using ParallelTestRunner +using MyPackage + +testsuite = Dict( + "big_alloc" => quote + # This test allocates ~4 GB and should not overlap with other tests + @test true + end, + "huge_matrix" => quote + @test true + end, + "fast_unit" => quote + @test 1 + 1 == 2 + end, + "fast_integration" => quote + @test true + end, +) + +# "big_alloc" and "huge_matrix" run one at a time; the rest run in parallel +runtests(MyPackage, ["--verbose"]; testsuite, serial=["big_alloc", "huge_matrix"]) +``` + +By default serial tests run **before** the parallel batch. +Use `serial_position=:after` to run them after instead: + +```@example mypackage +runtests(MyPackage, ["--verbose"]; testsuite, serial=["big_alloc", "huge_matrix"], serial_position=:after) +``` + +Serial tests participate in the same ordering logic as parallel tests (sorted by historical +duration, longest first) and their results appear in the same overall summary. + +!!! tip + With automatic test discovery via [`find_tests`](@ref), the `serial` names are the same + keys that appear in the testsuite dictionary (e.g. `"subdir/memory_test"`). + +!!! note + If the user filters tests via positional arguments (e.g. `julia test/runtests.jl unit`), + any serial test names that were filtered out are silently removed from the serial list. + ## Custom Workers For tests that require specific environment variables or Julia flags, you can use the `test_worker` keyword argument to [`runtests`](@ref) to assign tests to custom workers: @@ -254,3 +301,5 @@ function jltest { Having few long-running test files and other short-running ones hinders scalability. 1. **Use custom workers sparingly**: Custom workers add overhead. Only use them when tests genuinely require different configurations. + +1. **Use `serial` for resource-intensive tests**: If a test allocates significant memory or uses exclusive hardware resources, mark it as serial rather than reducing `--jobs` globally. This keeps the rest of your suite running in parallel. diff --git a/docs/src/api.md b/docs/src/api.md index 9890f75..c7ade69 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -39,12 +39,13 @@ addworkers default_njobs ``` -## Internal Types +## Internal Functionalities -These are internal types, not subject to semantic versioning contract (could be changed or removed at any point without notice), not intended for consumption by end-users. +These are internal types or functions, not subject to semantic versioning contract (could be changed or removed at any point without notice), not intended for consumption by end-users. They are documented here exclusively for `ParallelTestRunner` developers and contributors. ```@docs ParsedArgs WorkerTestSet +partition_tests ``` diff --git a/docs/src/index.md b/docs/src/index.md index 65349d6..5b83e30 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -107,6 +107,13 @@ Pkg.test("MyPackage"; test_args=`--verbose --jobs=4 integration`) Tests run concurrently in isolated worker processes, each inside own module. `ParallelTestRunner` records historical tests duration for each package, so that in subsequent runs long-running tests are executed first, to improve load balancing. +### Serial Test Support + +Certain tests (e.g. memory-hungry tests) may need to run one at a time. +The `serial` keyword argument to [`runtests`](@ref) lets you designate specific tests +for sequential execution, either before or after the parallel batch. +See [Serial Tests](@ref) in the advanced usage guide for details. + ### Real-time Progress The test runner provides real-time output showing: diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl index a049488..a6390ed 100644 --- a/src/ParallelTestRunner.jl +++ b/src/ParallelTestRunner.jl @@ -693,6 +693,24 @@ function filter_tests!(testsuite, args::ParsedArgs) return false end +""" + partition_tests(tests::Vector{String}, serial::Vector{String}) -> (serial_tests, parallel_tests) + +Split `tests` into two ordered vectors: tests named in `serial` (preserving their +order in `tests`) and the remaining parallel tests. Throws `ArgumentError` if any +name in `serial` is not present in `tests`. +""" +function partition_tests(tests::Vector{String}, serial::Vector{String}) + serial_set = Set(serial) + unknown = setdiff(serial_set, Set(tests)) + if !isempty(unknown) + throw(ArgumentError("serial test(s) not found in testsuite: $(join(sort!(collect(unknown)), ", "))")) + end + serial_tests = filter(t -> t in serial_set, tests) + parallel_tests = filter(t -> !(t in serial_set), tests) + return serial_tests, parallel_tests +end + """ runtests(mod::Module, args::Union{ParsedArgs,Array{String}}; testsuite::Dict{String,Expr}=find_tests(pwd()), @@ -701,7 +719,9 @@ end test_worker = Returns(nothing), stdout = Base.stdout, stderr = Base.stderr, - max_worker_rss = get_max_worker_rss()) + max_worker_rss = get_max_worker_rss(), + serial = String[], + serial_position::Symbol = :before) runtests(mod::Module, ARGS; ...) Run Julia tests in parallel across multiple worker processes. @@ -725,6 +745,10 @@ Several keyword arguments are also supported: When returning `nothing`, the test will be assigned to any available default worker. - `stdout` and `stderr`: I/O streams to write to (default: `Base.stdout` and `Base.stderr`) - `max_worker_rss`: RSS threshold where a worker will be restarted once it is reached. +- `serial`: A vector of test names (keys of `testsuite`) that should be run one at a time + instead of in parallel. An `ArgumentError` is thrown if any name is not found in the testsuite. +- `serial_position`: When to run serial tests relative to the parallel batch. + Must be `:before` (default) or `:after`. ## Command Line Options @@ -792,6 +816,14 @@ end runtests(MyPackage, args; testsuite) ``` +Run memory-hungry tests serially before the parallel batch +```julia +using ParallelTestRunner +using MyPackage + +runtests(MyPackage, ARGS; serial=["big_alloc_test", "huge_matrix"]) +``` + ## Memory Management Workers are automatically recycled when they exceed memory limits to prevent out-of-memory @@ -800,7 +832,8 @@ issues during long test runs. The memory limit is set based on system architectu function runtests(mod::Module, args::ParsedArgs; testsuite::Dict{String,Expr} = find_tests(pwd()), init_code = :(), init_worker_code = :(), test_worker = Returns(nothing), - stdout = Base.stdout, stderr = Base.stderr, max_worker_rss = get_max_worker_rss()) + stdout = Base.stdout, stderr = Base.stderr, max_worker_rss = get_max_worker_rss(), + serial::Vector{String} = String[], serial_position::Symbol = :before) # # set-up # @@ -814,25 +847,37 @@ function runtests(mod::Module, args::ParsedArgs; exit(0) end + # validate serial_position + serial_position in (:before, :after) || + throw(ArgumentError("serial_position must be :before or :after, got :$serial_position")) + # filter tests filter_tests!(testsuite, args) + # filter serial list to only include tests that survived filtering + serial = filter(t -> haskey(testsuite, t), serial) + # determine test order tests = collect(keys(testsuite)) Random.shuffle!(tests) historical_durations = load_test_history(mod) sort!(tests, by = x -> -get(historical_durations, x, Inf)) + # partition into serial and parallel groups + serial_tests, parallel_tests = partition_tests(tests, serial) + # determine parallelism jobs = something(args.jobs, default_njobs()) - jobs = clamp(jobs, 1, length(tests)) - println(stdout, "Running $(length(tests)) tests using $jobs parallel jobs. If this is too many concurrent jobs, specify the `--jobs=N` argument to the tests, or set the `JULIA_CPU_THREADS` environment variable.") - !isnothing(args.verbose) && println(stdout, "Available memory: $(Base.format_bytes(available_memory()))") - sem = Base.Semaphore(max(1, jobs)) + jobs = clamp(jobs, 1, max(1, length(parallel_tests))) worker_pool = Channel{Union{Nothing, PTRWorker}}(jobs) for _ in 1:jobs put!(worker_pool, nothing) end + println(stdout, "Running $(length(tests)) tests using $jobs parallel jobs. If this is too many concurrent jobs, specify the `--jobs=N` argument to the tests, or set the `JULIA_CPU_THREADS` environment variable.") + if !isempty(serial_tests) + println(stdout, " $(length(serial_tests)) serial test(s) will run $(serial_position) the parallel batch.") + end + !isnothing(args.verbose) && println(stdout, "Available memory: $(Base.format_bytes(available_memory()))") t0 = time() results = [] @@ -842,6 +887,16 @@ function runtests(mod::Module, args::ParsedArgs; worker_tasks = Task[] + serial_worker = Ref{Union{Nothing, PTRWorker}}(nothing) + + test_phases = if serial_position === :before + ((serial_tests, Base.Semaphore(1), serial_worker), + (parallel_tests, Base.Semaphore(max(1, jobs)), nothing)) + else + ((parallel_tests, Base.Semaphore(max(1, jobs)), nothing), + (serial_tests, Base.Semaphore(1), serial_worker)) + end + done = false function stop_work() if !done @@ -1028,104 +1083,120 @@ function runtests(mod::Module, args::ParsedArgs; tests_to_start = Threads.Atomic{Int}(length(tests)) try - @sync for test in tests - push!(worker_tasks, Threads.@spawn begin - local p = nothing - acquired = false - try - Base.acquire(sem) - acquired = true - p = take!(worker_pool) - Threads.atomic_sub!(tests_to_start, 1) - - done && return - - test_t0 = Base.@lock test_lock begin - test_t0 = time() - running_tests[test] = test_t0 - end - - # pass in init_worker_code to custom worker function if defined - wrkr = if init_worker_code == :() - test_worker(test) - else - test_worker(test, init_worker_code) - end - if wrkr === nothing - wrkr = p - end - # if a worker failed, spawn a new one - if wrkr === nothing || !Malt.isrunning(wrkr) - wrkr = p = addworker(; init_worker_code, io_ctx.color) - end - - # run the test - put!(printer_channel, (:started, test, worker_id(wrkr))) - result = try - Malt.remote_eval_wait(Main, wrkr.w, :(import ParallelTestRunner)) - Malt.remote_call_fetch(invokelatest, wrkr.w, runtest, - testsuite[test], test, init_code, test_t0) - catch ex - if isa(ex, InterruptException) - # the worker got interrupted, signal other tasks to stop - stop_work() - return - end - - ex - end - test_t1 = time() - output = Base.@lock wrkr.io_lock String(take!(wrkr.io)) - Base.@lock results_lock push!(results, (; test, result, output, test_t0, test_t1)) - - # act on the results - if result isa AbstractTestRecord - put!(printer_channel, (:finished, test, worker_id(wrkr), result)) - if anynonpass(result[]) && args.quickfail !== nothing - stop_work() - return - end - - if memory_usage(result) > max_worker_rss - # the worker has reached the max-rss limit, recycle it - # so future tests start with a smaller working set - Malt.stop(wrkr) - end - else - # One of Malt.TerminatedWorkerException, Malt.RemoteException, or ErrorException - @assert result isa Exception - put!(printer_channel, (:crashed, test, worker_id(wrkr))) - if args.quickfail !== nothing - stop_work() - return - end - - # the worker encountered some serious failure, recycle it - Malt.stop(wrkr) - end - - # get rid of the custom worker - if wrkr != p - Malt.stop(wrkr) - end - - Base.@lock test_lock begin - delete!(running_tests, test) - end - catch ex - isa(ex, InterruptException) || rethrow() - finally - if acquired - # stop the worker if no more tests will need one from the pool - if tests_to_start[] == 0 && p !== nothing && Malt.isrunning(p) - Malt.stop(p) - p = nothing - end - put!(worker_pool, p) - Base.release(sem) - end - end - end) + for (phase_tests, sem, shared_worker) in test_phases + isempty(phase_tests) && continue + # for serial phases, reserve one pool slot for the shared worker + if !isnothing(shared_worker) + shared_worker[] = take!(worker_pool) + end + @sync for test in phase_tests + push!(worker_tasks, Threads.@spawn begin + local p = nothing + acquired = false + try + Base.acquire(sem) + acquired = true + p = !isnothing(shared_worker) ? shared_worker[] : take!(worker_pool) + Threads.atomic_sub!(tests_to_start, 1) + + done && return + + test_t0 = Base.@lock test_lock begin + test_t0 = time() + running_tests[test] = test_t0 + end + + # pass in init_worker_code to custom worker function if defined + wrkr = if init_worker_code == :() + test_worker(test) + else + test_worker(test, init_worker_code) + end + if wrkr === nothing + wrkr = p + end + # if a worker failed, spawn a new one + if wrkr === nothing || !Malt.isrunning(wrkr) + wrkr = p = addworker(; init_worker_code, io_ctx.color) + end + + # run the test + put!(printer_channel, (:started, test, worker_id(wrkr))) + result = try + Malt.remote_eval_wait(Main, wrkr.w, :(import ParallelTestRunner)) + Malt.remote_call_fetch(invokelatest, wrkr.w, runtest, + testsuite[test], test, init_code, test_t0) + catch ex + if isa(ex, InterruptException) + # the worker got interrupted, signal other tasks to stop + stop_work() + return + end + + ex + end + test_t1 = time() + output = Base.@lock wrkr.io_lock String(take!(wrkr.io)) + Base.@lock results_lock push!(results, (; test, result, output, test_t0, test_t1)) + + # act on the results + if result isa AbstractTestRecord + put!(printer_channel, (:finished, test, worker_id(wrkr), result)) + if anynonpass(result[]) && args.quickfail !== nothing + stop_work() + return + end + + if memory_usage(result) > max_worker_rss + # the worker has reached the max-rss limit, recycle it + # so future tests start with a smaller working set + Malt.stop(wrkr) + end + else + # One of Malt.TerminatedWorkerException, Malt.RemoteException, or ErrorException + @assert result isa Exception + put!(printer_channel, (:crashed, test, worker_id(wrkr))) + if args.quickfail !== nothing + stop_work() + return + end + + # the worker encountered some serious failure, recycle it + Malt.stop(wrkr) + end + + # get rid of the custom worker + if wrkr != p + Malt.stop(wrkr) + end + + Base.@lock test_lock begin + delete!(running_tests, test) + end + catch ex + isa(ex, InterruptException) || rethrow() + finally + if acquired + if !isnothing(shared_worker) + shared_worker[] = p + else + # stop the worker if no more tests will need one from the pool + if tests_to_start[] == 0 && p !== nothing && Malt.isrunning(p) + Malt.stop(p) + p = nothing + end + put!(worker_pool, p) + end + Base.release(sem) + end + end + end) + end + # return the serial worker to the pool for potential reuse + if !isnothing(shared_worker) + put!(worker_pool, shared_worker[]) + shared_worker[] = nothing + end end catch err if !(err isa InterruptException) diff --git a/test/runtests.jl b/test/runtests.jl index 0a0efb6..93bfa61 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,6 +1,21 @@ using ParallelTestRunner using Test +# Helper macro to show output of tests in case they fail. Useful for debugging. +macro show_if_error(io, expr) + quote + try + @elapsed $(esc(expr)) + catch + output = String(take!($(esc(io)))) + printstyled(stderr, "Output of failed test >>>>>>>>>>>>>>>>>>>>\n", color=:red, bold=true) + println(stderr, output) + printstyled(stderr, "End of output <<<<<<<<<<<<<<<<<<<<<<<<<<<<\n", color=:red, bold=true) + rethrow() + end + end +end + cd(@__DIR__) include(joinpath(@__DIR__, "utils.jl")) @@ -475,17 +490,8 @@ end njobs = 2 io = IOBuffer() ioc = IOContext(io, :color => true) - try - runtests(ParallelTestRunner, ["--jobs=$(njobs)", "--verbose"]; - testsuite, stdout=ioc, stderr=ioc, init_code=:(include($(joinpath(@__DIR__, "utils.jl"))))) - catch - # Show output in case of failure, to help debugging. - output = String(take!(io)) - printstyled(stderr, "Output of failed test >>>>>>>>>>>>>>>>>>>>\n", color=:red, bold=true) - println(stderr, output) - printstyled(stderr, "End of output <<<<<<<<<<<<<<<<<<<<<<<<<<<<\n", color=:red, bold=true) - rethrow() - end + @show_if_error io runtests(ParallelTestRunner, ["--jobs=$(njobs)", "--verbose"]; + testsuite, stdout=ioc, stderr=ioc, init_code=:(include($(joinpath(@__DIR__, "utils.jl"))))) # Make sure we didn't spawn more workers than expected. @test ParallelTestRunner.ID_COUNTER[] == old_id_counter + njobs # Allow a moment for worker processes to exit @@ -706,7 +712,7 @@ end # ── Integration tests ──────────────────────────────────────────────────────── @testset "non-verbose mode" begin - testsuite = Dict("quiet" => quote @test true end) + testsuite = Dict("quiet" => :()) io = IOBuffer() runtests(ParallelTestRunner, String[]; testsuite, stdout=io, stderr=io) str = String(take!(io)) @@ -802,6 +808,228 @@ end @test contains(str, "SUCCESS") end +# ── Serial tests ───────────────────────────────────────────────────────────── + +@testset "serial tests" begin + @testset "partition_tests" begin + @testset "basic partitioning preserves order" begin + tests = ["a", "b", "c", "d", "e"] + serial, parallel = ParallelTestRunner.partition_tests(tests, ["c", "a"]) + @test serial == ["a", "c"] + @test parallel == ["b", "d", "e"] + end + + @testset "empty serial list" begin + tests = ["x", "y", "z"] + serial, parallel = ParallelTestRunner.partition_tests(tests, String[]) + @test isempty(serial) + @test parallel == tests + end + + @testset "all tests serial" begin + tests = ["a", "b"] + serial, parallel = ParallelTestRunner.partition_tests(tests, ["a", "b"]) + @test serial == ["a", "b"] + @test isempty(parallel) + end + + @testset "unknown serial name throws" begin + tests = ["a", "b"] + @test_throws ArgumentError ParallelTestRunner.partition_tests(tests, ["a", "missing"]) + end + end + + @testset "serial tests run before parallel (default)" begin + testsuite = Dict( + "serial_a" => :(), + "serial_b" => :(), + "parallel_1" => :(), + "parallel_2" => :(), + ) + io = IOBuffer() + jobs = 2 + old_id_counter = ParallelTestRunner.ID_COUNTER[] + runtests(ParallelTestRunner, ["--jobs=$(jobs)", "--verbose"]; + testsuite, stdout=io, stderr=io, + serial=["serial_a", "serial_b"]) + str = String(take!(io)) + @test contains(str, "2 serial test(s) will run before") + @test contains(str, "SUCCESS") + @test ParallelTestRunner.ID_COUNTER[] == old_id_counter + jobs + end + + @testset "serial tests run after parallel" begin + testsuite = Dict( + "serial_x" => quote + children = _count_child_pids($(getpid())) + # Make sure serial tests run alone. + if children >= 0 + @test children == 1 + end + end, + "parallel_y" => :(), + ) + io = IOBuffer() + ioc = IOContext(io, :color => true) + old_id_counter = ParallelTestRunner.ID_COUNTER[] + @show_if_error io runtests(ParallelTestRunner, ["--jobs=2", "--verbose"]; + testsuite, stdout=ioc, stderr=ioc, + init_code=:(include($(joinpath(@__DIR__, "utils.jl")))), + serial=["serial_x"], serial_position=:after) + str = String(take!(io)) + @test contains(str, "Running 2 tests using 1 parallel jobs") + @test contains(str, "1 serial test(s) will run after") + @test contains(str, "SUCCESS") + @test ParallelTestRunner.ID_COUNTER[] == old_id_counter + 1 + end + + @testset "serial_position validation" begin + testsuite = Dict("a" => :()) + io = IOBuffer() + @test_throws ArgumentError runtests(ParallelTestRunner, String[]; + testsuite, stdout=io, stderr=io, + serial_position=:middle) + end + + @testset "serial tests actually run sequentially" begin + serial_test_body = quote + sleep(1.0) + children = _count_child_pids($(getpid())) + # Make sure serial tests run alone. + if children >= 0 + @test children == 1 + end + end + + testsuite = Dict( + "s1" => serial_test_body, + "s2" => serial_test_body, + "s3" => serial_test_body, + "p1" => :(), + "p2" => :(), + ) + io = IOBuffer() + ioc = IOContext(io, :color => true) + old_id_counter = ParallelTestRunner.ID_COUNTER[] + jobs = 2 + elapsed = @elapsed begin + @show_if_error io runtests(ParallelTestRunner, ["--jobs=$(jobs)", "--verbose"]; + testsuite, stdout=ioc, stderr=ioc, + init_code=:(include($(joinpath(@__DIR__, "utils.jl")))), + serial=["s1", "s2", "s3"]) + end + str = String(take!(io)) + @test contains(str, "SUCCESS") + @test ParallelTestRunner.ID_COUNTER[] == old_id_counter + jobs + # Serial tests sleeping 1.0s each should take >= 3s total (sequential), + # not ~1.0s (parallel). + @test elapsed >= 3.0 + end + + @testset "all tests serial" begin + testsuite = Dict( + "a" => :(), + "b" => :(), + "c" => :(), + "d" => :(), + ) + io = IOBuffer() + old_id_counter = ParallelTestRunner.ID_COUNTER[] + runtests(ParallelTestRunner, ["--jobs=3", "--verbose"]; + testsuite, stdout=io, stderr=io, + serial=["a", "b", "c", "d"]) + str = String(take!(io)) + @test contains(str, "Running 4 tests using 1 parallel jobs") + @test contains(str, "4 serial test(s) will run before") + @test contains(str, "SUCCESS") + @test ParallelTestRunner.ID_COUNTER[] == old_id_counter + 1 + end + + @testset "empty serial list is a no-op" begin + testsuite = Dict( + "a" => :(), + "b" => :(), + ) + io = IOBuffer() + runtests(ParallelTestRunner, ["--jobs=2"]; testsuite, stdout=io, stderr=io, + serial=String[]) + str = String(take!(io)) + @test !contains(str, "serial") + @test contains(str, "SUCCESS") + end + + @testset "parallel tests less than requested jobs" begin + testsuite = Dict( + "s1" => :(), + "s2" => :(), + "p1" => :(), + "p2" => :(), + ); + io = IOBuffer() + old_id_counter = ParallelTestRunner.ID_COUNTER[] + runtests(ParallelTestRunner, ["--jobs=3"]; testsuite, stdout=io, stderr=io, + serial=["s1", "s2"]) + str = String(take!(io)) + # We have 4 total tests, requested 3 jobs, but only 2 tests are run in parallel, so + # 2 is the maximum parallelism we expect, and the number of new workers we spawn. + @test contains(str, "Running 4 tests using 2 parallel jobs") + @test contains(str, "2 serial test(s)") + @test contains(str, "SUCCESS") + @test ParallelTestRunner.ID_COUNTER[] == old_id_counter + 2 + end + + @testset "serial names filtered by positional args" begin + testsuite = Dict( + "unit/a" => :(), + "unit/b" => :(), + # This test file shouldn't called, we use `@test false` to make sure it's not. + "integration/c" => :(@test false), + ) + io = IOBuffer() + runtests(ParallelTestRunner, ["unit"]; testsuite, stdout=io, stderr=io, + serial=["unit/a", "integration/c"]) + str = String(take!(io)) + @test contains(str, "Running 2 tests") + @test contains(str, "1 serial test(s)") + @test contains(str, "SUCCESS") + end + + @testset "crashing serial test" begin + serial_test_body = quote + children = _count_child_pids($(getpid())) + # Make sure serial tests run alone. + if children >= 0 + @test children == 1 + end + end + + testsuite = Dict( + "s1" => serial_test_body, + "s2" => serial_test_body, + "s3" => serial_test_body, + "s4" => :(ccall(:abort, Nothing, ())), + "p1" => :(), + "p2" => :(), + ) + io = IOBuffer() + ioc = IOContext(io, :color => true) + old_id_counter = ParallelTestRunner.ID_COUNTER[] + jobs = 2 + @test_throws Test.FallbackTestSetException("Test run finished with errors") begin + runtests(ParallelTestRunner, ["--jobs=$(jobs)", "--verbose"]; + testsuite, stdout=ioc, stderr=ioc, + init_code=:(include($(joinpath(@__DIR__, "utils.jl")))), + serial=["s1", "s2", "s3", "s4"]) + end + str = String(take!(io)) + @test contains(str, "Running 6 tests using 2 parallel jobs") + @test contains(str, "4 serial test(s)") + @test contains(str, "FAILURE") + # We'll use jobs + 1 workers because one will crash. + @test ParallelTestRunner.ID_COUNTER[] == old_id_counter + jobs + 1 + end +end + # This testset should always be the last one, don't add anything after this. # We want to make sure there are no running workers at the end of the tests. @testset "no workers running" begin