diff --git a/etc/junit4-missing-features.txt b/etc/junit4-missing-features.txt index 9f6016d..98ab839 100644 --- a/etc/junit4-missing-features.txt +++ b/etc/junit4-missing-features.txt @@ -1,24 +1,5 @@ [ai generated overview of junit4 features] -8. Timeouts - - Standard JUnit @Test(timeout=N) is honoured - - @Timeout(millis=N) annotation provides an explicit alternative - - Termination sequence: Thread.interrupt() → Thread.stop() → zombie - detection; all attempts are logged with stack traces - -9. Thread-leak detection - - Threads that escape a test's ThreadGroup boundary are killed and cause - a test failure - - Encourages explicit Thread.join() before a test method returns - -10. Lingering threads and advanced thread-leak control - - @ThreadLeakLingering(linger=N) waits up to N ms for stray threads to - finish naturally (useful for Executor pools or other uncontrolled threads) - - Additional annotations for fine-grained policy: - @ThreadLeakScope – suite vs. test scope - @ThreadLeakAction – warn vs. fail - @ThreadLeakZombies – ignore vs. fail on zombie threads - 11. Nightly / scaled tests - @Nightly marks a test that only runs when nightly mode is active (-Dtests.nightly=true) @@ -38,4 +19,12 @@ - predictably shuffled test execution order - blowing up test reps using tests.iters -- \ No newline at end of file + +[to check/ add tests of] + +- is the seed stack trace frame injected for leaked threads + randomized testing ext? +- can we enforce the order of extensions (randomized testing > leaked threads) +- how are jupiter timeouts working together with leaked threads ext.? +- maybe bring back thread leak zombies annotation (if we can't cleanly terminate leaked threads, ignore all remaining tests). +- maybe move some of the implementation details to a non-exposed package? +- regenerate the javadocs with public API only. \ No newline at end of file diff --git a/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/DetectThreadLeaks.java b/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/DetectThreadLeaks.java new file mode 100644 index 0000000..b2cab4f --- /dev/null +++ b/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/DetectThreadLeaks.java @@ -0,0 +1,67 @@ +package com.carrotsearch.randomizedtesting.jupiter; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.util.function.Predicate; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * Detects threads started within the annotated test class that are still alive after the configured + * scope ends. + * + *

Only functional in sequential (same-thread) execution mode. Emits a warning and skips + * detection if tests run concurrently. + */ +@Target({ElementType.TYPE}) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@ExtendWith(DetectThreadLeaksExtension.class) +@Inherited +public @interface DetectThreadLeaks { + /** Scope at which thread leak detection is performed. */ + Scope scope() default Scope.SUITE; + + enum Scope { + /** Disable thread leak detection entirely. */ + NONE, + /** Check for leaked threads once after all tests in the class complete. */ + SUITE, + /** Check for leaked threads after each individual test method. */ + TEST + } + + /** + * Milliseconds to wait for leaked threads to self-terminate before declaring a failure. If all + * leaked threads terminate within this window, the test passes. Default is 0 (no lingering). + * + *

Place this annotation on the same class or method as {@link DetectThreadLeaks}. A + * method-level annotation takes precedence over a class-level one. + */ + @Target({ElementType.TYPE, ElementType.METHOD}) + @Retention(RetentionPolicy.RUNTIME) + @Documented + @Inherited + @interface LingerTime { + int millis(); + } + + /** + * Excludes threads matched by any of the given {@link Predicate} classes from leak detection. A + * thread is excluded when at least one predicate returns {@code true} for it. + * + *

Annotations are collected hierarchically from the class and its superclasses, and the + * filters from all levels are combined. + * + * @see SystemThreadFilter + */ + @Target({ElementType.TYPE}) + @Retention(RetentionPolicy.RUNTIME) + @Documented + @interface ExcludeThreads { + Class>[] value() default {SystemThreadFilter.class}; + } +} diff --git a/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/DetectThreadLeaksExtension.java b/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/DetectThreadLeaksExtension.java new file mode 100644 index 0000000..72627ab --- /dev/null +++ b/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/DetectThreadLeaksExtension.java @@ -0,0 +1,286 @@ +package com.carrotsearch.randomizedtesting.jupiter; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.parallel.ExecutionMode; + +/** JUnit Jupiter extension implementing {@link DetectThreadLeaks}. */ +public class DetectThreadLeaksExtension + implements BeforeAllCallback, AfterAllCallback, BeforeEachCallback, AfterEachCallback { + + private static final Logger LOGGER = Logger.getLogger(DetectThreadLeaksExtension.class.getName()); + private static final ExtensionContext.Namespace EXTENSION_NAMESPACE = + ExtensionContext.Namespace.create(DetectThreadLeaksExtension.class); + private static final String THREAD_SNAPSHOT_KEY = "snapshot"; + private static final String CONCURRENT_KEY = "concurrent"; + private static final String UNCAUGHT_EXCEPTION_HANDLER_KEY = "uncaught-exception-handler"; + + /** Total time budget to join interrupted threads before giving up. */ + private static final Duration INTERRUPT_JOIN_MS = Duration.ofSeconds(3); + + @Override + public void beforeAll(ExtensionContext context) { + if (scope(context) == DetectThreadLeaks.Scope.NONE) { + return; + } + + if (context.getExecutionMode() != ExecutionMode.SAME_THREAD) { + LOGGER.warning( + "Thread leak detection is disabled: tests in [" + + context.getDisplayName() + + "] run in concurrent execution mode."); + context.getStore(EXTENSION_NAMESPACE).put(CONCURRENT_KEY, Boolean.TRUE); + return; + } + + var store = context.getStore(EXTENSION_NAMESPACE); + var filter = buildFilter(context); + store.put(UNCAUGHT_EXCEPTION_HANDLER_KEY, installUncaughtExceptionHandler()); + store.put(THREAD_SNAPSHOT_KEY, liveThreads(filter)); + } + + @Override + public void beforeEach(ExtensionContext context) { + if (isConcurrentMode(context) || scope(context) != DetectThreadLeaks.Scope.TEST) { + return; + } + + var store = context.getStore(EXTENSION_NAMESPACE); + var filter = buildFilter(context); + store.put(THREAD_SNAPSHOT_KEY, liveThreads(filter)); + } + + @Override + public void afterEach(ExtensionContext context) { + if (isConcurrentMode(context) || scope(context) != DetectThreadLeaks.Scope.TEST) { + return; + } + + var store = context.getStore(EXTENSION_NAMESPACE); + var handler = store.get(UNCAUGHT_EXCEPTION_HANDLER_KEY, UncaughtExceptionsHandler.class); + try { + checkLeaks( + store, + "test [" + context.getDisplayName() + "]", + linger(context), + buildFilter(context), + handler); + } finally { + if (handler != null) handler.restore(); + } + } + + @Override + public void afterAll(ExtensionContext context) { + if (isConcurrentMode(context) || scope(context) == DetectThreadLeaks.Scope.NONE) { + return; + } + + var store = context.getStore(EXTENSION_NAMESPACE); + var handler = store.get(UNCAUGHT_EXCEPTION_HANDLER_KEY, UncaughtExceptionsHandler.class); + try { + checkLeaks( + store, + "suite [" + context.getDisplayName() + "]", + linger(context), + buildFilter(context), + handler); + } finally { + if (handler != null) handler.restore(); + } + } + + private static UncaughtExceptionsHandler installUncaughtExceptionHandler() { + var handler = new UncaughtExceptionsHandler(Thread.getDefaultUncaughtExceptionHandler()); + Thread.setDefaultUncaughtExceptionHandler(handler); + return handler; + } + + private static DetectThreadLeaks.Scope scope(ExtensionContext context) { + return context.getRequiredTestClass().getAnnotation(DetectThreadLeaks.class).scope(); + } + + private static int linger(ExtensionContext context) { + var methodAnn = + context + .getTestMethod() + .map(m -> m.getAnnotation(DetectThreadLeaks.LingerTime.class)) + .orElse(null); + if (methodAnn != null) return methodAnn.millis(); + + var classAnn = context.getRequiredTestClass().getAnnotation(DetectThreadLeaks.LingerTime.class); + return classAnn == null ? 0 : classAnn.millis(); + } + + @DetectThreadLeaks.ExcludeThreads() + private static class AnnotationDefaultsSource {} + + /** + * Collects {@link DetectThreadLeaks.ExcludeThreads} filter classes from the entire hierarchy + * (method to class to superclasses) and returns a combined predicate that excludes a thread when + * any filter matches it. + */ + private static Predicate buildFilter(ExtensionContext context) { + List excludeThreads = new ArrayList<>(); + + for (Class cls = context.getRequiredTestClass(); cls != null; cls = cls.getSuperclass()) { + var ann = cls.getAnnotation(DetectThreadLeaks.ExcludeThreads.class); + if (ann != null) { + excludeThreads.add(ann); + } + } + + if (excludeThreads.isEmpty()) { + excludeThreads.add( + AnnotationDefaultsSource.class.getAnnotation(DetectThreadLeaks.ExcludeThreads.class)); + } + + var filterClasses = new LinkedHashSet>(); + for (var ann : excludeThreads) { + for (var cls : ann.value()) { + try { + filterClasses.add(cls.getDeclaredConstructor().newInstance()); + } catch (ReflectiveOperationException e) { + throw new RuntimeException("Cannot instantiate thread filter: " + cls.getName(), e); + } + } + } + + if (filterClasses.isEmpty()) { + return t -> false; + } else { + return t -> filterClasses.stream().anyMatch(p -> p.test(t)); + } + } + + private static boolean isConcurrentMode(ExtensionContext context) { + return context + .getParent() + .map( + p -> + Boolean.TRUE.equals( + p.getStore(EXTENSION_NAMESPACE).get(CONCURRENT_KEY, Boolean.class))) + .orElse(false); + } + + private static void checkLeaks( + ExtensionContext.Store store, + String description, + int lingerMs, + Predicate filter, + UncaughtExceptionsHandler handler) { + var snapshot = store.get(THREAD_SNAPSHOT_KEY, HashSet.class); + AssertionError leakError = null; + + if (snapshot != null) { + var leaked = leakedSince(snapshot, filter); + + // Linger: poll until threads self-terminate or the window expires. + if (!leaked.isEmpty() && lingerMs > 0) { + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(lingerMs); + while (!leaked.isEmpty() && System.nanoTime() < deadline) { + try { + long remainingMs = TimeUnit.NANOSECONDS.toMillis(deadline - System.nanoTime()); + Thread.sleep(Math.max(1L, Math.min(100L, remainingMs))); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + leaked = leakedSince(snapshot, filter); + } + } + + if (!leaked.isEmpty()) { + // Suppress uncaught exception reporting during the interrupt/join phase to avoid + // capturing expected InterruptedException-related exceptions from cleaned-up threads. + if (handler != null) { + handler.stopReporting(); + } + + try { + // Send an interrupt to all threads. + leaked.keySet().forEach(Thread::interrupt); + + // Wait for all those threads. + long joinDeadline = System.nanoTime() + INTERRUPT_JOIN_MS.toNanos(); + for (Thread t : leaked.keySet()) { + long remaining = TimeUnit.NANOSECONDS.toMillis(joinDeadline - System.nanoTime()); + if (remaining <= 0) { + break; + } + + try { + t.join(remaining); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + } finally { + if (handler != null) { + handler.resumeReporting(); + } + } + + var sb = new StringBuilder(leaked.size() + " thread(s) leaked from " + description + ":"); + int cnt = 1; + for (var entry : leaked.entrySet()) { + sb.append(String.format("%n %2d) %s", cnt++, Threads.threadName(entry.getKey()))); + for (var ste : entry.getValue()) { + sb.append(String.format("%n at %s", ste)); + } + } + leakError = new AssertionError(sb.toString()); + } + } + + // Collect uncaught exceptions regardless of whether threads leaked. + List uncaught = + handler != null ? handler.getAndClear() : List.of(); + + if (leakError == null && uncaught.isEmpty()) return; + + // Combine: leak error first (if any), uncaught exceptions after; all but the first + // are attached as suppressed on the thrown error. + var errors = new ArrayList(); + if (leakError != null) errors.add(leakError); + for (var ue : uncaught) { + errors.add( + new AssertionError("Uncaught exception in thread [" + ue.threadName() + "]", ue.error())); + } + var first = errors.get(0); + errors.subList(1, errors.size()).forEach(first::addSuppressed); + throw first; + } + + private static Map leakedSince( + HashSet snapshot, Predicate filter) { + var current = liveThreadsWithStacks(filter); + current.keySet().removeAll(snapshot); + return current; + } + + private static HashSet liveThreads(Predicate filter) { + return new HashSet<>(liveThreadsWithStacks(filter).keySet()); + } + + private static Map liveThreadsWithStacks(Predicate filter) { + return Thread.getAllStackTraces().entrySet().stream() + .filter(e -> e.getKey().isAlive()) + .filter(e -> !filter.test(e.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } +} diff --git a/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/Seed.java b/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/Seed.java index 75b9bc1..bafefb0 100644 --- a/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/Seed.java +++ b/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/Seed.java @@ -1,5 +1,6 @@ package com.carrotsearch.randomizedtesting.jupiter; +/** A single randomization seed (typically part of a larger {@link SeedChain}). */ public record Seed(long value) { private static final char[] HEX = "0123456789ABCDEF".toCharArray(); static final Seed UNSPECIFIED = new Seed(0); diff --git a/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/SeedChain.java b/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/SeedChain.java index ea181d1..7946f64 100644 --- a/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/SeedChain.java +++ b/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/SeedChain.java @@ -2,11 +2,15 @@ import java.util.List; import java.util.Locale; -import java.util.Random; import java.util.stream.Collectors; import java.util.stream.Stream; -record SeedChain(List seeds) { +/** + * A seed chain determines randomization if {@link Randomized} extension is used. A seed chain is a + * sequence of {@link Seed}s, typically associated with one or more hierarchical junit jupiter + * contexts. + */ +public record SeedChain(List seeds) { private static final SeedChain EMPTY = new SeedChain(List.of()); static SeedChain parse(String chain) { @@ -37,7 +41,7 @@ public String toString() { record FirstAndRest(Seed first, SeedChain rest) {} - public FirstAndRest pop() { + FirstAndRest pop() { if (seeds.isEmpty()) { return new FirstAndRest(Seed.UNSPECIFIED, SeedChain.EMPTY); } @@ -46,8 +50,4 @@ public FirstAndRest pop() { var rest = new SeedChain(seeds.subList(1, seeds.size())); return new FirstAndRest(first, rest); } - - private long nextRandomValue() { - return new Random().nextLong(); - } } diff --git a/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/SystemThreadFilter.java b/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/SystemThreadFilter.java new file mode 100644 index 0000000..c0cdbeb --- /dev/null +++ b/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/SystemThreadFilter.java @@ -0,0 +1,38 @@ +package com.carrotsearch.randomizedtesting.jupiter; + +import java.util.function.Predicate; +import java.util.regex.Pattern; + +/** + * The default filter containing sane defaults excluding system and ignorable threads when {@link + * DetectThreadLeaks} extension is used. + * + * @see DetectThreadLeaks.ExcludeThreads + */ +public class SystemThreadFilter implements Predicate { + private static final Pattern KNOWN_SUBSTRINGS = + Pattern.compile("(^ForkJoinPool\\.)|(Poller SunPKCS11)"); + + @Override + public boolean test(Thread t) { + ThreadGroup tgroup = t.getThreadGroup(); + + // Ignore the entire system thread group. + if (tgroup != null && "system".equals(tgroup.getName()) && tgroup.getParent() == null) { + return true; + } + + // These are some of the "known" threads that should be ignored. + var tName = t.getName(); + return switch (tName) { + case "JFR request timer", + "YJPAgent-Telemetry", + "MemoryPoolMXBean notification dispatcher", + "AWT-AppKit", + "process reaper", + "JUnit5-serializer-daemon" -> + true; + default -> KNOWN_SUBSTRINGS.matcher(tName).find(); + }; + } +} diff --git a/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/Threads.java b/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/Threads.java index f7adb2f..c5355ac 100644 --- a/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/Threads.java +++ b/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/Threads.java @@ -1,10 +1,10 @@ package com.carrotsearch.randomizedtesting.jupiter; -public final class Threads { +final class Threads { Threads() {} /** Collect thread information, JVM vendor insensitive. */ - public static String threadName(Thread t) { + static String threadName(Thread t) { return "Thread[" + ("id=" + t.threadId()) + (", name=" + t.getName()) diff --git a/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/UncaughtExceptionsHandler.java b/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/UncaughtExceptionsHandler.java new file mode 100644 index 0000000..927b9ab --- /dev/null +++ b/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/UncaughtExceptionsHandler.java @@ -0,0 +1,57 @@ +package com.carrotsearch.randomizedtesting.jupiter; + +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** Collects uncaught exceptions from threads during test execution. */ +class UncaughtExceptionsHandler implements Thread.UncaughtExceptionHandler { + private static final Logger LOGGER = Logger.getLogger(UncaughtExceptionsHandler.class.getName()); + + record UncaughtException(String threadName, Throwable error) {} + + private final Thread.UncaughtExceptionHandler previous; + private final List exceptions = new ArrayList<>(); + private boolean reporting = true; + + UncaughtExceptionsHandler(Thread.UncaughtExceptionHandler previous) { + this.previous = previous; + } + + @Override + public void uncaughtException(Thread t, Throwable e) { + synchronized (exceptions) { + if (reporting) { + LOGGER.log(Level.SEVERE, "Uncaught exception in thread: " + Threads.threadName(t), e); + exceptions.add(new UncaughtException(Threads.threadName(t), e)); + } + } + if (previous != null) previous.uncaughtException(t, e); + } + + void stopReporting() { + synchronized (exceptions) { + reporting = false; + } + } + + void resumeReporting() { + synchronized (exceptions) { + reporting = true; + } + } + + List getAndClear() { + synchronized (exceptions) { + var copy = new ArrayList<>(exceptions); + exceptions.clear(); + return copy; + } + } + + /** Restores the previous default uncaught exception handler. */ + void restore() { + Thread.setDefaultUncaughtExceptionHandler(previous); + } +} diff --git a/randomizedtesting-jupiter/src/main/java/module-info.java b/randomizedtesting-jupiter/src/main/java/module-info.java index c833ea1..ab4c2ab 100644 --- a/randomizedtesting-jupiter/src/main/java/module-info.java +++ b/randomizedtesting-jupiter/src/main/java/module-info.java @@ -1,6 +1,7 @@ module com.carrotsearch.randomizedtesting { requires org.junit.jupiter.api; requires org.junit.jupiter.params; + requires java.logging; exports com.carrotsearch.randomizedtesting.jupiter; diff --git a/randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/F005_ThreadLeaks.java b/randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/F005_ThreadLeaks.java new file mode 100644 index 0000000..a5ffdc6 --- /dev/null +++ b/randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/F005_ThreadLeaks.java @@ -0,0 +1,555 @@ +package com.carrotsearch.randomizedtesting.jupiter; + +import static com.carrotsearch.randomizedtesting.jupiter.infra.TestInfra.*; +import static org.junit.platform.testkit.engine.EventConditions.*; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.*; + +import com.carrotsearch.randomizedtesting.jupiter.infra.IgnoreInStandaloneRuns; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; +import org.assertj.core.api.Condition; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; + +/** Verify that {@link DetectThreadLeaks} detects threads leaked from tests. */ +@Execution(ExecutionMode.SAME_THREAD) +public class F005_ThreadLeaks { + private static final List forkedThreads = new ArrayList<>(); + + @AfterEach + void interruptAndJoinForkedThreads() throws InterruptedException { + for (var t : forkedThreads) t.interrupt(); + for (var t : forkedThreads) t.join(); + forkedThreads.clear(); + } + + @Nested + class TestSuiteScope { + @TestFactory + Stream leakedThreadIsDetected() { + return Stream.of( + LeakInBeforeAll.class, + LeakInBeforeEach.class, + LeakInConstructor.class, + LeakInTestMethod.class, + LeakInAfterEach.class, + LeakInAfterAll.class) + .map( + clazz -> + DynamicTest.dynamicTest( + clazz.getSimpleName(), + () -> { + collectExecutionResults(testKitBuilder(clazz)) + .results() + .allEvents() + .finished() + .failed() + .assertEventsMatchExactly( + event(finishedWithFailure(instanceOf(AssertionError.class)))); + })); + } + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.SUITE) + static class LeakInBeforeAll extends IgnoreInStandaloneRuns { + @BeforeAll + static void beforeAll() { + startSleepingThread(); + } + + @Test + void testMethod() {} + } + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.SUITE) + static class LeakInBeforeEach extends IgnoreInStandaloneRuns { + @BeforeEach + void beforeEach() { + startSleepingThread(); + } + + @Test + void testMethod() {} + } + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.SUITE) + static class LeakInConstructor extends IgnoreInStandaloneRuns { + LeakInConstructor() { + startSleepingThread(); + } + + @Test + void testMethod() {} + } + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.SUITE) + static class LeakInTestMethod extends IgnoreInStandaloneRuns { + @Test + void testMethod() { + startSleepingThread(); + } + } + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.SUITE) + static class LeakInAfterEach extends IgnoreInStandaloneRuns { + @Test + void testMethod() {} + + @AfterEach + void afterEach() { + startSleepingThread(); + } + } + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.SUITE) + static class LeakInAfterAll extends IgnoreInStandaloneRuns { + @Test + void testMethod() {} + + @AfterAll + static void afterAll() { + startSleepingThread(); + } + } + } + + @Nested + class TestTestScope { + @TestFactory + Stream leakedThreadIsDetected() { + return Stream.of( + LeakInBeforeAll.class, + LeakInBeforeEach.class, + LeakInConstructor.class, + LeakInTestMethod.class, + LeakInAfterEach.class, + LeakInAfterAll.class) + .map( + clazz -> + DynamicTest.dynamicTest( + clazz.getSimpleName(), + () -> { + collectExecutionResults(testKitBuilder(clazz)) + .results() + .allEvents() + .finished() + .failed() + .assertEventsMatchExactly( + event(finishedWithFailure(instanceOf(AssertionError.class)))); + })); + } + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.TEST) + static class LeakInBeforeAll extends IgnoreInStandaloneRuns { + @BeforeAll + static void beforeAll() { + startSleepingThread(); + } + + @Test + void testMethod() {} + } + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.TEST) + static class LeakInBeforeEach extends IgnoreInStandaloneRuns { + @BeforeEach + void beforeEach() { + startSleepingThread(); + } + + @Test + void testMethod() {} + } + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.TEST) + static class LeakInConstructor extends IgnoreInStandaloneRuns { + LeakInConstructor() { + startSleepingThread(); + } + + @Test + void testMethod() {} + } + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.TEST) + static class LeakInTestMethod extends IgnoreInStandaloneRuns { + @Test + void testMethod() { + startSleepingThread(); + } + } + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.TEST) + static class LeakInAfterEach extends IgnoreInStandaloneRuns { + @Test + void testMethod() {} + + @AfterEach + void afterEach() { + startSleepingThread(); + } + } + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.TEST) + static class LeakInAfterAll extends IgnoreInStandaloneRuns { + @Test + void testMethod() {} + + @AfterAll + static void afterAll() { + startSleepingThread(); + } + } + } + + @Nested + class TestStackTracesInMessage { + @Test + void leakErrorMessageContainsStackTrace() { + collectExecutionResults(testKitBuilder(SuiteScopeWithLeak.class)) + .results() + .allEvents() + .finished() + .failed() + .assertEventsMatchExactly( + event( + finishedWithFailure( + instanceOf(AssertionError.class), + new Condition<>( + t -> t.getMessage().contains(getClass().getPackageName()), + "error message contains stack frames")))); + } + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.SUITE) + static class SuiteScopeWithLeak extends IgnoreInStandaloneRuns { + @Test + void testMethod() { + startSleepingThread(); + } + } + } + + @Nested + class TestLinger { + @Test + void threadTerminatingWithinLingerWindowPasses() { + collectExecutionResults(testKitBuilder(ShortLivedLeak.class)) + .results() + .allEvents() + .assertThatEvents() + .doNotHave(event(finishedWithFailure())); + } + + @Test + void methodLingerTakesPrecedenceOverAbsentClassLinger() { + collectExecutionResults(testKitBuilder(MethodLingerOverridesAbsentClassLinger.class)) + .results() + .allEvents() + .assertThatEvents() + .doNotHave(event(finishedWithFailure())); + } + + @Test + void methodLingerTakesPrecedenceOverClassLinger() { + collectExecutionResults(testKitBuilder(MethodLingerOverridesClassLinger.class)) + .results() + .allEvents() + .assertThatEvents() + .doNotHave(event(finishedWithFailure())); + } + + @Test + void threadOutlastingLingerWindowFails() { + collectExecutionResults(testKitBuilder(LongLivedLeak.class)) + .results() + .allEvents() + .finished() + .failed() + .assertEventsMatchExactly(event(finishedWithFailure(instanceOf(AssertionError.class)))); + } + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.SUITE) + @DetectThreadLeaks.LingerTime(millis = 10_000) + static class ShortLivedLeak extends IgnoreInStandaloneRuns { + @Test + void testMethod() { + startSleepingThread(Duration.ofMillis(100)); + } + } + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.SUITE) + @DetectThreadLeaks.LingerTime(millis = 50) + static class LongLivedLeak extends IgnoreInStandaloneRuns { + @Test + void testMethod() { + startSleepingThread(); + } + } + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.TEST) + static class MethodLingerOverridesAbsentClassLinger extends IgnoreInStandaloneRuns { + @Test + @DetectThreadLeaks.LingerTime(millis = 10_000) + void testMethod() { + startSleepingThread(Duration.ofMillis(100)); + } + } + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.TEST) + @DetectThreadLeaks.LingerTime(millis = 50) + static class MethodLingerOverridesClassLinger extends IgnoreInStandaloneRuns { + @Test + @DetectThreadLeaks.LingerTime(millis = 10_000) + void testMethod() { + startSleepingThread(Duration.ofMillis(100)); + } + } + } + + @Nested + class TestConcurrentMode { + @Test + void leakedThreadDoesNotFailInConcurrentMode() { + // In concurrent mode the extension is disabled: no AssertionErrors, even with a leak. + collectExecutionResults(testKitBuilder(ConcurrentWithLeak.class)) + .results() + .allEvents() + .assertThatEvents() + .doNotHave(event(finishedWithFailure())); + } + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.TEST) + @Execution(ExecutionMode.CONCURRENT) + static class ConcurrentWithLeak extends IgnoreInStandaloneRuns { + @Test + void testMethod() { + startSleepingThread(); + } + } + } + + @Nested + class TestExcludeThreads { + @Test + void excludedThreadDoesNotFail() { + collectExecutionResults(testKitBuilder(ExcludedByClassFilter.class)) + .results() + .allEvents() + .assertThatEvents() + .doNotHave(event(finishedWithFailure())); + } + + @Test + void nonExcludedThreadStillFails() { + collectExecutionResults(testKitBuilder(NonExcludedStillFails.class)) + .results() + .allEvents() + .finished() + .failed() + .assertEventsMatchExactly(event(finishedWithFailure(instanceOf(AssertionError.class)))); + } + + @Test + void methodAndClassFiltersStackHierarchically() { + collectExecutionResults(testKitBuilder(HierarchicalFilters.class)) + .results() + .allEvents() + .assertThatEvents() + .doNotHave(event(finishedWithFailure())); + } + + @Test + void forkJoinPoolStartup() { + collectExecutionResults(testKitBuilder(SysFjPool.class)) + .results() + .allEvents() + .assertThatEvents() + .doNotHave(event(finishedWithFailure())); + } + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.SUITE) + static class SysFjPool extends IgnoreInStandaloneRuns { + @Test + void testMethod() { + ForkJoinPool.commonPool().submit(() -> {}).join(); + } + } + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.SUITE) + @DetectThreadLeaks.ExcludeThreads(ExcludeNamedAFilter.class) + static class ExcludedByClassFilter extends IgnoreInStandaloneRuns { + @Test + void testMethod() { + startSleepingThread("excluded-a-1"); + } + } + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.SUITE) + @DetectThreadLeaks.ExcludeThreads(ExcludeNamedAFilter.class) + static class NonExcludedStillFails extends IgnoreInStandaloneRuns { + @Test + void testMethod() { + startSleepingThread(); + } + } + + @DetectThreadLeaks.ExcludeThreads(ExcludeNamedAFilter.class) + static class Superclass extends IgnoreInStandaloneRuns {} + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.TEST) + @DetectThreadLeaks.ExcludeThreads(ExcludeNamedBFilter.class) + static class HierarchicalFilters extends Superclass { + @Test + void testMethod() { + startSleepingThread("excluded-a-1"); + startSleepingThread("excluded-b-1"); + } + } + } + + /** Predicate that excludes threads whose names start with "excluded-a-". */ + public static class ExcludeNamedAFilter implements java.util.function.Predicate { + @Override + public boolean test(Thread t) { + return t.getName().startsWith("excluded-a-"); + } + } + + /** Predicate that excludes threads whose names start with "excluded-b-". */ + public static class ExcludeNamedBFilter implements java.util.function.Predicate { + @Override + public boolean test(Thread t) { + return t.getName().startsWith("excluded-b-"); + } + } + + @Nested + class TestUncaughtExceptions { + @Test + void uncaughtExceptionFailsTheTest() { + collectExecutionResults(testKitBuilder(UncaughtInTestMethod.class)) + .results() + .allEvents() + .finished() + .failed() + .assertEventsMatchExactly( + event( + finishedWithFailure( + instanceOf(AssertionError.class), + new Condition<>( + t -> + t.getCause() instanceof RuntimeException rc + && "uncaught-test-exception".equals(rc.getMessage()), + "cause is the original RuntimeException")))); + } + + @Test + void uncaughtExceptionsWithThreadLeaksAreNotReported() { + collectExecutionResults(testKitBuilder(UncaughtWithLeak.class)) + .results() + .allEvents() + .finished() + .failed() + .assertEventsMatchExactly( + event( + finishedWithFailure( + instanceOf(AssertionError.class), + new Condition<>(t -> t.getCause() == null, "cause is empty.")))); + } + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.SUITE) + static class UncaughtInTestMethod extends IgnoreInStandaloneRuns { + @Test + void testMethod() throws InterruptedException { + startThread( + "bg-thread", + () -> { + throw new RuntimeException("uncaught-test-exception"); + }) + .join(); + } + } + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.TEST) + static class UncaughtWithLeak extends IgnoreInStandaloneRuns { + @Test + void testMethod() { + startThread( + "bg-thread", + () -> { + try { + Thread.sleep(TimeUnit.MINUTES.toMillis(1)); + } catch (InterruptedException ignored) { + throw new RuntimeException("uncaught-test-exception"); + } + }); + } + } + } + + @Nested + class TestNoneScope { + @Test + void leakedThreadDoesNotFailWithNoneScope() { + collectExecutionResults(testKitBuilder(NoneScope.class)) + .results() + .allEvents() + .assertThatEvents() + .doNotHave(event(finishedWithFailure())); + } + + @DetectThreadLeaks(scope = DetectThreadLeaks.Scope.NONE) + static class NoneScope extends IgnoreInStandaloneRuns { + @Test + void testMethod() { + startSleepingThread(); + } + } + } + + /** Starts a thread that sleeps long enough to be observable as a leak. */ + private static void startSleepingThread() { + startSleepingThread(Duration.ofMinutes(1)); + } + + /** Starts a named thread that sleeps long enough to be observable as a leak. */ + private static void startSleepingThread(String name) { + startThread( + name, + () -> { + try { + Thread.sleep(Duration.ofMinutes(1)); + } catch (InterruptedException ignored) { + } + }); + } + + private static void startSleepingThread(Duration duration) { + startThread( + "sleeping-thread", + () -> { + try { + Thread.sleep(duration.toMillis()); + } catch (InterruptedException ignored) { + } + }); + } + + private static Thread startThread(String name, Runnable r) { + var t = new Thread(r, name); + forkedThreads.add(t); + t.start(); + return t; + } +} diff --git a/randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/F005_ThreadLeaks.md b/randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/F005_ThreadLeaks.md new file mode 100644 index 0000000..df6ba8e --- /dev/null +++ b/randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/F005_ThreadLeaks.md @@ -0,0 +1,106 @@ +# Feature: detecting tests that "leak" threads + +## Functionality + +* It should be possible to add a `@DetectThreadLeaks` extension which detects new threads forked within the test + container. This extension takes a parameter - the scope of detection. Either we care about threads leaked + from the entire container or from each individual test. Here is an example of use: + +```java + +@DetectThreadLeaks(scope = DetectThreadLeaks.Scope.SUITE) +public class TestClass { + @Test + public void testMethod() { + new Thread(() -> { + try { + Thread.sleep(1000); + } catch (Exception e) { + } + }).start(); + } +} +``` + +* The extension is **only functional in sequential mode**. It should emit a warning and do nothing if tests are + run in concurrent mode. + +* Occasionally there will be threads that cannot be joined but will eventually terminate. One can specify an additional + "linger" time before the thread leak is reported, for example one second, below: + +```java + +@DetectThreadLeaks(scope = DetectThreadLeaks.Scope.SUITE) +@DetectThreadLeaks.LingerTime(millis = 1_000) +public class TestClass { + @Test + public void testMethod() { + new Thread(() -> { + try { + Thread.sleep(1000); + } catch (Exception e) { + } + }).start(); + } +} +``` + +* In certain cases, system threads or other threads beyond the test's control may be started and cannot be terminated + within the test's scope. The `@DetectThreadLeaks.ExcludeThreads` annotation can provide programmatic filters which + tell the extension to ignore certain threads. + +* When `@DetectThreadLeaks` is active, it also detects **uncaught exceptions** thrown by any thread + during the scope. Such exceptions are collected and reported as test failures. If both a thread leak and uncaught + exceptions occur, all are reported: the leak error is thrown and the + uncaught-exception errors are attached as suppressed exceptions. + +```java + +@DetectThreadLeaks(scope = DetectThreadLeaks.Scope.TEST) +public class TestClass { + @Test + public void testMethod() throws InterruptedException { + Thread t = new Thread(() -> { + throw new RuntimeException("background failure"); + }); + t.start(); + t.join(); + } +} +``` + +## Migration notes (from randomizedtesting for junit4) + +* `@ThreadLeakScope` is replaced with `@DetectThreadLeaks(scope = ...)`. + The default scope changed: the old default was `TEST`; the new default is `SUITE`. + +* `@ThreadLeakLingering(linger = N)` is replaced with `@DetectThreadLeaks.LingerTime(millis = N)`. + The annotation can now be placed on individual test methods as well as on the class, with the + method-level value taking precedence. + +* `@ThreadLeakFilters(filters = {MyFilter.class})`: replace with + `@DetectThreadLeaks.ExcludeThreads(MyFilter.class)`. The filter interface changed from + `ThreadFilter.reject(Thread)` (return `true` to exclude) to `Predicate.test(Thread)` + (return `true` to exclude). Rename and invert the logic accordingly. The `defaultFilters` + flag has no equivalent; built-in system-thread filters are always applied. Filters are now + collected hierarchically from the method, the class, and all superclasses, and combined with OR. + +* `@ThreadLeakAction`: interrupt-on-leak is now always performed as cleanup (no annotation + needed). Leaked threads are interrupted and joined before the failure is reported. There is no + equivalent of the `WARN`-only mode. + +* `@ThreadLeakZombies`: zombie tracking (marking remaining tests as aborted when a leaked + thread could not be killed) is not implemented. Threads that survive the interrupt/join budget + are still reported in the failure message but subsequent tests are not skipped. + +* `@ThreadLeakGroup` — the group scope (`ALL` / `MAIN` / `TESTGROUP`) is not configurable. + The extension always uses `Thread.getAllStackTraces()`, equivalent to the old `ALL` group, and + filters out known system threads automatically. + +* There are subtle differences in the implementation concerning how threads are interrupted (and + how retry attempts are implemented). In particular, the number of retries and their period is not + user-controlled (`tests.killattempts`, `tests.killwait` properties in `RandomizedRunner`). + +* The default value of `@DetectThreadLeaks.ExcludeThreads` points at `SystemThreadFilter`. There is + no `defaultFilters` parameter; just use the `SystemThreadFilter` in any custom declarations filter + declarations. \ No newline at end of file diff --git a/randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/F999_RemovedFeatures.md b/randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/F999_RemovedFeatures.md index 78d78dc..cd1074e 100644 --- a/randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/F999_RemovedFeatures.md +++ b/randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/F999_RemovedFeatures.md @@ -1,30 +1,30 @@ # Migration notes: features removed or replaced by junit5 functionality -The following features of junit4-compatible `RandomizedRunner` have been +The following features of junit4-compatible `RandomizedRunner` have been removed or replaced by built-in functionality in junit5. ## Parameterized tests (`@ParametersFactory`) * This feature is already part of JUnit5: use `@ParameterizedClass`, `@ParameterizedTest` -or any other way to create dynamic tests that JUnit5 offers. + or any other way to create dynamic tests that JUnit5 offers. ## Test groups and test filtering * This feature is already part of JUnit5 in the -form of[`@Tag` annotations](https://docs.junit.org/6.0.3/writing-tests/tagging-and-filtering.html). + form of[`@Tag` annotations](https://docs.junit.org/6.0.3/writing-tests/tagging-and-filtering.html). * Direct use of `tests.filter` and `tests.[group-name]` properties -is not as straightforward as it was with `RandomizedRunner`. These properties -need to be converted and passed to JUnit Jupiter using your build system's -facilities: system properties alone won't have any effect. Here is -an [example for gradle](https://docs.gradle.org/current/userguide/java_testing.html#test_grouping). - + is not as straightforward as it was with `RandomizedRunner`. These properties + need to be converted and passed to JUnit Jupiter using your build system's + facilities: system properties alone won't have any effect. Here is + an [example for gradle](https://docs.gradle.org/current/userguide/java_testing.html#test_grouping). + ## Custom test case order (`@TestCaseOrdering`) * This feature is already part of JUnit5 (`@TestMethodOrder`). However, -It is technically not possible to write a reorderer that will use the root -seed value to reorder methods consistently (because method orderers -are run in discovery phase and execution contexts are not available then). + It is technically not possible to write a reorderer that will use the root + seed value to reorder methods consistently (because method orderers + are run in discovery phase and execution contexts are not available then). ## Test instance creation control (`@TestCaseInstanceProvider`). @@ -37,26 +37,34 @@ are run in discovery phase and execution contexts are not available then). ## Custom test-method providers (`@TestMethodProviders()`) * JUnit5/ Jupiter offers many ways to discover tests dynamically. `@TestFactory` -methods, templates, etc. While there is no one-to-one replacement, it should -be possible to reimplement custom test providers with some minor refactorings. -For example, if a `TestMethodProvider` was including all `test*` methods (JUnit3-style), -you could write a `@TestFactory` in a common superclass and then just extend it. This -test factory would something like this: + methods, templates, etc. While there is no one-to-one replacement, it should + be possible to reimplement custom test providers with some minor refactorings. + For example, if a `TestMethodProvider` was including all `test*` methods (JUnit3-style), + you could write a `@TestFactory` in a common superclass and then just extend it. This + test factory would something like this: + ```java + @TestFactory Stream includeTestMethodsWithNoAnnotations() { return Arrays.stream(getClass().getDeclaredMethods()) - .filter(m -> m.getName().startsWith("test") - && m.getParameterCount() == 0 - && !Modifier.isStatic(m.getModifiers())) - .map(m -> DynamicTest.dynamicTest(m.getName(), () -> { - m.invoke(this); - })); + .filter(m -> m.getName().startsWith("test") + && m.getParameterCount() == 0 + && !Modifier.isStatic(m.getModifiers())) + .map(m -> DynamicTest.dynamicTest(m.getName(), () -> { + m.invoke(this); + })); } ``` ## Repeating tests with @Repeat -* Use standard JUnit5 test repetition facilities (like test templates, or `@RepeatedTest` -annotation). To repeat the same test with a constant seed, `@FixSeed` at class or test level. -If the seed is not fixed, it will be different for each test repetition by default. +* Use standard JUnit5 test repetition facilities (like test templates, or `@RepeatedTest` + annotation). To repeat the same test with a constant seed, `@FixSeed` at class or test level. + If the seed is not fixed, it will be different for each test repetition by default. + +## Multiple extensions instead of a single `RandomizedRunner` + +* The `RandomizedRunner` combined multiple features in one class. This has been replaced by multiple + extensions, which can be used independently. So `@Randomized` and `@DetectThreadLeaks` can be used + completely independently of each other, for example.