From 10ed2a0027038e6ba589b2a8226cb681a7f30236 Mon Sep 17 00:00:00 2001 From: Dawid Weiss Date: Tue, 24 Feb 2026 12:41:47 +0100 Subject: [PATCH] Add FixSeed annotation. --- etc/junit4-missing-features.txt | 8 +- .../randomizedtesting/jupiter/FixSeed.java | 24 +++++ .../jupiter/RandomizedContext.java | 24 ++++- .../jupiter/F004_SeedFixing.java | 98 +++++++++++++++++++ .../jupiter/F004_SeedFixing.md | 80 +++++++++++++++ .../JupiterCallbackMethodOrder.java | 23 +++-- 6 files changed, 245 insertions(+), 12 deletions(-) create mode 100644 randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/FixSeed.java create mode 100644 randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/F004_SeedFixing.java create mode 100644 randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/F004_SeedFixing.md diff --git a/etc/junit4-missing-features.txt b/etc/junit4-missing-features.txt index 65f5d04..5e9c63f 100644 --- a/etc/junit4-missing-features.txt +++ b/etc/junit4-missing-features.txt @@ -1,6 +1,6 @@ [ai generated overview of junit4 features] -4. Shuffled test execution order and seed annotations +4. seed annotations - @Seed on a class fixes the main seed, making execution fully deterministic - @Seeds / @Seed on a method pins a per-method seed for regression coverage while still running once with a fresh random seed @@ -44,3 +44,9 @@ - Utility methods on RandomizedTest: randomInt(), randomIntBetween(), randomBoolean(), randomFloat(), etc. - Encourages testing over a broad input domain rather than fixed values + +[possibly doable with a custom test engine] + +- predictably shuffled test execution order +- blowing up test reps using tests.iters +- \ No newline at end of file diff --git a/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/FixSeed.java b/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/FixSeed.java new file mode 100644 index 0000000..0432c39 --- /dev/null +++ b/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/FixSeed.java @@ -0,0 +1,24 @@ +package com.carrotsearch.randomizedtesting.jupiter; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * This annotation should be placed on classes or methods that are {@link Randomized} and would like + * to use a constant seed (for reproducing a problem or other reasons). + * + *

Note that seed fixing is always possible by setting {@link + * com.carrotsearch.randomizedtesting.jupiter.RandomizedContextSupplier.SysProps#TESTS_SEED} system + * property, this is just convenience. + */ +@Target({ElementType.TYPE, ElementType.METHOD}) +@Documented +@Retention(RetentionPolicy.RUNTIME) +@ExtendWith(RandomizedContextSupplier.class) +public @interface FixSeed { + String value(); +} diff --git a/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/RandomizedContext.java b/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/RandomizedContext.java index e9c20ee..26095a8 100644 --- a/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/RandomizedContext.java +++ b/randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/RandomizedContext.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.Locale; import java.util.Objects; import java.util.Random; import org.junit.jupiter.api.extension.ExtensionContext; @@ -76,14 +77,33 @@ RandomizedContext deriveNew(ExtensionContext extensionContext) { } } - var firstAndRest = this.remainingSeedChain.pop(); + SeedChain seedChain; + var annotationSeed = extensionContext.getElement().map(e -> e.getAnnotation(FixSeed.class)); + if (annotationSeed.isPresent()) { + seedChain = SeedChain.parse(annotationSeed.get().value()); + for (var seed : seedChain.seeds()) { + if (seed.isUnspecified()) { + throw new RuntimeException( + String.format( + Locale.ROOT, + "@%s annotatoin must declare concrete seeds or seed chains on: %s", + FixSeed.class.getName(), + extensionContext.getElement().get())); + } + } + } else { + seedChain = this.remainingSeedChain; + } + + var firstAndRest = seedChain.pop(); var nextSeed = firstAndRest.first(); + var remainingChain = firstAndRest.rest(); if (nextSeed.isUnspecified()) { nextSeed = new Seed(this.seed.value() ^ Hashing.longHash(extensionContext.getUniqueId())); } return new RandomizedContext( - extensionContext.getUniqueId(), this, randomFactory, nextSeed, firstAndRest.rest()); + extensionContext.getUniqueId(), this, randomFactory, nextSeed, remainingChain); } @Override diff --git a/randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/F004_SeedFixing.java b/randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/F004_SeedFixing.java new file mode 100644 index 0000000..571e191 --- /dev/null +++ b/randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/F004_SeedFixing.java @@ -0,0 +1,98 @@ +package com.carrotsearch.randomizedtesting.jupiter; + +import static com.carrotsearch.randomizedtesting.jupiter.infra.TestInfra.*; +import static org.junit.platform.testkit.engine.EventConditions.*; + +import com.carrotsearch.randomizedtesting.jupiter.infra.IgnoreInStandaloneRuns; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.RepeatedTest; +import org.junit.jupiter.api.Test; + +/** Ensure there is a way to quickly "fix" the seed for the given method or class. */ +public class F004_SeedFixing { + @Nested + class TestSeedAnnotation { + @Test + public void testSeedFixing() { + collectExecutionResults( + testKitBuilder(T1.class) + .configurationParameter( + RandomizedContextSupplier.SysProps.TESTS_SEED.propertyKey, "dead:beef:cafe")) + .results() + .allEvents() + .assertThatEvents() + .doNotHave(event(finishedWithFailure())); + } + + @Randomized + static class T1 extends IgnoreInStandaloneRuns { + @Test + @FixSeed("babe") + void simpleTest(RandomizedContext ctx) { + Assertions.assertThat(ctx.getSeedChain().toString()).isEqualTo("[DEAD:BEEF:BABE]"); + } + } + + @Test + public void testClassSeedFixing() { + collectExecutionResults( + testKitBuilder(T2.class) + .configurationParameter( + RandomizedContextSupplier.SysProps.TESTS_SEED.propertyKey, "dead")) + .results() + .allEvents() + .assertThatEvents() + .doNotHave(event(finishedWithFailure())); + } + + @Randomized + @FixSeed("babe") + static class T2 extends IgnoreInStandaloneRuns { + @Test + void ta(RandomizedContext ctx) { + Assertions.assertThat(ctx.getSeedChain().toString()).startsWith("[DEAD:BABE:"); + } + + @Test + void tb(RandomizedContext ctx) { + Assertions.assertThat(ctx.getSeedChain().toString()).startsWith("[DEAD:BABE:"); + } + } + + @Randomized + @FixSeed("babe:caca") + static class T3 extends IgnoreInStandaloneRuns { + @Test + void ta(RandomizedContext ctx) { + Assertions.assertThat(ctx.getSeedChain().toString()).startsWith("[DEAD:BABE:CACA]"); + } + + @Test + void tb(RandomizedContext ctx) { + Assertions.assertThat(ctx.getSeedChain().toString()).startsWith("[DEAD:BABE:CACA]"); + } + } + + @Test + public void testRepeatedTests() { + collectExecutionResults( + testKitBuilder(T4.class) + .configurationParameter( + RandomizedContextSupplier.SysProps.TESTS_SEED.propertyKey, "dead")) + .results() + .allEvents() + .assertThatEvents() + .doNotHave(event(finishedWithFailure())); + } + + @Randomized + static class T4 extends IgnoreInStandaloneRuns { + @RepeatedTest(5) + @FixSeed("babe") + void simpleTest(RandomizedContext ctx) { + Assertions.assertThat(ctx.getSeedChain().toString()).endsWith(":BABE]"); + } + } + } +} diff --git a/randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/F004_SeedFixing.md b/randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/F004_SeedFixing.md new file mode 100644 index 0000000..7168cd8 --- /dev/null +++ b/randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/F004_SeedFixing.md @@ -0,0 +1,80 @@ +# Feature: seed fixing using `FixSeed` annotation + +## Functionality + +* It should be possible to "fix" (make constant, regardless of the + randomization state) the random seed for methods and classes. This can be achieved using `@FixSeed` annotation + +```java + +@Randomized +@FixSeed("cafebabe") +public class TestClass { + @Test + public void testMethod(Random ctx) { + } +} +``` + +it is also possible to fix the seed for a particular method, although this allows state randomization at +class level: + +```java + +@Randomized +public class TestClass { + @Test + @FixSeed("cafebabe") + public void testMethod(Random ctx) { + } +} +``` + +* The value of the `@FixSeed` annotation can be a single seed or a chain of seeds, affecting nested contexts. + +```java + +@Randomized +@FixSeed("cafebabe:deadbeef") +public class TestClass { + @Test + public void testMethod(Random ctx) { + } +} +``` + +* `@FixSeed` can be used to rerun the same tests multiple times with a constant seed or predictably varying seed. For + example, this test runs 5 times with the same seed/ randomness at the test level: + +```java + +@Randomized +public class TestClass { + @RepeatedTest(5) + @FixSeed("babe") + public void testMethod(Random ctx) { + } +} +``` + +but this test runs 5 times, each time with a different (but predictable, derived from the parent) seed: + +```java + +@Randomized +@FixSeed("babe") +public class TestClass { + @RepeatedTest(5) + public void testMethod(Random ctx) { + } +} +``` + +## Migration notes (from randomizedtesting for junit4) + +* `@FixSeed` is renamed from the `@Seed` annotation, used previously. + +* There is no way to provide multiple seeds (`@Seeds` annotation), there is no replacement for this functionality. + +* There are subtle differences in how the annotation propagates but overall think of the seed annotation placement +(method, class) as affecting the corresponding JUnit5 extension context (its path in the test's UniqueId). diff --git a/randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/experiments/JupiterCallbackMethodOrder.java b/randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/experiments/JupiterCallbackMethodOrder.java index 5df0d45..a2fc338 100644 --- a/randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/experiments/JupiterCallbackMethodOrder.java +++ b/randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/experiments/JupiterCallbackMethodOrder.java @@ -33,37 +33,41 @@ @ValueSource(strings = {"1", "2", "3", "4", "5", "6", "7", "8", "9", "10"}) public class JupiterCallbackMethodOrder { static { - System.out.println("Static constructor."); + System.out.println("tclass: static constructor."); } public JupiterCallbackMethodOrder(String param) { - System.out.println("constructor: " + param); + System.out.println("tclass: constructor: " + param); } @BeforeAll public static void beforeAll() { - System.out.println("Before all."); + System.out.println("tclass: beforeAll."); } @AfterAll public static void afterAll() { - System.out.println("After all."); + System.out.println("tclass: afterAll."); } @BeforeEach - public void before() {} + public void before() { + System.out.println("tclass: beforeEach."); + } @AfterEach - public void after() {} + public void after() { + System.out.println("tclass: afterEach."); + } @Test public void b() { - System.out.println("Test b."); + System.out.println("tclass: test b."); } @Test public void a() { - System.out.println("Test a."); + System.out.println("tclass: test a."); } public static class DebugExt @@ -80,7 +84,8 @@ public static class DebugExt InvocationInterceptor { private static void log(String callback, ExtensionContext context) { - System.out.println(callback + " | " + context.getUniqueId()); + System.out.println( + "ext: " + callback + "\n " + context.getUniqueId() + "\n " + context.getElement()); } @Override