Add support for deterministic testing with JUnit (#1146)

Author: seongahjo

fixture-monkey-api/src/main/java/com/navercorp/fixturemonkey/api/engine/EngineUtils.java

@@ -0,0 +1,60 @@
+/*
+ * Fixture Monkey
+ *
+ * Copyright (c) 2021-present NAVER Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.navercorp.fixturemonkey.api.engine;
+
+import org.apiguardian.api.API;
+import org.apiguardian.api.API.Status;
+
+/**
+ * The {@link EngineUtils} class provides utility methods for engine.
+ * Engine is a library that provides a way to generate random values.
+ */
+@API(since = "1.1.9", status = Status.EXPERIMENTAL)
+public abstract class EngineUtils {
+	private static final boolean USE_JQWIK_ENGINE;
+	private static final boolean USE_KOTEST_ENGINE;
+
+	static {
+		boolean useJqwikEngine;
+		boolean useKotestEngine;
+		try {
+			Class.forName("net.jqwik.engine.SourceOfRandomness");
+			useJqwikEngine = true;
+		} catch (ClassNotFoundException e) {
+			useJqwikEngine = false;
+		}
+		USE_JQWIK_ENGINE = useJqwikEngine;
+
+		try {
+			Class.forName("io.kotest.property.Arb");
+			useKotestEngine = true;
+		} catch (ClassNotFoundException e) {
+			useKotestEngine = false;
+		}
+		USE_KOTEST_ENGINE = useKotestEngine;
+	}
+
+	public static boolean useJqwikEngine() {
+		return USE_JQWIK_ENGINE;
+	}
+
+	public static boolean useKotestEngine() {
+		return USE_KOTEST_ENGINE;
+	}
+}

fixture-monkey-api/src/main/java/com/navercorp/fixturemonkey/api/random/Randoms.java

@@ -27,27 +27,20 @@
 
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 
+import com.navercorp.fixturemonkey.api.engine.EngineUtils;
+
 /**
  * Reference jqwik SourceOfRandomness
  */
 @API(since = "0.4.0", status = Status.INTERNAL)
 @SuppressFBWarnings("DMI_RANDOM_USED_ONLY_ONCE")
 public abstract class Randoms {
-	private static final boolean USE_JQWIK_ENGINE;
 	private static final ThreadLocal<Random> CURRENT;
 	private static final ThreadLocal<Long> SEED;
 
 	static {
-		boolean useJqwikEngine;
-		try {
-			Class.forName("net.jqwik.engine.SourceOfRandomness");
-			useJqwikEngine = true;
-		} catch (ClassNotFoundException e) {
-			useJqwikEngine = false;
-		}
-		USE_JQWIK_ENGINE = useJqwikEngine;
 		SEED = ThreadLocal.withInitial(System::nanoTime);
-		CURRENT = ThreadLocal.withInitial(() -> Randoms.create(SEED.get()));
+		CURRENT = ThreadLocal.withInitial(() -> Randoms.newGlobalSeed(SEED.get()));
 	}
 
 	/**
@@ -60,12 +53,32 @@ public static Random create(String seed) {
 		return CURRENT.get();
 	}
 
+	/**
+	 * sets the initialized seed value.
+	 * If the seed has been initialized, it will no longer be changed.
+	 *
+	 * @param seed the seed value
+	 */
 	public static void setSeed(long seed) {
 		SEED.set(seed);
 	}
 
+	/**
+	 * Creates a new random instance with the given seed.
+	 * It affects the global seed value across multiple FixtureMonkey instances.
+	 * It is not recommended to use this method directly unless you intend to.
+	 * It is generally recommended to use {@link #setSeed(long)} instead.
+	 *
+	 * @param seed the seed value
+	 * @return a new random instance
+	 */
+	public static Random newGlobalSeed(long seed) {
+		initializeGlobalSeed(seed);
+		return CURRENT.get();
+	}
+
 	public static Random current() {
-		return USE_JQWIK_ENGINE
+		return EngineUtils.useJqwikEngine()
 			? SourceOfRandomness.current()
 			: CURRENT.get();
 	}
@@ -78,25 +91,26 @@ public static int nextInt(int bound) {
 		return current().nextInt(bound);
 	}
 
-	private static Random create(long seed) {
-		if (USE_JQWIK_ENGINE) {
-			SEED.set(seed);
-			return SourceOfRandomness.create(String.valueOf(seed));
-		}
-
+	/**
+	 * Creates a new random instance with the given seed. It is not thread safe.
+	 * It is generally recommended to use {@link #setSeed(long)} instead.
+	 * It affects the global seed value across multiple FixtureMonkey instances.
+	 *
+	 * @param seed the seed value
+	 */
+	private static void initializeGlobalSeed(long seed) {
 		try {
 			Random random = newRandom(seed);
 			CURRENT.set(random);
 			SEED.set(seed);
-			return random;
 		} catch (NumberFormatException nfe) {
 			throw new IllegalArgumentException(String.format("[%s] is not a valid random seed.", seed));
 		}
 	}
 
 	private static Random newRandom(final long seed) {
-		return USE_JQWIK_ENGINE
-			? SourceOfRandomness.newRandom(seed)
+		return EngineUtils.useJqwikEngine()
+			? SourceOfRandomness.create(String.valueOf(seed))
 			: new XorShiftRandom(seed);
 	}
 

fixture-monkey-benchmarks/fixture-monkey-benchmark/src/jmh/java/com/navercorp/fixturemonkey/SeedBenchmark.java

@@ -0,0 +1,73 @@
+/*
+ * Fixture Monkey
+ *
+ * Copyright (c) 2021-present NAVER Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.navercorp.fixturemonkey;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.infra.Blackhole;
+
+import com.navercorp.fixturemonkey.api.random.Randoms;
+import com.navercorp.fixturemonkey.api.type.TypeCache;
+import com.navercorp.fixturemonkey.javax.validation.plugin.JavaxValidationPlugin;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+public class SeedBenchmark {
+	private static final int SEED = 1234;
+	private static final int COUNT = 500;
+	private static final FixtureMonkey SUT = FixtureMonkey.builder()
+		.plugin(new JavaxValidationPlugin())
+		.seed(SEED)
+		.build();
+
+	@Setup(value = Level.Iteration)
+	public void setUp() {
+		TypeCache.clearCache();
+	}
+
+	@Benchmark
+	public void eachIterationCreatesNewJqwikSeed(Blackhole blackhole) throws Exception {
+		Randoms.newGlobalSeed(SEED);
+		blackhole.consume(generateOrderSheet());
+	}
+
+	@Benchmark
+	public void eachIterationNotCreatesNewJqwikSeed(Blackhole blackhole) throws Exception {
+		blackhole.consume(generateOrderSheet());
+	}
+
+	private List<OrderSheet> generateOrderSheet() {
+		List<OrderSheet> result = new ArrayList<>();
+		for (int i = 0; i < COUNT; i++) {
+			result.add(SeedBenchmark.SUT.giveMeOne(OrderSheet.class));
+		}
+		return result;
+	}
+}

fixture-monkey-junit-jupiter/build.gradle.kts

@@ -1,5 +1,5 @@
 plugins {
-    id("org.jetbrains.kotlin.jvm") 
+    id("org.jetbrains.kotlin.jvm")
     id("com.navercorp.fixturemonkey.gradle.plugin.java-conventions")
     id("com.navercorp.fixturemonkey.gradle.plugin.maven-publish-conventions")
 }
@@ -16,5 +16,7 @@ dependencies {
 }
 
 tasks.withType<Test> {
-    useJUnitPlatform()
+    useJUnitPlatform {
+        includeEngines("junit-jupiter")
+    }
 }

fixture-monkey-junit-jupiter/src/main/java/com/navercorp/fixturemonkey/junit/jupiter/extension/FixtureMonkeySeedExtension.java

@@ -28,15 +28,30 @@
 import com.navercorp.fixturemonkey.api.random.Randoms;
 import com.navercorp.fixturemonkey.junit.jupiter.annotation.Seed;
 
+/**
+ * This extension sets the seed for generating random numbers before a test method is executed.
+ * It also logs the seed used for the test if the test fails.
+ * It aims to make the test deterministic and reproducible.
+ * <p>
+ * If the test method has a {@link Seed} annotation, it uses the value of the annotation as the seed.
+ * If the test method does not have a {@link Seed} annotation, it uses the hash code of the test method as the seed.
+ * <p>
+ * The {@link Seed} annotation has a higher priority than the option {@code seed} in FixtureMonkey.
+ */
 public final class FixtureMonkeySeedExtension implements BeforeTestExecutionCallback, AfterTestExecutionCallback {
 	private static final Logger LOGGER = LoggerFactory.getLogger(FixtureMonkeySeedExtension.class);
 
 	@Override
-	public void beforeTestExecution(ExtensionContext context) throws Exception {
+	public void beforeTestExecution(ExtensionContext context) {
 		Seed seed = context.getRequiredTestMethod().getAnnotation(Seed.class);
 		if (seed != null) {
 			setSeed(seed.value());
+			return;
 		}
+
+		Method testMethod = context.getRequiredTestMethod();
+		int methodHashCode = testMethod.hashCode();
+		setSeed(methodHashCode);
 	}
 
 	/**
@@ -45,7 +60,7 @@ public void beforeTestExecution(ExtensionContext context) throws Exception {
 	 * If the test failed, it logs the seed used for the test.
 	 **/
 	@Override
-	public void afterTestExecution(ExtensionContext context) throws Exception {
+	public void afterTestExecution(ExtensionContext context) {
 		if (context.getExecutionException().isPresent()) {
 			logSeedIfTestFailed(context);
 		}
@@ -55,7 +70,7 @@ public void afterTestExecution(ExtensionContext context) throws Exception {
 	 * Sets the seed for generating random numbers.
 	 **/
 	private void setSeed(long seed) {
-		Randoms.setSeed(seed);
+		Randoms.newGlobalSeed(seed);
 	}
 
 	/**

fixture-monkey-junit-jupiter/src/test/java/com/navercorp/fixturemonkey/junit/jupiter/extension/FixtureMonkeySeedExtensionTest.java

@@ -20,6 +20,7 @@
 import static org.assertj.core.api.BDDAssertions.then;
 
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -59,10 +60,7 @@ void latterValue() {
 	@Seed(1)
 	@RepeatedTest(100)
 	void containerReturnsSame() {
-		List<String> expected = Arrays.asList(
-			"仛禦催ᘓ蓊類౺阹瞻塢飖獾ࠒ⒐፨婵얎⽒竻·俌欕悳잸횑ٻ킐結",
-			"塸聩ዡ㘇뵥刲禮ᣮ鎊熇捺셾壍Ꜻꌩ垅凗❉償粐믩࠱哠"
-		);
+		List<String> expected = Collections.singletonList("仛禦催ᘓ蓊類౺阹瞻塢飖獾ࠒ⒐፨婵얎⽒竻·俌欕悳잸횑ٻ킐結");
 
 		List<String> actual = SUT.giveMeOne(new TypeReference<List<String>>() {
 		});
@@ -73,9 +71,7 @@ void containerReturnsSame() {
 	@Seed(1)
 	@RepeatedTest(100)
 	void containerMattersOrder() {
-		Set<String> expected = new HashSet<>(
-			Arrays.asList("仛禦催ᘓ蓊類౺阹瞻塢飖獾ࠒ⒐፨婵얎⽒竻·俌欕悳잸횑ٻ킐結", "塸聩ዡ㘇뵥刲禮ᣮ鎊熇捺셾壍Ꜻꌩ垅凗❉償粐믩࠱哠")
-		);
+		Set<String> expected = new HashSet<>(Collections.singletonList("仛禦催ᘓ蓊類౺阹瞻塢飖獾ࠒ⒐፨婵얎⽒竻·俌欕悳잸횑ٻ킐結"));
 
 		Set<String> actual = SUT.giveMeOne(new TypeReference<Set<String>>() {
 		});
@@ -94,4 +90,22 @@ void multipleContainerReturnsDiff() {
 
 		then(firstSet).isNotEqualTo(secondList);
 	}
+
+	@Seed(1)
+	@RepeatedTest(100)
+	void multipleFixtureMonkeyInstancesReturnsAsOneInstance() {
+		List<String> expected = Arrays.asList(
+			"✠섨ꝓ仛禦催ᘓ蓊類౺阹瞻塢飖獾ࠒ⒐፨",
+			"欕悳잸"
+		);
+		FixtureMonkey firstFixtureMonkey = FixtureMonkey.create();
+		FixtureMonkey secondFixtureMonkey = FixtureMonkey.create();
+
+		List<String> actual = Arrays.asList(
+			firstFixtureMonkey.giveMeOne(String.class),
+			secondFixtureMonkey.giveMeOne(String.class)
+		);
+
+		then(actual).isEqualTo(expected);
+	}
 }

fixture-monkey/src/main/java/com/navercorp/fixturemonkey/FixtureMonkeyBuilder.java

@@ -504,9 +504,14 @@ public FixtureMonkeyBuilder pushCustomizeValidOnly(TreeMatcher matcher, boolean
 	}
 
 	/**
-	 * It is deprecated. Please use {@code @Seed} in fixture-monkey-junit-jupiter module.
+	 * sets the seed for generating random numbers.
+	 * <p>
+	 * If you use the {@code fixture-monkey-junit-jupiter} module,
+	 * the seed value can be overridden by the {@code Seed} annotation.
+	 *
+	 * @param seed seed value for generating random numbers.
+	 * @return FixtureMonkeyBuilder
 	 */
-	@Deprecated
 	public FixtureMonkeyBuilder seed(long seed) {
 		this.seed = seed;
 		return this;