make: unit test flake hunter

For convenient flake hunting.

Author: ellemouton

Makefile

@@ -198,6 +198,12 @@ unit:
 	mkdir -p app/build && touch app/build/index.html
 	$(UNIT)
 
+#? unit-debug: Run unit tests with debug log output enabled
+unit-debug:
+	@$(call print, "Running unit tests.")
+	mkdir -p app/build && touch app/build/index.html
+	$(UNIT_DEBUG)
+
 unit-cover: $(GOACC_BIN)
 	@$(call print, "Running unit coverage tests.")
 	$(GOACC_BIN) $(COVER_PKG)
@@ -308,6 +314,11 @@ sqlc-check: sqlc
 	@$(call print, "Verifying sql code generation.")
 	if test -n "$$(git status --porcelain '*.go')"; then echo "SQL models not properly generated!"; git status --porcelain '*.go'; exit 1; fi
 
+#? flakehunter-unit: Run the unit tests continuously until one fails
+flakehunter-unit:
+	@$(call print, "Flake hunting unit test.")
+	scripts/unit-test-flake-hunter.sh ${pkg} ${case}
+
 # Prevent make from interpreting any of the defined goals as folders or files to
 # include in the build process.
 .PHONY: default all yarn-install build install go-build go-build-noui \

make/testing_flags.mk

@@ -51,10 +51,12 @@ UNIT_TARGETED ?= no
 # targeted case. Otherwise, default to running all tests.
 ifeq ($(UNIT_TARGETED), yes)
 UNIT := $(GOTEST) -tags="$(DEV_TAGS) $(COMPILE_TAGS)" $(TEST_FLAGS) $(UNITPKG)
+UNIT_DEBUG := $(GOTEST) -v -tags="$(DEV_TAGS) $(COMPILE_TAGS)" $(TEST_FLAGS) $(UNITPKG)
 endif
 
 ifeq ($(UNIT_TARGETED), no)
 UNIT := $(GOLIST) | $(XARGS) env $(GOTEST) -tags="$(DEV_TAGS) $(COMPILE_TAGS)" $(TEST_FLAGS)
+UNIT_DEBUG := $(GOLIST) | $(XARGS) env $(GOTEST) -v -tags="$(DEV_TAGS) $(COMPILE_TAGS)" $(TEST_FLAGS)
 endif
 
 UNIT_RACE := $(UNIT) -race

scripts/unit-test-flake-hunter.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Check if pkg and case variables are provided.
+if [ $# -lt 2 ] || [ $# -gt 3 ]; then
+	echo "Usage: $0 <pkg> <case> [timeout]"
+	exit 1
+fi
+
+pkg=$1
+case=$2
+timeout=${3:-30s} # Default to 30s if not provided.
+
+counter=0
+
+# Run the command in a loop until it fails.
+while output=$(go clean -testcache && make unit-debug log="stdlog trace" pkg=$pkg case=$case timeout=$timeout 2>&1); do
+	((counter++))
+	echo "Test $case passed, count: $counter"
+done
+
+# Only log the output when it fails.
+echo "Test $case failed. Output:"
+echo "$output"