feat: add test_flakiness target and AwaitCommandTestCase for improved testing reliability

- Introduced a new Makefile target `test_flakiness` to run tests multiple times for flakiness detection.
- Added `AwaitCommandTestCase` struct to handle command execution with await and retry logic in tests.
- Updated `testStreamsXreadBlockMaxID` to utilize the new `AwaitCommandTestCase` for better synchronization and error handling.
- Enhanced GitHub Actions workflow to include flakiness testing step.

Author: andy1li

.github/workflows/test.yml

@@ -45,3 +45,16 @@ jobs:
         with:
           version: "2023.1"
           install-go: false
+
+  test-flakiness:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.23.x
+
+      - run: RUNS=25 make test_flakiness

Makefile

@@ -29,6 +29,19 @@ record_fixtures:
 update_tester_utils:
 	go get -u github.com/codecrafters-io/tester-utils
 
+TEST_TARGET ?= test
+RUNS ?= 100
+test_flakiness:
+	@$(foreach i,$(shell seq 1 $(RUNS)), \
+		echo "Running iteration $(i)/$(RUNS) of \"make $(TEST_TARGET)\"" ; \
+		make $(TEST_TARGET) > /tmp/test ; \
+		if [ "$$?" -ne 0 ]; then \
+			echo "Test failed on iteration $(i)" ; \
+			cat /tmp/test ; \
+			exit 1 ; \
+		fi ;\
+	)
+
 test_base_with_redis: build
 	CODECRAFTERS_REPOSITORY_DIR=./internal/test_helpers/pass_all \
 	CODECRAFTERS_TEST_CASES_JSON="[{\"slug\":\"jm1\",\"tester_log_prefix\":\"stage-1\",\"title\":\"Stage #1: Bind to a port\"},{\"slug\":\"rg2\",\"tester_log_prefix\":\"stage-2\",\"title\":\"Stage #2: Respond to PING\"},{\"slug\":\"wy1\",\"tester_log_prefix\":\"stage-3\",\"title\":\"Stage #3: Respond to multiple PINGs\"},{\"slug\":\"zu2\",\"tester_log_prefix\":\"stage-4\",\"title\":\"Stage #4: Handle concurrent clients\"},{\"slug\":\"qq0\",\"tester_log_prefix\":\"stage-5\",\"title\":\"Stage #5: Implement the ECHO command\"},{\"slug\":\"la7\",\"tester_log_prefix\":\"stage-6\",\"title\":\"Stage #6: Implement the SET \u0026 GET commands\"},{\"slug\":\"yz1\",\"tester_log_prefix\":\"stage-7\",\"title\":\"Stage #7: Expiry\"}]" \

internal/test_cases/await_command_test_case.go

@@ -0,0 +1,84 @@
+package test_cases
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	resp_client "github.com/codecrafters-io/redis-tester/internal/resp/connection"
+	resp_value "github.com/codecrafters-io/redis-tester/internal/resp/value"
+	"github.com/codecrafters-io/redis-tester/internal/resp_assertions"
+	"github.com/codecrafters-io/tester-utils/logger"
+)
+
+type AwaitCommandTestCase struct {
+	Command                   string
+	Args                      []string
+	Assertion                 resp_assertions.RESPAssertion
+	ShouldSkipUnreadDataCheck bool
+	Retries                   int
+	ShouldRetryFunc           func(resp_value.Value) bool
+
+	// ReceivedResponse is set after the test case is run
+	ReceivedResponse resp_value.Value
+
+	AwaitChan chan bool
+	DoneChan  chan bool
+}
+
+func (t *AwaitCommandTestCase) Run(client *resp_client.RespConnection, logger *logger.Logger) error {
+	var value resp_value.Value
+	var err error
+
+	for attempt := 0; attempt <= t.Retries; attempt++ {
+		if attempt > 0 {
+			logger.Infof("Retrying... (%d/%d attempts)", attempt, t.Retries)
+		}
+
+		command := strings.ToUpper(t.Command)
+
+		if err = client.SendCommand(command, t.Args...); err != nil {
+			return err
+		}
+
+		value, err = client.ReadValue()
+		if err != nil {
+			return err
+		}
+
+		if t.Retries == 0 {
+			break
+		}
+
+		if t.ShouldRetryFunc == nil {
+			panic(fmt.Sprintf("Received SendCommand with retries: %d but no ShouldRetryFunc.", t.Retries))
+		} else {
+			if t.ShouldRetryFunc(value) {
+				// If ShouldRetryFunc returns true, we sleep and retry.
+				time.Sleep(500 * time.Millisecond)
+			} else {
+				break
+			}
+		}
+	}
+
+	t.ReceivedResponse = value
+
+	<-t.AwaitChan
+
+	if err = t.Assertion.Run(value); err != nil {
+		return err
+	}
+
+	if !t.ShouldSkipUnreadDataCheck {
+		client.ReadIntoBuffer() // Let's make sure there's no extra data
+
+		if client.UnreadBuffer.Len() > 0 {
+			return fmt.Errorf("Found extra data: %q", client.UnreadBuffer.String())
+		}
+	}
+
+	logger.Successf("Received %s", value.FormattedString())
+	t.DoneChan <- true
+	return nil
+}

internal/test_streams_xread_block_max_id.go

@@ -42,7 +42,8 @@ func testStreamsXreadBlockMaxID(stageHarness *test_case_harness.TestCaseHarness)
 		return err
 	}
 
-	waitChan := make(chan bool, 1)
+	xReadDone := make(chan bool, 1)
+	xAddDone := make(chan bool, 1)
 	randomInt = testerutils_random.RandomInt(1, 100)
 
 	go func() error {
@@ -53,20 +54,16 @@ func testStreamsXreadBlockMaxID(stageHarness *test_case_harness.TestCaseHarness)
 				FieldValuePairs: [][]string{{"temperature", strconv.Itoa(randomInt)}},
 			}},
 		}})
-		xreadCommandTestCase := &test_cases.SendCommandTestCase{
+		awaitXReadTestCase := &test_cases.AwaitCommandTestCase{
 			Command:                   "XREAD",
 			Args:                      []string{"block", "0", "streams", randomKey, "$"},
 			Assertion:                 assertion,
 			ShouldSkipUnreadDataCheck: true,
+			AwaitChan:                 xAddDone,
+			DoneChan:                  xReadDone,
 		}
 
-		if err := xreadCommandTestCase.Run(client1, logger); err != nil {
-			logger.Errorf("Error: %v", err)
-			return err
-		}
-
-		waitChan <- true
-		return nil
+		return awaitXReadTestCase.Run(client1, logger)
 	}()
 
 	time.Sleep(1000 * time.Millisecond)
@@ -89,7 +86,8 @@ func testStreamsXreadBlockMaxID(stageHarness *test_case_harness.TestCaseHarness)
 		return err
 	}
 
-	<-waitChan
+	xAddDone <- true
+	<-xReadDone
 
 	xreadCommandTestCase := &test_cases.SendCommandTestCase{
 		Command:                   "XREAD",