fix(e2e): Address flakes in TestCliSearch (#6080)

* fix(e2e): Address flake in session count

There was a failure where this returned 1 session when the test expected 0. This commit updates the search to only look at sessions within the created project in this test.

* fix(e2e): Address flake in cache status

This commit addresses a flake where the test attempted to stop and start a cache, but when it goes to start the cache, it claims that the cache is already running. A theory is that the test tries start the cache too quickly after stopping it. Now, we add in a status call to check the status of the cache before proceeding.

Author: moduli

testing/internal/e2e/tests/base/search_test.go

@@ -287,6 +287,7 @@ func TestCliSearch(t *testing.T) {
 		e2e.WithArgs(
 			"search",
 			"-resource", "sessions",
+			"-query", fmt.Sprintf(`scope_id = "%s"`, projectId),
 			"-force-refresh", "true",
 			"-format", "json",
 		),
@@ -370,9 +371,26 @@ func TestCliSearch(t *testing.T) {
 	)
 
 	// Restart cache and confirm search works
-	t.Log("Restarting cache...")
+	t.Log("Stopping cache...")
 	output = e2e.RunCommand(ctx, "boundary", e2e.WithArgs("cache", "stop"))
 	require.NoError(t, output.Err, string(output.Stderr))
+	err = backoff.RetryNotify(
+		func() error {
+			output := e2e.RunCommand(ctx, "boundary", e2e.WithArgs("cache", "status", "-format", "json"))
+
+			if strings.Contains(string(output.Stderr), "The cache process is not running") {
+				return nil
+			}
+
+			return fmt.Errorf("Waiting for cache to stop: %s", string(output.Stdout))
+		},
+		backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), 5),
+		func(err error, td time.Duration) {
+			t.Logf("%s. Retrying...", err.Error())
+		},
+	)
+	require.NoError(t, err)
+	t.Log("Starting cache...")
 	output = e2e.RunCommand(ctx, "boundary",
 		e2e.WithArgs(
 			"cache", "start",
@@ -419,9 +437,26 @@ func TestCliSearch(t *testing.T) {
 
 	// Log out and restart cache. Log in and confirm search works
 	output = e2e.RunCommand(ctx, "boundary", e2e.WithArgs("logout"))
-	t.Log("Restarting cache...")
+	t.Log("Stopping cache...")
 	output = e2e.RunCommand(ctx, "boundary", e2e.WithArgs("cache", "stop"))
 	require.NoError(t, output.Err, string(output.Stderr))
+	err = backoff.RetryNotify(
+		func() error {
+			output := e2e.RunCommand(ctx, "boundary", e2e.WithArgs("cache", "status", "-format", "json"))
+
+			if strings.Contains(string(output.Stderr), "The cache process is not running") {
+				return nil
+			}
+
+			return fmt.Errorf("Waiting for cache to stop: %s", string(output.Stdout))
+		},
+		backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), 5),
+		func(err error, td time.Duration) {
+			t.Logf("%s. Retrying...", err.Error())
+		},
+	)
+	require.NoError(t, err)
+	t.Log("Starting cache...")
 	output = e2e.RunCommand(ctx, "boundary",
 		e2e.WithArgs(
 			"cache", "start",