e2e: reduce subtest interdependencies

The jsonpatch change for bm-tcb-specs surfaced some issues where we
implicitly depended on unrelated test runs. This commit aims to reduce
these dependencies by not sharing as much state between tests, or by
using an entirely fresh ContrastTest instance where feasible.

Author: burgerdev

e2e/policy/deterministic_test.go

@@ -7,6 +7,7 @@ package policy
 
 import (
 	"encoding/json"
+	"fmt"
 	"os"
 	"path"
 	"testing"
@@ -22,7 +23,6 @@ func TestDeterminsticPolicyGeneration(t *testing.T) {
 	require := require.New(t)
 	platform, err := platforms.FromString(contrasttest.Flags.PlatformStr)
 	require.NoError(err)
-	ct := contrasttest.New(t)
 
 	// create K8s resources
 	runtimeHandler, err := manifest.RuntimeHandler(platform)
@@ -33,26 +33,37 @@ func TestDeterminsticPolicyGeneration(t *testing.T) {
 	resources = kuberesource.PatchRuntimeHandlers(resources, runtimeHandler)
 	unstructuredResources, err := kuberesource.ResourcesToUnstructured(resources)
 	require.NoError(err)
-	buf, err := kuberesource.EncodeUnstructured(unstructuredResources)
+	resourcesBytes, err := kuberesource.EncodeUnstructured(unstructuredResources)
 	require.NoError(err)
 
 	// generate policy 5 times and check if the policy hash is the same
 	var expectedPolicies map[manifest.HexString]manifest.PolicyEntry
 	for i := range 5 {
-		t.Log("Generate run", i)
-		require.NoError(os.WriteFile(path.Join(ct.WorkDir, "resources.yml"), buf, 0o644)) // reset file for each run
-		require.True(t.Run("generate", ct.Generate), "contrast generate needs to succeed for subsequent tests")
-		manifestBytes, err := os.ReadFile(ct.ManifestPath())
-		require.NoError(err)
-
-		// verify that policies are deterministic
-		var m manifest.Manifest
-		require.NoError(json.Unmarshal(manifestBytes, &m))
-		if expectedPolicies != nil {
-			require.Equal(expectedPolicies, m.Policies, "expected deterministic policy generation")
-		} else {
-			expectedPolicies = m.Policies // only set policies on the first run
-		}
+		t.Run(fmt.Sprint(i), func(t *testing.T) {
+			policies := runGenerate(t, resourcesBytes)
+
+			// verify that policies are deterministic
+			if expectedPolicies != nil {
+				require.Equal(expectedPolicies, policies, "expected deterministic policy generation")
+			} else {
+				expectedPolicies = policies // only set policies on the first run
+			}
+		})
 	}
 	t.Log("Policies are deterministic")
 }
+
+func runGenerate(t *testing.T, resources []byte) map[manifest.HexString]manifest.PolicyEntry {
+	require := require.New(t)
+	ct := contrasttest.New(t)
+
+	require.NoError(os.WriteFile(path.Join(ct.WorkDir, "resources.yml"), resources, 0o644)) // reset file for each run
+	require.True(t.Run("generate", ct.Generate), "contrast generate needs to succeed for subsequent tests")
+	manifestBytes, err := os.ReadFile(ct.ManifestPath())
+	require.NoError(err)
+
+	var m manifest.Manifest
+	require.NoError(json.Unmarshal(manifestBytes, &m))
+
+	return m.Policies
+}

e2e/proxy/proxy_test.go

@@ -42,15 +42,6 @@ func TestHTTPProxy(t *testing.T) {
 
 	runtimeHandler, err := manifest.RuntimeHandler(platform)
 	require.NoError(t, err)
-
-	ct := contrasttest.New(t)
-
-	resources := kuberesource.CoordinatorBundle()
-	resources = kuberesource.PatchRuntimeHandlers(resources, runtimeHandler)
-	resources = kuberesource.AddPortForwarders(resources)
-
-	ct.Init(t, resources)
-
 	// Start a proxy server
 
 	proxy := goproxy.NewProxyHttpServer()
@@ -116,6 +107,13 @@ func TestHTTPProxy(t *testing.T) {
 			assert := assert.New(t)
 			require := require.New(t)
 
+			resources := kuberesource.CoordinatorBundle()
+			resources = kuberesource.PatchRuntimeHandlers(resources, runtimeHandler)
+			resources = kuberesource.AddPortForwarders(resources)
+
+			ct := contrasttest.New(t)
+			ct.Init(t, resources)
+
 			if tc.wantErrMsg != "" {
 				// Run generate and apply in this process so that they don't use the bad proxy.
 				require.True(t.Run("generate", ct.Generate))

e2e/regression/regression_test.go

@@ -14,6 +14,7 @@ import (
 	"context"
 	"errors"
 	"flag"
+	"io/fs"
 	"os"
 	"path"
 	"path/filepath"
@@ -68,6 +69,12 @@ func TestRegression(t *testing.T) {
 		t.Run(dir.Name(), func(t *testing.T) {
 			require := require.New(t)
 
+			// Make sure we start with a fresh manifest, not the one of the previous test or the
+			// outer generate.
+			if err := os.Remove(ct.ManifestPath()); err != nil {
+				require.ErrorIs(err, fs.ErrNotExist)
+			}
+
 			c := kubeclient.NewForTest(t)
 
 			pattern := filepath.Join(dataDir, dir.Name(), "*.yml")