Fix ambient multi-cluster integration tests (#58466)

It appears that TestServerSideLB is flaky in multi-network setup.
I think there are two contributing factors for that:

1. With two clusters we have twice as many backends to hit, so the same
   10 attempts may not always be enough
2. Specifically when waypoints are involved, we may be affected by
   istio/istio#58039, though after running the
   test locally, it does not seem critical.

Aside from that, I also cleaned up some of the skips that were added
when we just started running ambient integration tests in multi-cluster
environment.

As things stand now there are just three known reasons why we should skip
a test in ambient multi-cluster:

1. When in test workload behind a sidecar proxy tries to talk to a
   workload behind ztunnel (but not other way around) - the fix for that
   is tracked in istio/istio#57878, but given
   that interoperability between ambient and sidecar is not officially
   supported, it's ok to skip those tests.
2. We skip TestServiceDynamicEnroll because it's flaky in general and
   running it in single cluster mode as well as in multi-cluster mode just
   increases the chances of hitting a flake for somewhat unclear benefits
   at the moment; there is a ticket open to fix the test -
   istio/istio#58228.
3. We skip TestTrafficSplit at the moment because it relies on
   subsetting and we lose that information when we cross network
   boundary; it's debatable if we want to support this feature at all in
   multi-cluster, but for now we track it in
   istio/istio#58140.

All other tests should be running and passing in ambient multi-cluster
mode.

Fixes #58020
Fixes #58080
Fixes #56228

Signed-off-by: Mikhail Krinkin <[email protected]>

Author: krinkinmu

tests/integration/ambient/baseline_test.go

@@ -179,7 +179,7 @@ func TestServices(t *testing.T) {
 			opt.Check = tcpValidator
 		}
 
-		if src.Config().HasSidecar() && t.Settings().AmbientMultiNetwork {
+		if t.Settings().AmbientMultiNetwork && src.Config().HasSidecar() && !dst.Config().HasSidecar() {
 			t.Skip("https://github.com/istio/istio/issues/57878")
 		}
 
@@ -355,8 +355,9 @@ func TestServerSideLB(t *testing.T) {
 			shouldBalance := dst.Config().HasServiceAddressedWaypointProxy()
 			// Istio client will not reuse connections for HTTP/1.1
 			opt.HTTP.HTTP2 = true
-			// Make sure we make multiple calls
-			opt.Count = 10
+			// Make sure we make multiple calls and scale the number of calls with the number of clusters as well.
+			// We need to make sure that we run enough requests to cover all the backends we want to hit.
+			opt.Count = 10 * len(t.AllClusters())
 			c := singleHost
 			if shouldBalance {
 				c = multipleHost
@@ -950,19 +951,6 @@ spec:
 	})
 }
 
-func sidecarInjected(t framework.TestContext, es ...echo.Target) bool {
-	t.Helper()
-
-	for _, e := range es {
-		for _, w := range e.WorkloadsOrFail(t) {
-			if w.Sidecar() != nil {
-				return true
-			}
-		}
-	}
-	return false
-}
-
 func TestAuthorizationL4(t *testing.T) {
 	framework.NewTest(t).Run(func(t framework.TestContext) {
 		applyDrainingWorkaround(t)
@@ -972,7 +960,7 @@ func TestAuthorizationL4(t *testing.T) {
 				t.Skip("this test is only for L4/TCP")
 			}
 
-			if t.Settings().AmbientMultiNetwork && sidecarInjected(t, src) && !sidecarInjected(t, dst) {
+			if t.Settings().AmbientMultiNetwork && src.Config().HasSidecar() && !dst.Config().HasSidecar() {
 				// Currently sidecars cannot talk to ambient endpoints on a remote network.
 				//
 				// Sidecars can't use double-HBONE and therefore cannot use ambient E/W gateways.
@@ -1029,7 +1017,7 @@ func TestAuthorizationL4(t *testing.T) {
 
 			for _, tc := range authzCases {
 				t.NewSubTest(tc.name).Run(func(t framework.TestContext) {
-					if t.Settings().AmbientMultiNetwork && (src.Config().HasSidecar() || dst.Config().HasSidecar()) {
+					if t.Settings().AmbientMultiNetwork && src.Config().HasSidecar() && !dst.Config().HasSidecar() {
 						// Sidecar + Ambient is not supported
 						t.Skip("https://github.com/istio/istio/issues/57878")
 					}
@@ -1141,9 +1129,6 @@ func TestAuthorizationGateway(t *testing.T) {
 					Check:   check.OK(),
 					To:      dst,
 				}
-				if t.Settings().AmbientMultiNetwork {
-					t.Skip("https://github.com/istio/istio/issues/57878")
-				}
 				f(t, istio.DefaultIngressOrFail(t, t), dst, opt)
 			})
 		}
@@ -2881,7 +2866,7 @@ func runTestContextForCalls(
 		t.NewSubTestf("from %v %v", src.Config().Cluster.Name(), src.Config().Service).Run(func(t framework.TestContext) {
 			for _, dst := range getAllInstancesByServiceName() {
 				t.NewSubTestf("to all %v", dst.Config().Service).Run(func(t framework.TestContext) {
-					if t.Settings().AmbientMultiNetwork && (src.Config().HasSidecar() || dst.Config().HasSidecar()) {
+					if t.Settings().AmbientMultiNetwork && src.Config().HasSidecar() && !dst.Config().HasSidecar() {
 						// Skip sidecar to sidecar in multinetwork, as they will use the east-west gateway which is not tested here.
 						t.Skip("https://github.com/istio/istio/issues/57878")
 					}
@@ -3803,9 +3788,6 @@ spec:
 func TestWaypointWithSidecarBackend(t *testing.T) {
 	framework.NewTest(t).
 		Run(func(t framework.TestContext) {
-			if t.Settings().AmbientMultiNetwork {
-				t.Skip("https://github.com/istio/istio/issues/57878")
-			}
 			// Ensure we go through the waypoint (verified by modifying the request) and that we are doing mTLS.
 			t.ConfigIstio().
 				Eval(apps.Namespace.Name(), apps.Namespace.Name(), `apiVersion: gateway.networking.k8s.io/v1