Skip to content

Commit 15bb9a4

Browse files
jweite-amazonjweite-amazon
committed
First draft of Toxiproxy-based network interruption E2E test.
1 parent 734b333 commit 15bb9a4

File tree

4 files changed

+241
-5
lines changed

4 files changed

+241
-5
lines changed

‎test/e2e/helpers/toxiProxy.go

Lines changed: 30 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,38 @@
1+
/*
2+
Copyright 2022 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
117
package helpers
218

319
import (
420
"context"
521
"fmt"
6-
toxiproxyapi "github.com/Shopify/toxiproxy/v2/client"
7-
. "github.com/onsi/gomega"
8-
corev1 "k8s.io/api/core/v1"
922
"net"
1023
"os"
1124
"path"
1225
"regexp"
26+
"strconv"
27+
"strings"
28+
29+
toxiproxyapi "github.com/Shopify/toxiproxy/v2/client"
30+
. "github.com/onsi/gomega"
31+
corev1 "k8s.io/api/core/v1"
1332
"sigs.k8s.io/cluster-api/test/framework"
1433
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
1534
"sigs.k8s.io/cluster-api/test/framework/exec"
1635
"sigs.k8s.io/controller-runtime/pkg/client"
17-
"strconv"
18-
"strings"
1936
)
2037

2138
func ToxiProxyServerExec(ctx context.Context) error {
@@ -133,6 +150,14 @@ func (tp *ToxiProxyContext) AddLatencyToxic(latencyMs int, jitterMs int, toxicit
133150
return toxicName
134151
}
135152

153+
func (tp *ToxiProxyContext) Disable() {
154+
tp.ToxiProxy.Disable()
155+
}
156+
157+
func (tp *ToxiProxyContext) Enable() {
158+
tp.ToxiProxy.Enable()
159+
}
160+
136161
func SetupForToxiProxyTestingACS(ctx context.Context, clusterName string, clusterProxy framework.ClusterProxy, e2eConfig *clusterctl.E2EConfig, configPath string) *ToxiProxyContext {
137162
// Get the cloud-config secret that CAPC will use to access CloudStack
138163
fdEndpointSecretObjectKey := client.ObjectKey{

‎test/e2e/helpers/utilities.go

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
Copyright 2022 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package helpers
18+
19+
import "time"
20+
21+
func InterruptibleSleep(sleepDuration time.Duration, resolution time.Duration, interruptionChannel chan bool) bool {
22+
for entryTime := time.Now(); time.Since(entryTime) < sleepDuration; {
23+
select {
24+
case <-interruptionChannel:
25+
return true
26+
default:
27+
time.Sleep(resolution)
28+
break
29+
}
30+
}
31+
return false
32+
}

‎test/e2e/network_interruption_toxi.go

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
/*
2+
Copyright 2022 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package e2e
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"os"
23+
"path/filepath"
24+
"runtime"
25+
"time"
26+
27+
. "github.com/onsi/ginkgo"
28+
. "github.com/onsi/gomega"
29+
corev1 "k8s.io/api/core/v1"
30+
"k8s.io/utils/pointer"
31+
"sigs.k8s.io/cluster-api-provider-cloudstack-staging/test/e2e/helpers"
32+
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
33+
"sigs.k8s.io/cluster-api/util"
34+
)
35+
36+
// DeployAppToxiSpec implements a test that verifies that an app deployed to the workload cluster works.
37+
func NetworkInterruptionToxiSpec(ctx context.Context, inputGetter func() CommonSpecInput) {
38+
var (
39+
specName = "network-interruption-toxi"
40+
input CommonSpecInput
41+
namespace *corev1.Namespace
42+
cancelWatches context.CancelFunc
43+
clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult
44+
cloudStackToxiProxyContext *helpers.ToxiProxyContext
45+
clusterName = fmt.Sprintf("%s-%s", specName, util.RandomString(6))
46+
networkInterruptorShutdownChannel = make(chan bool, 2)
47+
)
48+
49+
BeforeEach(func() {
50+
// ToxiProxy running in a docker container requires docker host networking, only available in linux.
51+
Expect(runtime.GOOS).To(Equal("linux"))
52+
53+
Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName)
54+
input = inputGetter()
55+
Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName)
56+
Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName)
57+
Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName)
58+
Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName)
59+
Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion))
60+
61+
// Set up a toxiProxy for CloudStack
62+
cloudStackToxiProxyContext = helpers.SetupForToxiProxyTestingACS(
63+
ctx,
64+
clusterName,
65+
input.BootstrapClusterProxy,
66+
input.E2EConfig,
67+
input.ClusterctlConfigPath,
68+
)
69+
70+
// Set up a Namespace to host objects for this spec and create a watcher for the namespace events.
71+
namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder)
72+
clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult)
73+
74+
})
75+
76+
It("Should be able to create a cluster despite a network interruption during that process", func() {
77+
By("Creating a workload cluster")
78+
79+
flavor := clusterctl.DefaultFlavor
80+
if input.Flavor != nil {
81+
flavor = *input.Flavor
82+
}
83+
namespace := namespace.Name
84+
85+
// While I'd prefer to closely synchronize the network interruption (ToxiProxy disable) to a particular point in the cluster provisioning
86+
// process, doing so for this asynchronously running process would be harder and more impactful than I can tackle right now. So I'm going
87+
// to give CAPC a short period to get started with the provisioning, and then interrupt the network for a fixed time, and then restore it.
88+
// CAPC should tolerate this and ultimately succeed.
89+
// To do this while ApplyClusterTemplateAndWait() is waiting, I'm going to use a concurrent goroutine and an interruptible version of sleep
90+
// so it can be shut down cleanly.
91+
go networkInterruptor(cloudStackToxiProxyContext, networkInterruptorShutdownChannel)
92+
93+
clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
94+
ClusterProxy: input.BootstrapClusterProxy,
95+
CNIManifestPath: input.E2EConfig.GetVariable(CNIPath),
96+
ConfigCluster: clusterctl.ConfigClusterInput{
97+
LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()),
98+
ClusterctlConfigPath: input.ClusterctlConfigPath,
99+
KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(),
100+
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
101+
Flavor: flavor,
102+
Namespace: namespace,
103+
ClusterName: clusterName,
104+
KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion),
105+
ControlPlaneMachineCount: pointer.Int64Ptr(1),
106+
WorkerMachineCount: pointer.Int64Ptr(2),
107+
},
108+
WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"),
109+
WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"),
110+
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
111+
}, clusterResources)
112+
113+
By("PASSED!")
114+
})
115+
116+
AfterEach(func() {
117+
// Stop the networkInterruptor (in case it's still running because tests failed before it completed)
118+
networkInterruptorShutdownChannel <- true
119+
120+
// Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself.
121+
dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup)
122+
123+
// Tear down the ToxiProxies
124+
helpers.TearDownToxiProxyACS(ctx, input.BootstrapClusterProxy, cloudStackToxiProxyContext)
125+
})
126+
}
127+
128+
func networkInterruptor(toxiProxyContext *helpers.ToxiProxyContext, shutdownChannel chan bool) {
129+
// Wait for ApplyClusterTemplateAndWait() to make some progress
130+
helpers.InterruptibleSleep(15*time.Second, time.Second, shutdownChannel)
131+
132+
// Disable communications to ACS
133+
toxiProxyContext.Disable()
134+
135+
// Leave the network disabled for some period of time
136+
helpers.InterruptibleSleep(30*time.Second, time.Second, shutdownChannel)
137+
138+
// Restore communications to ACS
139+
toxiProxyContext.Enable()
140+
}

‎test/e2e/network_interruption_toxi_test.go

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
//go:build e2e
2+
// +build e2e
3+
4+
/*
5+
Copyright 2020 The Kubernetes Authors.
6+
7+
Licensed under the Apache License, Version 2.0 (the "License");
8+
you may not use this file except in compliance with the License.
9+
You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing, software
14+
distributed under the License is distributed on an "AS IS" BASIS,
15+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
See the License for the specific language governing permissions and
17+
limitations under the License.
18+
*/
19+
20+
package e2e
21+
22+
import (
23+
"context"
24+
. "github.com/onsi/ginkgo"
25+
)
26+
27+
var _ = Describe("When testing app deployment to the workload cluster with network interruption [ToxiProxy]", func() {
28+
29+
NetworkInterruptionToxiSpec(context.TODO(), func() CommonSpecInput {
30+
return CommonSpecInput{
31+
E2EConfig: e2eConfig,
32+
ClusterctlConfigPath: clusterctlConfigPath,
33+
BootstrapClusterProxy: bootstrapClusterProxy,
34+
ArtifactFolder: artifactFolder,
35+
SkipCleanup: skipCleanup,
36+
}
37+
})
38+
39+
})

0 commit comments

Comments
 (0)