Skip to content

Commit 9365272

Browse files
pjohnst5pjohnst5
authored
Aks swift test update (#838)
* AKS-Swift-test update to use underlay in runners sub
1 parent 85cd950 commit 9365272

File tree

10 files changed

+115
-16
lines changed

10 files changed

+115
-16
lines changed

‎.gitignore‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,7 @@ ipam-*.xml
1313
*.out
1414

1515
# Goland
16-
.idea/*
16+
.idea/*
17+
18+
# Logs
19+
*.log

‎.pipelines/singletenancy/aks-swift/e2e-job-template.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ stages:
2323
GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
2424
GOBIN: "$(GOPATH)/bin" # Go binaries path
2525
ImageTag: $[ stagedependencies.build_and_test.unit_tests.outputs['EnvironmentalVariables.ImageTag'] ]
26+
modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
2627
steps:
2728
- template: e2e-step-template.yaml
2829
parameters:

‎.pipelines/singletenancy/aks-swift/e2e-step-template.yaml‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,45 @@ steps:
3131
addSpnToEnvironment: true
3232
inlineScript: |
3333
mkdir -p ~/.kube/
34-
az keyvault secret show --name $(KV_AKS_SWIFT_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//'
35-
az keyvault secret show --name $(KV_AKS_SWIFT_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//' > ~/.kube/config
34+
echo "DNC Underlay config"
35+
az keyvault secret show --name $(KV_DNC_UNDERLAY_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//'
36+
az keyvault secret show --name $(KV_DNC_UNDERLAY_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//' > ~/.kube/dnc-underlay
37+
echo "Dirty ACN cx cluster config"
38+
az keyvault secret show --name $(KV_AKS_SWIFT_CX_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//'
39+
az keyvault secret show --name $(KV_AKS_SWIFT_CX_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//' > ~/.kube/config
3640
displayName: Pull Kubeconfig from Keyvault
3741
condition: succeeded()
3842

3943
- script: |
4044
ls -lah
4145
pwd
46+
echo "installing kubectl"
47+
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
48+
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
49+
kubectl cluster-info
50+
kubectl get po -owide -A
4251
sudo -E env "PATH=$PATH" VERSION=$(ImageTag) INSTALL_CNI=true INSTALL_CNS=true make test-integration
4352
name: "aksswifte2e"
4453
displayName: "Run AKS Swift E2E"
4554
55+
- script: |
56+
echo "Logs will be available as a build artifact"
57+
ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/aks-swift-output/
58+
echo $ARTIFACT_DIR
59+
sudo rm -rf $ARTIFACT_DIR
60+
sudo mkdir $ARTIFACT_DIR
61+
sudo cp test/integration/logs/* $ARTIFACT_DIR
62+
export KUBECONFIG=~/.kube/dnc-underlay
63+
kubectl logs -n $STABLE_DNC_NAMESPACE -l app=dnc --tail -1 -c dnc --prefix > dnc-logs.txt
64+
kubectl logs -n $STABLE_DNC_NAMESPACE -l app=dnc --tail -1 -c requestcontroller --prefix > dnc-rc-logs.txt
65+
66+
sudo cp *.txt $ARTIFACT_DIR
67+
name: "GetLogs"
68+
displayName: "Get logs"
69+
condition: always()
4670
71+
- task: PublishBuildArtifacts@1
72+
inputs:
73+
artifactName: aks-swift-output
74+
pathtoPublish: "$(Build.ArtifactStagingDirectory)/aks-swift-output"
75+
condition: always()

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -266,7 +266,7 @@ tools: acncli
266266

267267
.PHONY: tools-images
268268
tools-images:
269-
docker build -f ./tools/acncli/Dockerfile --build-arg VERSION=$(VERSION) -t $(AZURE_CNI_IMAGE):$(VERSION) .
269+
docker build --no-cache -f ./tools/acncli/Dockerfile --build-arg VERSION=$(VERSION) -t $(AZURE_CNI_IMAGE):$(VERSION) .
270270

271271
# Build the Azure CNM plugin image, installable with "docker plugin install".
272272
.PHONY: azure-vnet-plugin-image

‎test/integration/manifests/cni/manager.yaml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,9 @@ spec:
1919
hostNetwork: true
2020
containers:
2121
- name: azure-cni-installer
22-
image: mcr.microsoft.com/containernetworking/azure-cni-manager:v1.2.0-2-g0671b63
22+
image: acnpublic.azurecr.io/azure-cni-manager:v1.2.8-32-g77506640
23+
command: ["./acn"]
24+
args: ["cni", "manager", "--follow", "--mode", "transparent", "--ipam", "azure-cns"]
2325
imagePullPolicy: Always
2426
env:
2527
- name: AZURE_CNI_OS

‎test/integration/manifests/cns/daemonset.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ spec:
4141
effect: NoSchedule
4242
containers:
4343
- name: cns-container
44-
image: mcr.microsoft.com/containernetworking/azure-cns:v1.2.0
44+
image: acnpublic.azurecr.io/azure-cns:v1.2.8
4545
imagePullPolicy: IfNotPresent
4646
args: [ "-c", "tcp://$(CNSIpAddress):$(CNSPort)", "-t", "$(CNSLogTarget)"]
4747
volumeMounts:

‎test/integration/setup_test.go‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@ const (
3636
cnsRoleBindingPath = cnsManifestFolder + "/rolebinding.yaml"
3737
cnsServiceAccountPath = cnsManifestFolder + "/serviceaccount.yaml"
3838
cnsLabelSelector = "k8s-app=azure-cns"
39+
40+
// relative log directory
41+
logDir = "logs/"
3942
)
4043

4144
func TestMain(m *testing.M) {
@@ -75,8 +78,8 @@ func TestMain(m *testing.M) {
7578
ctx := context.Background()
7679
if installopt := os.Getenv(envInstallCNI); installopt != "" {
7780
// create dirty cni-manager ds
78-
if installCNI, err := strconv.ParseBool(installopt); err != nil && installCNI == true {
79-
if cnicleanup, err = installCNIManagerDaemonset(ctx, clientset, os.Getenv(envImageTag)); err != nil {
81+
if installCNI, err := strconv.ParseBool(installopt); err == nil && installCNI == true {
82+
if cnicleanup, err = installCNIManagerDaemonset(ctx, clientset, os.Getenv(envImageTag), logDir); err != nil {
8083
log.Print(err)
8184
return
8285
}
@@ -87,8 +90,8 @@ func TestMain(m *testing.M) {
8790

8891
if installopt := os.Getenv(envInstallCNS); installopt != "" {
8992
// create dirty cns ds
90-
if installCNS, err := strconv.ParseBool(installopt); err != nil && installCNS == true {
91-
if cnscleanup, err = installCNSDaemonset(ctx, clientset, os.Getenv(envImageTag)); err != nil {
93+
if installCNS, err := strconv.ParseBool(installopt); err == nil && installCNS == true {
94+
if cnscleanup, err = installCNSDaemonset(ctx, clientset, os.Getenv(envImageTag), logDir); err != nil {
9295
return
9396
}
9497
}
@@ -99,7 +102,7 @@ func TestMain(m *testing.M) {
99102
exitCode = m.Run()
100103
}
101104

102-
func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, imageTag string) (func() error, error) {
105+
func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, imageTag, logDir string) (func() error, error) {
103106
var (
104107
err error
105108
cns v1.DaemonSet
@@ -119,7 +122,6 @@ func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, i
119122
cnsDaemonsetClient := clientset.AppsV1().DaemonSets(cns.Namespace)
120123

121124
log.Printf("Installing CNS with image %s", cns.Spec.Template.Spec.Containers[0].Image)
122-
123125
// setup the CNS configmap
124126
if err := mustSetupConfigMap(ctx, clientset, cnsConfigMapPath); err != nil {
125127
return nil, err
@@ -144,6 +146,9 @@ func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, i
144146
}
145147

146148
cleanupds := func() error {
149+
if err := exportLogsByLabelSelector(ctx, clientset, cns.Namespace, cnsLabelSelector, logDir); err != nil {
150+
return err
151+
}
147152
if err := mustDeleteDaemonset(ctx, cnsDaemonsetClient, cns); err != nil {
148153
return err
149154
}
@@ -153,7 +158,7 @@ func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, i
153158
return cleanupds, nil
154159
}
155160

156-
func installCNIManagerDaemonset(ctx context.Context, clientset *kubernetes.Clientset, imageTag string) (func() error, error) {
161+
func installCNIManagerDaemonset(ctx context.Context, clientset *kubernetes.Clientset, imageTag, logDir string) (func() error, error) {
157162
var (
158163
err error
159164
cni v1.DaemonSet
@@ -179,6 +184,9 @@ func installCNIManagerDaemonset(ctx context.Context, clientset *kubernetes.Clien
179184
}
180185

181186
cleanupds := func() error {
187+
if err := exportLogsByLabelSelector(ctx, clientset, cni.Namespace, cniLabelSelector, logDir); err != nil {
188+
return err
189+
}
182190
if err := mustDeleteDaemonset(ctx, cniDaemonsetClient, cni); err != nil {
183191
return err
184192
}

‎test/integration/utils_test.go‎

Lines changed: 57 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@
33
package k8s
44

55
import (
6+
"bytes"
67
"context"
78
"errors"
9+
"io"
810
"log"
911
"strings"
1012
"time"
@@ -220,6 +222,60 @@ func waitForPodsRunning(ctx context.Context, clientset *kubernetes.Clientset, na
220222
return nil
221223
}
222224

223-
retrier := retry.Retrier{Attempts: 10, Delay: 2 * time.Second}
225+
retrier := retry.Retrier{Attempts: 10, Delay: 6 * time.Second}
224226
return retrier.Do(ctx, checkPodIPsFn)
225227
}
228+
229+
230+
func exportLogsByLabelSelector(ctx context.Context, clientset *kubernetes.Clientset, namespace, labelselector, logDir string) error {
231+
podsClient := clientset.CoreV1().Pods(namespace)
232+
podLogOpts := corev1.PodLogOptions{}
233+
logExtension := ".log"
234+
podList, err := podsClient.List(ctx, metav1.ListOptions{LabelSelector: labelselector})
235+
if err != nil {
236+
return err
237+
}
238+
239+
for _, pod := range podList.Items {
240+
req := podsClient.GetLogs(pod.Name, &podLogOpts)
241+
podLogs, err := req.Stream(ctx)
242+
if err != nil {
243+
return err
244+
}
245+
defer podLogs.Close()
246+
247+
buf := new(bytes.Buffer)
248+
_, err = io.Copy(buf, podLogs)
249+
if err != nil {
250+
return err
251+
}
252+
str := buf.String()
253+
err = writeToFile(logDir, pod.Name + logExtension, str)
254+
if err != nil {
255+
return err
256+
}
257+
}
258+
return nil
259+
}
260+
261+
func writeToFile(dir, fileName, str string) error {
262+
if _, err := os.Stat(dir); os.IsNotExist(err) {
263+
// your dir does not exist
264+
os.MkdirAll(dir, 0666)
265+
}
266+
// open output file
267+
f, err := os.Create(dir + fileName)
268+
if err != nil {
269+
return err
270+
}
271+
// close fo on exit and check for its returned error
272+
defer func() {
273+
if err := f.Close(); err != nil {
274+
panic(err)
275+
}
276+
}()
277+
278+
// If write went ok then err is nil
279+
_, err = f.WriteString(str)
280+
return err
281+
}

‎tools/acncli/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,4 @@ ENV AZURE_CNI_TENANCY=singletenancy
1919
ENV AZURE_CNI_IPAM=azure-cns
2020
ENV AZURE_CNI_MODE=transparent
2121

22-
CMD ["./acn", "manager", "-f"]
22+
ENTRYPOINT ["./acn", "cni", "manager", "--follow", "--mode", "transparent"]

‎tools/acncli/api/constants.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ var (
8282
FlagTenancy: Singletenancy,
8383
FlagIPAM: AzureVNETIPAM,
8484
FlagExempt: AzureTelemetryBin + "," + AzureTelemetryConfig,
85-
FlagMode: Bridge,
85+
FlagMode: Transparent,
8686
FlagTarget: Local,
8787
FlagBinDirectory: DefaultBinDirLinux,
8888
FlagConflistDirectory: DefaultConflistDirLinux,

0 commit comments

Comments
 (0)