Skip to content

Commit 9df5fe3

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #258 from davidz627/feature/migrationConfig
Add test config for testing CSI Migration with PD Driver
2 parents 1778a00 + 63a0f21 commit 9df5fe3

File tree

6 files changed

+127
-25
lines changed

6 files changed

+127
-25
lines changed

‎.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ test/k8s-integration/src/
1414
# Test binary, build with `go test -c`
1515
*.test
1616

17+
# Kubernetes Integration Test YAML
18+
test/k8s-integration/config/test-config.yaml
19+
1720
# Output of the go coverage tool, specifically when used with LiteIDE
1821
*.out
1922

‎test/k8s-integration/main.go

Lines changed: 60 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -31,20 +31,29 @@ import (
3131
)
3232

3333
var (
34-
teardownCluster = flag.Bool("teardown-cluster", true, "teardown the cluster after the e2e test")
35-
teardownDriver = flag.Bool("teardown-driver", true, "teardown the driver after the e2e test")
36-
bringupCluster = flag.Bool("bringup-cluster", true, "build kubernetes and bringup a cluster")
37-
stagingImage = flag.String("staging-image", "", "name of image to stage to")
38-
kubeVersion = flag.String("kube-version", "master", "version of Kubernetes to download and use")
39-
inProw = flag.Bool("run-in-prow", false, "is the test running in PROW")
40-
saFile = flag.String("service-account-file", "", "path of service account file")
41-
deployOverlayName = flag.String("deploy-overlay-name", "", "which kustomize overlay to deploy the driver with")
42-
localK8sDir = flag.String("local-k8s-dir", "", "local kubernetes/kubernetes directory to run e2e tests from")
43-
doDriverBuild = flag.Bool("do-driver-build", true, "building the driver from source")
34+
// Kubernetes cluster flags
35+
teardownCluster = flag.Bool("teardown-cluster", true, "teardown the cluster after the e2e test")
36+
teardownDriver = flag.Bool("teardown-driver", true, "teardown the driver after the e2e test")
37+
bringupCluster = flag.Bool("bringup-cluster", true, "build kubernetes and bringup a cluster")
38+
gceZone = flag.String("gce-zone", "", "zone that the gce k8s cluster is created/found in")
39+
kubeVersion = flag.String("kube-version", "master", "version of Kubernetes to download and use")
40+
kubeFeatureGates = flag.String("kube-feature-gates", "", "feature gates to set on new kubernetes cluster")
41+
localK8sDir = flag.String("local-k8s-dir", "", "local kubernetes/kubernetes directory to run e2e tests from")
42+
43+
// Test infrastructure flags
4444
boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve")
4545
storageClassFile = flag.String("storageclass-file", "", "name of storageclass yaml file to use for test relative to test/k8s-integration/config")
46-
kubeFeatureGates = flag.String("kube-feature-gates", "", "feature gates to set on new kubernetes cluster")
47-
testFocus = flag.String("test-focus", "", "test focus for Kubernetes e2e")
46+
inProw = flag.Bool("run-in-prow", false, "is the test running in PROW")
47+
48+
// Driver flags
49+
stagingImage = flag.String("staging-image", "", "name of image to stage to")
50+
saFile = flag.String("service-account-file", "", "path of service account file")
51+
deployOverlayName = flag.String("deploy-overlay-name", "", "which kustomize overlay to deploy the driver with")
52+
doDriverBuild = flag.Bool("do-driver-build", true, "building the driver from source")
53+
54+
// Test flags
55+
migrationTest = flag.Bool("migration-test", false, "sets the flag on the e2e binary signalling migration")
56+
testFocus = flag.String("test-focus", "", "test focus for Kubernetes e2e")
4857
)
4958

5059
const (
@@ -70,8 +79,12 @@ func main() {
7079
glog.Fatalf("deploy-overlay-name is a required flag")
7180
}
7281

73-
if len(*storageClassFile) == 0 {
74-
glog.Fatalf("storageclass-file is a required flag")
82+
if len(*storageClassFile) == 0 && !*migrationTest {
83+
glog.Fatalf("One of storageclass-file and migration-test must be set")
84+
}
85+
86+
if len(*storageClassFile) != 0 && *migrationTest {
87+
glog.Fatalf("storage-class-file and migration-test cannot both be set")
7588
}
7689

7790
if !*bringupCluster && len(*kubeFeatureGates) > 0 {
@@ -82,6 +95,10 @@ func main() {
8295
glog.Fatalf("test-focus is a required flag")
8396
}
8497

98+
if len(*gceZone) == 0 {
99+
glog.Fatalf("gce-zone is a required flag")
100+
}
101+
85102
err := handle()
86103
if err != nil {
87104
glog.Fatalf("Failed to run integration test: %v", err)
@@ -179,7 +196,7 @@ func handle() error {
179196
glog.V(4).Infof("Set Kubernetes feature gates: %v", *kubeFeatureGates)
180197
}
181198

182-
err = clusterUp(k8sDir)
199+
err = clusterUp(k8sDir, *gceZone)
183200
if err != nil {
184201
return fmt.Errorf("failed to cluster up: %v", err)
185202
}
@@ -210,7 +227,15 @@ func handle() error {
210227
if len(*localK8sDir) != 0 {
211228
k8sDir = *localK8sDir
212229
}
213-
err = runTests(pkgDir, k8sDir, *storageClassFile, *testFocus)
230+
231+
if len(*storageClassFile) != 0 {
232+
err = runCSITests(pkgDir, k8sDir, *testFocus, *storageClassFile, *gceZone)
233+
} else if *migrationTest {
234+
err = runMigrationTests(pkgDir, k8sDir, *testFocus, *gceZone)
235+
} else {
236+
return fmt.Errorf("Did not run either CSI or Migration test")
237+
}
238+
214239
if err != nil {
215240
return fmt.Errorf("failed to run tests: %v", err)
216241
}
@@ -231,13 +256,21 @@ func setEnvProject(project string) error {
231256
return nil
232257
}
233258

234-
func runTests(pkgDir, k8sDir, storageClassFile, testFocus string) error {
259+
func runMigrationTests(pkgDir, k8sDir, testFocus, gceZone string) error {
260+
return runTestsWithConfig(pkgDir, k8sDir, gceZone, testFocus, "-storage.migratedPlugins=kubernetes.io/gce-pd")
261+
}
262+
263+
func runCSITests(pkgDir, k8sDir, testFocus, storageClassFile, gceZone string) error {
235264
testDriverConfigFile, err := generateDriverConfigFile(pkgDir, storageClassFile)
236265
if err != nil {
237266
return err
238267
}
268+
testConfigArg := fmt.Sprintf("-storage.testdriver=%s", testDriverConfigFile)
269+
return runTestsWithConfig(pkgDir, k8sDir, gceZone, testFocus, testConfigArg)
270+
}
239271

240-
err = os.Chdir(k8sDir)
272+
func runTestsWithConfig(pkgDir, k8sDir, gceZone, testFocus, testConfigArg string) error {
273+
err := os.Chdir(k8sDir)
241274
if err != nil {
242275
return err
243276
}
@@ -248,7 +281,6 @@ func runTests(pkgDir, k8sDir, storageClassFile, testFocus string) error {
248281
artifactsDir, _ := os.LookupEnv("ARTIFACTS")
249282
reportArg := fmt.Sprintf("-report-dir=%s", artifactsDir)
250283

251-
driverConfigArg := fmt.Sprintf("-storage.testdriver=%s", testDriverConfigFile)
252284
testFocusArg := fmt.Sprintf("-focus=%s", testFocus)
253285

254286
cmd := exec.Command(filepath.Join(k8sBuildBinDir, "ginkgo"),
@@ -258,7 +290,9 @@ func runTests(pkgDir, k8sDir, storageClassFile, testFocus string) error {
258290
filepath.Join(k8sBuildBinDir, "e2e.test"),
259291
"--",
260292
reportArg,
261-
driverConfigArg)
293+
"-provider=gce",
294+
fmt.Sprintf("-gce-zone=%s", gceZone),
295+
testConfigArg)
262296

263297
err = runCommand("Running Tests", cmd)
264298
if err != nil {
@@ -306,9 +340,13 @@ func buildKubernetes(k8sDir string) error {
306340
return nil
307341
}
308342

309-
func clusterUp(k8sDir string) error {
343+
func clusterUp(k8sDir, gceZone string) error {
344+
err := os.Setenv("KUBE_GCE_ZONE", gceZone)
345+
if err != nil {
346+
return err
347+
}
310348
cmd := exec.Command(filepath.Join(k8sDir, "hack", "e2e-internal", "e2e-up.sh"))
311-
err := runCommand("Starting E2E Cluster", cmd)
349+
err = runCommand("Starting E2E Cluster", cmd)
312350
if err != nil {
313351
return fmt.Errorf("failed to bring up kubernetes e2e cluster: %v", err)
314352
}

‎test/run-k8s-integration-local.sh

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,17 @@ ensure_var GCE_PD_CSI_STAGING_IMAGE
1010
ensure_var GCE_PD_SA_DIR
1111

1212
make -C ${PKGDIR} test-k8s-integration
13-
${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json --deploy-overlay-name=dev --storageclass-file=sc-standard.yaml --test-focus="External.Storage"
13+
14+
${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false \
15+
--staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
16+
--deploy-overlay-name=dev --storageclass-file=sc-standard.yaml \
17+
--test-focus="External.Storage" --gce-zone="us-central1-b"
1418

1519
# This version of the command does not build the driver or K8s, points to a
1620
# local K8s repo to get the e2e.test binary, and does not bring up or down the cluster
1721
#
18-
# ${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json --deploy-overlay-name=dev --bringup-cluster=false --teardown-cluster=false --local-k8s-dir=$KTOP --storageclass-file=sc-standard.yaml --do-driver-build=false
22+
# ${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false \
23+
# --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
24+
# --deploy-overlay-name=dev --bringup-cluster=false --teardown-cluster=false --local-k8s-dir=$KTOP \
25+
# --storageclass-file=sc-standard.yaml --do-driver-build=false --test-focus="External.Storage.*multiVolume" \
26+
# --gce-zone="us-central1-b"

‎test/run-k8s-integration-migration-local.sh

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/bash
2+
3+
set -o nounset
4+
set -o errexit
5+
6+
readonly PKGDIR=${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
7+
source "${PKGDIR}/deploy/common.sh"
8+
9+
ensure_var GCE_PD_CSI_STAGING_IMAGE
10+
ensure_var GCE_PD_SA_DIR
11+
12+
make -C ${PKGDIR} test-k8s-integration
13+
${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false \
14+
--staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
15+
--deploy-overlay-name=dev --test-focus="\[sig-storage\]\sIn-tree\sVolumes\s\[Driver:\sgcepd\].*" \
16+
--kube-feature-gates="CSIMigration=true,CSIMigrationGCE=true" --migration-test=true --gce-zone="us-central1-b"
17+
18+
# This version of the command does not build the driver or K8s, points to a
19+
# local K8s repo to get the e2e.test binary, and does not bring up or down the cluster
20+
#
21+
# ensure_var GCE_PD_ZONE
22+
# ${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false \
23+
# --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
24+
# --deploy-overlay-name=dev --test-focus="\[sig-storage\]\sIn-tree\sVolumes\s\[Driver:\sgcepd\].*" \
25+
# --bringup-cluster=false --teardown-cluster=false --local-k8s-dir=$KTOP --migration-test=true \
26+
# --do-driver-build=false --gce-zone=${GCE_PD_ZONE}

‎test/run-k8s-integration-migration.sh

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/bin/bash
2+
3+
# Optional environment variables
4+
# GCE_PD_OVERLAY_NAME: which Kustomize overlay to deploy with
5+
# GCE_PD_DO_DRIVER_BUILD: if set, don't build the driver from source and just
6+
# use the driver version from the overlay
7+
# GCE_PD_BOSKOS_RESOURCE_TYPE: name of the boskos resource type to reserve
8+
9+
set -o nounset
10+
set -o errexit
11+
12+
readonly PKGDIR=${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
13+
readonly overlay_name="${GCE_PD_OVERLAY_NAME:-stable}"
14+
readonly boskos_resource_type="${GCE_PD_BOSKOS_RESOURCE_TYPE:-gce-project}"
15+
readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}"
16+
export GCE_PD_VERBOSITY=9
17+
18+
make -C ${PKGDIR} test-k8s-integration
19+
${PKGDIR}/bin/k8s-integration-test --kube-version=${GCE_PD_KUBE_VERSION:-master} \
20+
--kube-feature-gates="CSIMigration=true,CSIMigrationGCE=true" --run-in-prow=true \
21+
--deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
22+
--do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} \
23+
--migration-test=true --test-focus="\[sig-storage\]\sIn-tree\sVolumes\s\[Driver:\sgcepd\].*" \
24+
--gce-zone="us-central1-b"

‎test/run-k8s-integration.sh

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,7 @@ readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}"
1616
export GCE_PD_VERBOSITY=9
1717

1818
make -C ${PKGDIR} test-k8s-integration
19-
${PKGDIR}/bin/k8s-integration-test --kube-version=${GCE_PD_KUBE_VERSION:-master} --run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} --do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} --storageclass-file=sc-standard.yaml --test-focus="External.Storage"
19+
${PKGDIR}/bin/k8s-integration-test --kube-version=${GCE_PD_KUBE_VERSION:-master} \
20+
--run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
21+
--do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} \
22+
--storageclass-file=sc-standard.yaml --test-focus="External.Storage" --gce-zone="us-central1-b"

0 commit comments

Comments
 (0)