Skip to content

Commit b3dc35f

Browse files
openshift-merge-bot[bot]openshift-merge-bot[bot]
authored
Merge pull request #1338 from vr4manta/test-ext-framework
SPLAT-2060: Create e2e tests for vSphere Data Disk feature
2 parents 56773e1 + e117e81 commit b3dc35f

File tree

4,550 files changed

+1054827
-130
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

4,550 files changed

+1054827
-130
lines changed

‎Dockerfile

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
FROM registry.ci.openshift.org/openshift/release:golang-1.23 AS builder
22
WORKDIR /go/src/github.com/openshift/machine-api-operator
33
COPY . .
4-
RUN NO_DOCKER=1 make build
4+
RUN NO_DOCKER=1 make build && \
5+
mkdir -p /tmp/build && \
6+
cp /go/src/github.com/openshift/machine-api-operator/bin/machine-api-tests-ext /tmp/build/machine-api-tests-ext && \
7+
gzip /tmp/build/machine-api-tests-ext
58

69
FROM registry.ci.openshift.org/openshift/origin-v4.0:base
710
COPY --from=builder /go/src/github.com/openshift/machine-api-operator/install manifests
@@ -10,5 +13,8 @@ COPY --from=builder /go/src/github.com/openshift/machine-api-operator/bin/nodeli
1013
COPY --from=builder /go/src/github.com/openshift/machine-api-operator/bin/machine-healthcheck .
1114
COPY --from=builder /go/src/github.com/openshift/machine-api-operator/bin/machineset ./machineset-controller
1215
COPY --from=builder /go/src/github.com/openshift/machine-api-operator/bin/vsphere ./machine-controller-manager
16+
COPY --from=builder /tmp/build/machine-api-tests-ext.gz .
1317

14-
LABEL io.openshift.release.operator true
18+
LABEL io.k8s.display-name="OpenShift Machine API Operator" \
19+
io.openshift.release.operator=true \
20+
io.openshift.tags="openshift,tests,e2e,e2e-extension"

‎Dockerfile.rhel

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.23-openshift-4.19 AS builder
22
WORKDIR /go/src/github.com/openshift/machine-api-operator
33
COPY . .
4-
RUN NO_DOCKER=1 make build
4+
RUN NO_DOCKER=1 make build && \
5+
mkdir -p /tmp/build && \
6+
cp /go/src/github.com/openshift/machine-api-operator/bin/machine-api-tests-ext /tmp/build/machine-api-tests-ext && \
7+
gzip /tmp/build/machine-api-tests-ext
58

69
FROM registry.ci.openshift.org/ocp/4.19:base-rhel9
710
COPY --from=builder /go/src/github.com/openshift/machine-api-operator/install manifests
@@ -10,5 +13,8 @@ COPY --from=builder /go/src/github.com/openshift/machine-api-operator/bin/nodeli
1013
COPY --from=builder /go/src/github.com/openshift/machine-api-operator/bin/machine-healthcheck .
1114
COPY --from=builder /go/src/github.com/openshift/machine-api-operator/bin/machineset ./machineset-controller
1215
COPY --from=builder /go/src/github.com/openshift/machine-api-operator/bin/vsphere ./machine-controller-manager
16+
COPY --from=builder /tmp/build/machine-api-tests-ext.gz .
1317

14-
LABEL io.openshift.release.operator true
18+
LABEL io.k8s.display-name="OpenShift Machine API Operator" \
19+
io.openshift.release.operator=true \
20+
io.openshift.tags="openshift,tests,e2e,e2e-extension"

‎Makefile

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ vendor:
6565
check: verify-crds-sync lint fmt vet test ## Run code validations
6666

6767
.PHONY: build
68-
build: machine-api-operator nodelink-controller machine-healthcheck machineset vsphere ## Build binaries
68+
build: machine-api-operator nodelink-controller machine-healthcheck machineset vsphere machine-api-tests-ext ## Build binaries
6969

7070
.PHONY: machine-api-operator
7171
machine-api-operator:
@@ -87,6 +87,10 @@ vsphere:
8787
machineset:
8888
$(DOCKER_CMD) ./hack/go-build.sh machineset
8989

90+
.PHONY: machine-api-tests-ext
91+
machine-api-tests-ext:
92+
$(DOCKER_CMD) ./hack/go-build.sh machine-api-tests-ext
93+
9094
.PHONY: test-e2e
9195
test-e2e: ## Run openshift specific e2e tests
9296
./hack/e2e.sh test-e2e

‎cmd/machine-api-tests-ext/main.go

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
package main
2+
3+
import (
4+
"flag"
5+
"os"
6+
"regexp"
7+
"strings"
8+
9+
"github.com/openshift-eng/openshift-tests-extension/pkg/cmd"
10+
e "github.com/openshift-eng/openshift-tests-extension/pkg/extension"
11+
"github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
12+
g "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo"
13+
"github.com/spf13/cobra"
14+
"github.com/spf13/pflag"
15+
utilflag "k8s.io/component-base/cli/flag"
16+
"k8s.io/component-base/logs"
17+
"k8s.io/kubernetes/test/e2e/framework"
18+
19+
// If using ginkgo, import your tests here
20+
_ "github.com/openshift/machine-api-operator/test/e2e/vsphere"
21+
)
22+
23+
func main() {
24+
logs.InitLogs()
25+
defer logs.FlushLogs()
26+
pflag.CommandLine.SetNormalizeFunc(utilflag.WordSepNormalizeFunc)
27+
28+
// These flags are used to pull in the default values to test context - required
29+
// so tests run correctly, even if the underlying flags aren't used.
30+
framework.RegisterCommonFlags(flag.CommandLine)
31+
framework.RegisterClusterFlags(flag.CommandLine)
32+
33+
// Create our registry of openshift-tests extensions
34+
extensionRegistry := e.NewRegistry()
35+
kubeTestsExtension := e.NewExtension("openshift", "payload", "machine-api-operator")
36+
extensionRegistry.Register(kubeTestsExtension)
37+
38+
// Carve up the kube tests into our openshift suites...
39+
kubeTestsExtension.AddSuite(e.Suite{
40+
Name: "mao/conformance/parallel",
41+
Parents: []string{
42+
"openshift/conformance/parallel",
43+
},
44+
Qualifiers: []string{`!labels.exists(l, l == "Serial") && labels.exists(l, l == "Conformance")`},
45+
})
46+
47+
kubeTestsExtension.AddSuite(e.Suite{
48+
Name: "mao/conformance/serial",
49+
Parents: []string{
50+
"openshift/conformance/serial",
51+
},
52+
Qualifiers: []string{`labels.exists(l, l == "Serial") && labels.exists(l, l == "Conformance")`},
53+
})
54+
55+
// Build our specs from ginkgo
56+
specs, err := g.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite()
57+
if err != nil {
58+
panic(err)
59+
}
60+
61+
// Initialization for kube ginkgo test framework needs to run before all tests execute
62+
specs.AddBeforeAll(func() {
63+
if err := initializeTestFramework(os.Getenv("TEST_PROVIDER")); err != nil {
64+
panic(err)
65+
}
66+
})
67+
68+
// Let's scan for tests with a platform label and create the rule for them such as [platform:vsphere]
69+
foundPlatforms := make(map[string]string)
70+
for _, test := range specs.Select(extensiontests.NameContains("[platform:")).Names() {
71+
re := regexp.MustCompile(`\[platform:[a-z]*]`)
72+
match := re.FindStringSubmatch(test)
73+
for _, platformDef := range match {
74+
if _, ok := foundPlatforms[platformDef]; !ok {
75+
platform := platformDef[strings.Index(platformDef, ":")+1 : len(platformDef)-1]
76+
foundPlatforms[platformDef] = platform
77+
specs.Select(extensiontests.NameContains(platformDef)).
78+
Include(extensiontests.PlatformEquals(platform))
79+
}
80+
}
81+
82+
}
83+
84+
kubeTestsExtension.AddSpecs(specs)
85+
86+
// Cobra stuff
87+
root := &cobra.Command{
88+
Long: "Machine API Operator tests extension for OpenShift",
89+
}
90+
91+
root.AddCommand(cmd.DefaultExtensionCommands(extensionRegistry)...)
92+
93+
if err := func() error {
94+
return root.Execute()
95+
}(); err != nil {
96+
os.Exit(1)
97+
}
98+
}

‎cmd/machine-api-tests-ext/provider.go

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"fmt"
7+
"os"
8+
"path/filepath"
9+
"strings"
10+
11+
"github.com/onsi/ginkgo/v2"
12+
"github.com/onsi/gomega"
13+
corev1 "k8s.io/api/core/v1"
14+
kclientset "k8s.io/client-go/kubernetes"
15+
"k8s.io/client-go/tools/clientcmd"
16+
"k8s.io/kubernetes/openshift-hack/e2e"
17+
conformancetestdata "k8s.io/kubernetes/test/conformance/testdata"
18+
"k8s.io/kubernetes/test/e2e/framework"
19+
"k8s.io/kubernetes/test/e2e/framework/testfiles"
20+
e2etestingmanifests "k8s.io/kubernetes/test/e2e/testing-manifests"
21+
testfixtures "k8s.io/kubernetes/test/fixtures"
22+
23+
// this appears to inexplicably auto-register global flags.
24+
_ "k8s.io/kubernetes/test/e2e/storage/drivers"
25+
26+
// these are loading important global flags that we need to get and set
27+
_ "k8s.io/kubernetes/test/e2e"
28+
_ "k8s.io/kubernetes/test/e2e/lifecycle"
29+
)
30+
31+
// copied directly from github.com/openshift/kubernetes/openshift-hack/cmd/k8s-tests-ext/provider.go
32+
// I attempted to use the clusterdiscovery.InitializeTestFramework in origin but it has too many additional parameters
33+
// that as an test-ext, I felt we shouldn't have to load all that. Hopefully origin's test-ext frameworks gets enhanced
34+
// to have a simple way to initialize all this w/o having to copy/pasta like the openshift/kubernetes project did.
35+
func initializeTestFramework(provider string) error {
36+
providerInfo := &ClusterConfiguration{}
37+
if err := json.Unmarshal([]byte(provider), &providerInfo); err != nil {
38+
return fmt.Errorf("provider must be a JSON object with the 'type' key at a minimum: %v", err)
39+
}
40+
if len(providerInfo.ProviderName) == 0 {
41+
return fmt.Errorf("provider must be a JSON object with the 'type' key")
42+
}
43+
config := &ClusterConfiguration{}
44+
if err := json.Unmarshal([]byte(provider), config); err != nil {
45+
return fmt.Errorf("provider must decode into the ClusterConfig object: %v", err)
46+
}
47+
48+
// update testContext with loaded config
49+
testContext := &framework.TestContext
50+
testContext.Provider = config.ProviderName
51+
testContext.CloudConfig = framework.CloudConfig{
52+
ProjectID: config.ProjectID,
53+
Region: config.Region,
54+
Zone: config.Zone,
55+
Zones: config.Zones,
56+
NumNodes: config.NumNodes,
57+
MultiMaster: config.MultiMaster,
58+
MultiZone: config.MultiZone,
59+
ConfigFile: config.ConfigFile,
60+
}
61+
testContext.AllowedNotReadyNodes = -1
62+
testContext.MinStartupPods = -1
63+
testContext.MaxNodesToGather = 0
64+
testContext.KubeConfig = os.Getenv("KUBECONFIG")
65+
66+
if ad := os.Getenv("ARTIFACT_DIR"); len(strings.TrimSpace(ad)) == 0 {
67+
if err := os.Setenv("ARTIFACT_DIR", filepath.Join(os.TempDir(), "artifacts")); err != nil {
68+
return fmt.Errorf("unable to set ARTIFACT_DIR: %v", err)
69+
}
70+
}
71+
72+
testContext.DeleteNamespace = os.Getenv("DELETE_NAMESPACE") != "false"
73+
testContext.VerifyServiceAccount = true
74+
testfiles.AddFileSource(e2etestingmanifests.GetE2ETestingManifestsFS())
75+
testfiles.AddFileSource(testfixtures.GetTestFixturesFS())
76+
testfiles.AddFileSource(conformancetestdata.GetConformanceTestdataFS())
77+
testContext.KubectlPath = "kubectl"
78+
testContext.KubeConfig = os.Getenv("KUBECONFIG")
79+
80+
// "debian" is used when not set. At least GlusterFS tests need "custom".
81+
// (There is no option for "rhel" or "centos".)
82+
testContext.NodeOSDistro = "custom"
83+
testContext.MasterOSDistro = "custom"
84+
85+
// load and set the host variable for kubectl
86+
clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: testContext.KubeConfig}, &clientcmd.ConfigOverrides{})
87+
cfg, err := clientConfig.ClientConfig()
88+
if err != nil {
89+
return err
90+
}
91+
testContext.Host = cfg.Host
92+
93+
// Ensure that Kube tests run privileged (like they do upstream)
94+
testContext.CreateTestingNS = func(ctx context.Context, baseName string, c kclientset.Interface, labels map[string]string) (*corev1.Namespace, error) {
95+
return e2e.CreateTestingNS(ctx, baseName, c, labels, true)
96+
}
97+
98+
gomega.RegisterFailHandler(ginkgo.Fail)
99+
100+
framework.AfterReadingAllFlags(testContext)
101+
testContext.DumpLogsOnFailure = true
102+
103+
// these constants are taken from kube e2e and used by tests
104+
testContext.IPFamily = "ipv4"
105+
if config.HasIPv6 && !config.HasIPv4 {
106+
testContext.IPFamily = "ipv6"
107+
}
108+
109+
testContext.ReportDir = os.Getenv("TEST_JUNIT_DIR")
110+
111+
return nil
112+
}

‎cmd/machine-api-tests-ext/types.go

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
package main
2+
3+
// copied directly from github.com/openshift/origin/test/extended/util/cluster/cluster.go
4+
type ClusterConfiguration struct {
5+
ProviderName string `json:"type"`
6+
7+
// These fields (and the "type" tag for ProviderName) chosen to match
8+
// upstream's e2e.CloudConfig.
9+
ProjectID string
10+
Region string
11+
Zone string
12+
NumNodes int
13+
MultiMaster bool
14+
MultiZone bool
15+
Zones []string
16+
ConfigFile string
17+
18+
// Disconnected is set for test jobs without external internet connectivity
19+
Disconnected bool
20+
21+
// SingleReplicaTopology is set for disabling disruptive tests or tests
22+
// that require high availability
23+
SingleReplicaTopology bool
24+
25+
// NetworkPlugin is the "official" plugin name
26+
NetworkPlugin string
27+
// NetworkPluginMode is an optional sub-identifier for the NetworkPlugin.
28+
// (Currently it is only used for OpenShiftSDN.)
29+
NetworkPluginMode string `json:",omitempty"`
30+
31+
// HasIPv4 and HasIPv6 determine whether IPv4-specific, IPv6-specific,
32+
// and dual-stack-specific tests are run
33+
HasIPv4 bool
34+
HasIPv6 bool
35+
36+
// HasSCTP determines whether SCTP connectivity tests can be run in the cluster
37+
HasSCTP bool
38+
39+
// IsProxied determines whether we are accessing the cluster through an HTTP proxy
40+
IsProxied bool
41+
42+
// IsIBMROKS determines whether the cluster is Managed IBM Cloud (ROKS)
43+
IsIBMROKS bool
44+
45+
// IsNoOptionalCapabilities indicates the cluster has no optional capabilities enabled
46+
HasNoOptionalCapabilities bool
47+
}

0 commit comments

Comments
 (0)