diff --git a/Makefile b/Makefile index 511bdd3dc..a7021fe34 100644 --- a/Makefile +++ b/Makefile @@ -114,8 +114,6 @@ test/e2e/k3d-%: test/e2e/testrunner-eks: KUBECONFIG=$(TMPDIR)/$(CLUSTER_NAME)-eks.kubeconfig.yaml bazel-bin/hack/bin/kubectl create -f hack/eks-storageclass.yaml bazel test --stamp //e2e/upgrades/... --action_env=KUBECONFIG=$(TMPDIR)/$(CLUSTER_NAME)-eks.kubeconfig.yaml - bazel test --stamp //e2e/create/... --action_env=KUBECONFIG=$(TMPDIR)/$(CLUSTER_NAME)-eks.kubeconfig.yaml - bazel test --stamp //e2e/decommission/... --action_env=KUBECONFIG=$(TMPDIR)/$(CLUSTER_NAME)-eks.kubeconfig.yaml # Use this target to run e2e tests with a eks cluster. # This target uses kubetest2 to start a eks k8s cluster and runs the e2e tests @@ -143,9 +141,6 @@ test/e2e/testrunner-gke: #bazel run --stamp --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 \ # //manifests:install_operator.apply bazel test --stamp //e2e/upgrades/... - bazel test --stamp //e2e/create/... - bazel test --stamp --test_arg=--pvc=true //e2e/pvcresize/... - bazel test --stamp //e2e/decommission/... # Use this target to run e2e tests with a gke cluster. # This target uses kubetest2 to start a gke k8s cluster and runs the e2e tests @@ -171,8 +166,6 @@ test/e2e/gke: .PHONY: test/e2e/testrunner-openshift test/e2e/testrunner-openshift: bazel test --stamp //e2e/upgrades/... --action_env=KUBECONFIG=$(HOME)/openshift-$(CLUSTER_NAME)/auth/kubeconfig - bazel test --stamp //e2e/create/... --action_env=KUBECONFIG=$(HOME)/openshift-$(CLUSTER_NAME)/auth/kubeconfig - bazel test --stamp //e2e/decommission/... --action_env=KUBECONFIG=$(HOME)/openshift-$(CLUSTER_NAME)/auth/kubeconfig # Use this target to run e2e tests with a openshift cluster. # This target uses kubetest2 to start a openshift cluster and runs the e2e tests diff --git a/hack/bin/BUILD.bazel b/hack/bin/BUILD.bazel index 9fc2bc241..ccd99c271 100644 --- a/hack/bin/BUILD.bazel +++ b/hack/bin/BUILD.bazel @@ -143,38 +143,16 @@ genrule( genrule( name = "fetch_kubetest2", srcs = select({ - ":m1": ["@kubetest2_darwin//file"], - ":darwin": ["@kubetest2_darwin//file"], - ":k8": ["@kubetest2_linux//file"], + ":m1": ["@kubetest2_darwin//:files"], + ":darwin": ["@kubetest2_darwin//:files"], + ":k8": ["@kubetest2_linux//:files"], }), - outs = ["kubetest2"], - cmd = "cp $(SRCS) $@", - visibility = ["//visibility:public"], -) - -# fetch_kubetest2_gke rules to fetch the binary for kubetest2_gke used on e2e tests that are saved on a google cloud bucket -genrule( - name = "fetch_kubetest2_gke", - srcs = select({ - ":m1": ["@kubetest2_gke_darwin//file"], - ":darwin": ["@kubetest2_gke_darwin//file"], - ":k8": ["@kubetest2_gke_linux//file"], - }), - outs = ["kubetest2-gke"], - cmd = "cp $(SRCS) $@", - visibility = ["//visibility:public"], -) - -# fetch_kubetest2_exe`rules to fetch the binary for kubetest2_exe used on e2e tests that are saved on a google cloud bucket -genrule( - name = "fetch_kubetest2_exe", - srcs = select({ - ":m1": ["@kubetest2_exe_darwin//file"], - ":darwin": ["@kubetest2_exe_darwin//file"], - ":k8": ["@kubetest2_exe_linux//file"], - }), - outs = ["kubetest2-tester-exec"], - cmd = "cp $(SRCS) $@", + outs = [ + "kubetest2", + "kubetest2-gke", + "kubetest2-tester-exec", + ], + cmd = "cp $(SRCS) $(@D)", visibility = ["//visibility:public"], ) diff --git a/hack/bin/deps.bzl b/hack/bin/deps.bzl index 45bdebd20..8d8e5abf4 100644 --- a/hack/bin/deps.bzl +++ b/hack/bin/deps.bzl @@ -64,8 +64,6 @@ def install(): install_k3d() install_kubetest2() install_kubetest2_aws() - install_kubetest2_exe() - install_kubetest2_gke() install_kustomize() install_oc() install_operator_sdk() @@ -266,54 +264,31 @@ filegroup( ## Fetch kubetest2 binary used during e2e tests def install_kubetest2(): - # install kubetest2 binary - http_file( - name = "kubetest2_darwin", - executable = 1, - sha256 = "5b20aadd05eca47dead180a7c8296d75e81c184aabf182d4a41ef96597db543d", - urls = ["https://storage.googleapis.com/crdb-bazel-artifacts/osx/kubetest2"], - ) + url = "https://github.com/pseudomuto/kubetest2/releases/download/v0.1.0/kubetest2_0.1.0_{}_amd64.tar.gz" + content = """ +filegroup( + name = "files", + srcs = [ + "kubetest2", + "kubetest2-gke", + "kubetest2-tester-exec", + ], + visibility = ["//visibility:public"], +) +""" - http_file( + http_archive( name = "kubetest2_linux", - executable = 1, - sha256 = "7f0b05654fa43ca1c607db297b5f3a775f65eea90355bb6b10137a7fffff5e1a", - urls = ["https://storage.googleapis.com/crdb-bazel-artifacts/linux/kubetest2"], + sha256 = "04319fb9cef1a5370548c47a0c65fa8d4f85a1b6d4417b318a93125d72590d1b", + urls = [url.format("linux")], + build_file_content = content, ) -## Fetch kubetest2-gke binary used during e2e tests -def install_kubetest2_gke(): - # install kubetest2-gke binary - # TODO osx support - http_file( - name = "kubetest2_gke_darwin", - executable = 1, - sha256 = "a1cbe02f61931dbe6c8d1662442f42cb538c81e4ec8cdd40f548f0e05cbd55a7", - urls = ["https://storage.googleapis.com/crdb-bazel-artifacts/osx/kubetest2-gke"], - ) - - http_file( - name = "kubetest2_gke_linux", - executable = 1, - sha256 = "9ac658234efc7f59968888662dd2d21908587789f6b812392ac5b6766b17c0b4", - urls = ["https://storage.googleapis.com/crdb-bazel-artifacts/linux/kubetest2-gke"], - ) -## Fetch kubetest2-tester-exe binary used during e2e tests -def install_kubetest2_exe(): - # install kubetest2-exe binary - # TODO osx support - http_file( - name = "kubetest2_exe_darwin", - executable = 1, - sha256 = "818690cb55590440e163b18dd139c8a8714df9480f869bafe19eb344047cf37c", - urls = ["https://storage.googleapis.com/crdb-bazel-artifacts/osx/kubetest2-tester-exec"], - ) - - http_file( - name = "kubetest2_exe_linux", - executable = 1, - sha256 = "4483f40f48b98e8a6aa41f58bfdf1f2787066a4e1ad1343e4281892aa1326736", - urls = ["https://storage.googleapis.com/crdb-bazel-artifacts/linux/kubetest2-tester-exec"], + http_archive( + name = "kubetest2_darwin", + sha256 = "972d639e9548f296c8470c3418630baa4176aabc2b2e5d47d62b9eb8ee2b91cf", + urls = [url.format("darwin")], + build_file_content = content, ) ## Fetch operator-sdk used on generating csv diff --git a/pkg/resource/BUILD.bazel b/pkg/resource/BUILD.bazel index f9d459125..34d3b85e1 100644 --- a/pkg/resource/BUILD.bazel +++ b/pkg/resource/BUILD.bazel @@ -57,8 +57,8 @@ go_library( go_test( name = "go_default_test", srcs = [ - "cluster_test.go", "certificate_test.go", + "cluster_test.go", "discovery_service_test.go", "pod_distruption_budget_test.go", "public_service_test.go", diff --git a/pkg/testutil/env/BUILD.bazel b/pkg/testutil/env/BUILD.bazel index 5adb680d5..3af829c90 100644 --- a/pkg/testutil/env/BUILD.bazel +++ b/pkg/testutil/env/BUILD.bazel @@ -24,6 +24,7 @@ go_library( "//apis/v1alpha1:go_default_library", "//pkg/client/clientset/versioned:go_default_library", "//pkg/kube:go_default_library", + "@com_github_cenkalti_backoff//:go_default_library", "@com_github_cockroachdb_errors//:go_default_library", "@com_github_stretchr_testify//require:go_default_library", "@io_k8s_api//core/v1:go_default_library", diff --git a/pkg/testutil/env/env.go b/pkg/testutil/env/env.go index bb606d6bf..c202dcb2a 100644 --- a/pkg/testutil/env/env.go +++ b/pkg/testutil/env/env.go @@ -20,12 +20,14 @@ import ( "flag" "fmt" "os" + "time" + "github.com/cenkalti/backoff" api "github.com/cockroachdb/cockroach-operator/apis/v1alpha1" customClient "github.com/cockroachdb/cockroach-operator/pkg/client/clientset/versioned" "github.com/cockroachdb/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - apiruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" @@ -40,7 +42,7 @@ import ( // environments. var testBinaries = flag.String("binaries", "hack/bin", "") -func NewEnv(builder apiruntime.SchemeBuilder) *Env { +func NewEnv(builder runtime.SchemeBuilder) *Env { flag.Parse() // ensure hack/bin is added to the path and KUBEBUILDER_ASSETS @@ -48,7 +50,7 @@ func NewEnv(builder apiruntime.SchemeBuilder) *Env { os.Setenv("KUBEBUILDER_ASSETS", p) PrependToPath(p) - scheme := apiruntime.NewScheme() + scheme := runtime.NewScheme() if err := kscheme.AddToScheme(scheme); err != nil { panic(err) @@ -76,7 +78,7 @@ func NewEnv(builder apiruntime.SchemeBuilder) *Env { type Env struct { envtest.Environment - Scheme *apiruntime.Scheme + Scheme *runtime.Scheme } func (env *Env) Start() *ActiveEnv { @@ -129,7 +131,7 @@ func (k k8s) namespaceableResource(gvr schema.GroupVersionResource) dynamic.Name type ActiveEnv struct { *k8s - scheme *apiruntime.Scheme + scheme *runtime.Scheme resources []schema.GroupVersionResource } @@ -139,8 +141,19 @@ func CreateActiveEnvForTest() *Env { } func loadResources(k *k8s) ([]schema.GroupVersionResource, error) { - lists, err := k.DiscoveryClient.ServerPreferredResources() - if err != nil { + b := backoff.NewExponentialBackOff() + b.MaxElapsedTime = 60 * time.Second + lists := make([]*metav1.APIResourceList, 0) + + // GKE was having issues fetching api resources. Giving it some time + // (tested via time.Sleep(10*time.Second)) seemed to resolve this issue. + // Rather than a sleep call, we should retry until it's ready or ultimately + // fails. + if err := backoff.Retry(func() error { + var err error + lists, err = k.DiscoveryClient.ServerPreferredResources() + return err + }, b); err != nil { return nil, errors.Wrap(err, "failed to fetch preferred server resource") }