This commit adds e2e tests for go/v3-alpha and refactors

test code such that it is easier to add other plugin tests
going forward.

test/e2e/utils: add util to remove tool binaries from path

test/e2e/v3: refactor generation code so all plugins can be tested

test/e2e/v2: rename test files

*: clean up scripts

Author: estroz

Makefile

@@ -52,15 +52,12 @@ install: ## Build and install the binary with the current source code. Use it to
 
 .PHONY: generate
 generate: ## Update/generate all mock data. You should run this commands to update the mock data after your changes.
-    # If exists, rremove the controller-gen installed locally. It ensures that the version which will be used
-    # is the version defined in the Makefile scaffolded.
-	- rm -rf $(CONTROLLER_GEN_BIN_PATH)
 	make generate-testdata
 	go mod tidy
 
 .PHONY: generate-testdata
 generate-testdata: ## Update/generate the testdata in $GOPATH/src/sigs.k8s.io/kubebuilder
-	GO111MODULE=on ./generate_testdata.sh
+	./generate_testdata.sh
 
 .PHONY: lint
 lint: golangci-lint ## Run golangci lint checks

common.sh

@@ -55,6 +55,7 @@ cd "$base_dir" || {
 }
 
 go_workspace=''
+export GOPATH=${GOPATH:-$(go env GOPATH)}
 for p in ${GOPATH//:/ }; do
   if [[ $PWD/ = $p/* ]]; then
     go_workspace=$p

generate_testdata.sh

@@ -14,15 +14,12 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-set -e
-
 source common.sh
 
 build_kb() {
     go build -o ./bin/kubebuilder sigs.k8s.io/kubebuilder/v2/cmd
 }
 
-
 #
 # This function scaffolds test projects given a project name and
 # project-version.
@@ -32,15 +29,19 @@ scaffold_test_project() {
     local version=$2
     local plugin=${3:-""}
 
-    testdata_dir=$(pwd)/testdata
+    local testdata_dir=$(pwd)/testdata
     mkdir -p ./testdata/$project
     rm -rf ./testdata/$project/*
     pushd .
     cd testdata/$project
-    kb=$testdata_dir/../bin/kubebuilder
+    local kb=$testdata_dir/../bin/kubebuilder
     # Set "--plugins $plugin" if $plugin is not null.
     local plugin_flag="${plugin:+--plugins $plugin}"
 
+    # Remove tool binaries so the correct versions are used for each plugin version.
+    rm -f "$(which controller-gen)"
+    rm -f "$(which kustomize)"
+
     if [ $version == "2" ] || [ $version == "3-alpha" ]; then
         if [ $version == "2" ] && [ -n "$plugin_flag" ]; then
           echo "--plugins flag may not be set for project versions less than 3"
@@ -50,8 +51,6 @@ scaffold_test_project() {
         header_text "Starting to generate projects with version $version"
         header_text "Generating $project"
 
-        export GO111MODULE=on
-        export PATH=$PATH:$(go env GOPATH)/bin
         go mod init sigs.k8s.io/kubebuilder/testdata/$project  # our repo autodetection will traverse up to the kb module if we don't do this
 
         header_text "initializing $project ..."
@@ -104,6 +103,9 @@ scaffold_test_project() {
 
 set -e
 
+export GO111MODULE=on
+export PATH="$PATH:$(go env GOPATH)/bin"
+
 build_kb
 scaffold_test_project project-v2 2
 scaffold_test_project project-v2-multigroup 2

scripts/setup.sh

@@ -26,7 +26,3 @@ export GO111MODULE=on
 
 # You can use --image flag to specify the cluster version you want, e.g --image=kindest/node:v1.13.6, the supported version are listed at https://hub.docker.com/r/kindest/node/tags
 kind create cluster -v 4 --name $KIND_NAME --retain --wait=1m --config test/kind-config.yaml --image=kindest/node:$K8S_VERSION
-
-# setup the go modules required
-go get sigs.k8s.io/controller-tools/cmd/[email protected]
-go get sigs.k8s.io/kustomize/kustomize/[email protected]

test.sh

@@ -90,8 +90,10 @@ EOF
 
 
 function test_project {
-  project_dir=$1
-  version=$2
+  local project_dir=$1
+  local version=$2
+  rm -f "$(which controller-gen)"
+  rm -f "$(which kustomize)"
   header_text "performing tests in dir $project_dir for project version v$version"
   cd testdata/$project_dir
   make all test
@@ -143,13 +145,13 @@ export GO111MODULE=on
 go test -race -v ./cmd/... ./pkg/... ./plugins/...
 
 # test project v2
-GO111MODULE=on test_project project-v2 2
-GO111MODULE=on test_project project-v2-multigroup 2
-GO111MODULE=on test_project project-v2-addon 2
+test_project project-v2 2
+test_project project-v2-multigroup 2
+test_project project-v2-addon 2
 
 # test project v3
-GO111MODULE=on test_project project-v3 3-alpha
-GO111MODULE=on test_project project-v3-multigroup 3-alpha
-GO111MODULE=on test_project project-v3-addon 3-alpha
+test_project project-v3 3-alpha
+test_project project-v3-multigroup 3-alpha
+test_project project-v3-addon 3-alpha
 
 exit $rc

test/e2e/utils/test_context.go

@@ -82,8 +82,18 @@ func NewTestContext(binaryName string, env ...string) (*TestContext, error) {
 	}, nil
 }
 
-// Prepare prepare a work directory for testing
+// Prepare prepares the test environment.
 func (t *TestContext) Prepare() error {
+	// Remove tools used by projects in the environment so the correct version is downloaded for each test.
+	fmt.Fprintf(GinkgoWriter, "cleaning up tools")
+	for _, toolName := range []string{"controller-gen", "kustomize"} {
+		if toolPath, err := exec.LookPath(toolName); err == nil {
+			if err := os.RemoveAll(toolPath); err != nil {
+				return err
+			}
+		}
+	}
+
 	fmt.Fprintf(GinkgoWriter, "preparing testing directory: %s\n", t.Dir)
 	return os.MkdirAll(t.Dir, 0755)
 }

test/e2e/v2/e2e_test.go renamed to test/e2e/v2/e2e_suite_test.go

@@ -31,7 +31,7 @@ import (
 )
 
 var _ = Describe("kubebuilder", func() {
-	Context("with v2 scaffolding", func() {
+	Context("with project version 2 scaffolding", func() {
 		var kbc *utils.TestContext
 		BeforeEach(func() {
 			var err error