Merge pull request #1972 from jchunkins/modify_makefiles

update makefile, workflow, and docs

Author: openshift-merge-robot

.github/workflows/e2e-tests.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - run: make -f x.mk e2e-local NODES=2 JUNIT_DIRECTORY=./artifacts/
+      - run: make e2e-local NODES=2 JUNIT_DIRECTORY=./artifacts/
       - name: Archive Test Artifacts # test results, failed or not, are always uploaded.
         if: ${{ always() }}
         uses: actions/upload-artifact@v2

.gitignore

@@ -454,3 +454,4 @@ apiserver.crt
 apiserver.key
 
 !vendor/**
+test/e2e-local.image.tar

Makefile

@@ -25,6 +25,9 @@ CONTROLLER_GEN := go run $(MOD_FLAGS) ./vendor/sigs.k8s.io/controller-tools/cmd/
 YQ_INTERNAL := go run $(MOD_FLAGS) ./vendor/github.com/mikefarah/yq/v3/
 KUBEBUILDER_ASSETS := $(or $(or $(KUBEBUILDER_ASSETS),$(dir $(shell command -v kubebuilder))),/usr/local/kubebuilder/bin)
 export KUBEBUILDER_ASSETS
+GO := GO111MODULE=on GOFLAGS="$(MOD_FLAGS)" go
+GINKGO := $(GO) run github.com/onsi/ginkgo/ginkgo
+BINDATA := $(GO) run github.com/go-bindata/go-bindata/v3/go-bindata
 
 # ART builds are performed in dist-git, with content (but not commits) copied 
 # from the source repo. Thus at build time if your code is inspecting the local
@@ -89,15 +92,15 @@ build-linux: clean $(CMDS)
 
 build-wait: clean bin/wait
 
-bin/wait:
+bin/wait: FORCE
 	GOOS=linux GOARCH=386 go build $(MOD_FLAGS) -o $@ $(PKG)/test/e2e/wait
 
 build-util-linux: arch_flags=GOOS=linux GOARCH=386
 build-util-linux: build-util
 
 build-util: bin/cpb
 
-bin/cpb:
+bin/cpb: FORCE
 	CGO_ENABLED=0 $(arch_flags) go build $(MOD_FLAGS) -ldflags '-extldflags "-static"' -o $@ ./util/cpb
 
 $(CMDS): version_flags=-ldflags "-X $(PKG)/pkg/version.GitCommit=$(GIT_COMMIT) -X $(PKG)/pkg/version.OLMVersion=`cat OLM_VERSION`"
@@ -128,9 +131,36 @@ setup-bare: clean e2e.namespace
 e2e:
 	go test -v $(MOD_FLAGS)  -failfast -timeout 150m ./test/e2e/... -namespace=openshift-operators -kubeconfig=${KUBECONFIG} -olmNamespace=openshift-operator-lifecycle-manager -dummyImage=bitnami/nginx:latest -ginkgo.flakeAttempts=3
 
-e2e-local: build-linux build-wait build-util-linux
-	. ./scripts/build_local.sh
-	. ./scripts/run_e2e_local.sh $(TEST)
+### Start: End To End Tests ###
+
+# Phony prerequisite for targets that rely on the go build cache to determine staleness.
+.PHONY: FORCE
+FORCE:
+
+# main entry point for running end to end tests. used by .github/workflows/e2e-tests.yml See test/e2e/README.md for details
+.PHONY: e2e-local
+e2e-local: bin/e2e-local.test test/e2e-local.image.tar
+	$(GINKGO) -nodes $(or $(NODES),1) -flakeAttempts 3 -randomizeAllSpecs $(if $(TEST),-focus '$(TEST)') -v -timeout 90m $< -- -namespace=operators -olmNamespace=operator-lifecycle-manager -dummyImage=bitnami/nginx:latest -kind.images=../test/e2e-local.image.tar
+
+# this target updates the zz_chart.go file with files found in deploy/chart
+# this will always fire since it has been marked as phony
+.PHONY: test/e2e/assets/chart/zz_chart.go
+test/e2e/assets/chart/zz_chart.go: $(shell find deploy/chart -type f)
+	$(BINDATA) -o $@ -pkg chart -prefix deploy/chart/ $^
+
+# execute kind and helm end to end tests
+bin/e2e-local.test: FORCE test/e2e/assets/chart/zz_chart.go
+	$(GO) test -c -tags kind,helm -o $@ ./test/e2e
+
+# set go env and other vars, ensure that the dockerfile exists, and then build wait, cpb, and other command binaries and finally the kind image archive
+test/e2e-local.image.tar: export GOOS=linux
+test/e2e-local.image.tar: export GOARCH=386
+test/e2e-local.image.tar: build_cmd=build
+test/e2e-local.image.tar: e2e.Dockerfile bin/wait bin/cpb $(CMDS)
+	docker build -t quay.io/operator-framework/olm:local -f $< bin
+	docker save -o $@ quay.io/operator-framework/olm:local
+
+### Finish: End To End Tests ###
 
 e2e-bare: setup-bare
 	. ./scripts/run_e2e_bare.sh $(TEST)

scripts/package_release.sh

@@ -17,7 +17,9 @@ charttmpdir=$(mktemp -d 2>/dev/null || mktemp -d -t charttmpdir)
 charttmpdir=${charttmpdir}/chart
 
 cp -R deploy/chart/ "${charttmpdir}"
-sed -i "s/^[Vv]ersion:.*\$/version: ${version}/" "${charttmpdir}/Chart.yaml"
+
+# overwrite the destination Chart.yaml file with a modified copy containing the version
+sed "s/^[Vv]ersion:.*\$/version: ${version}/" deploy/chart/Chart.yaml > "${charttmpdir}/Chart.yaml"
 
 mkdir -p "${chartdir}"
 

test/e2e/README.md

@@ -9,6 +9,49 @@ This runs a series of tests against the Kubernetes API to verify that OLM is fun
 
 ## How to use
 
-`make e2e-local` in the root of the repository will fetch golang dependencies, start Minikube or using an existing Kind environment, build the appropriate images and run the tests in a fresh namespace each time.
 
-Subsequent runs of the test suite do not need to go through the full setup process. Running individual tests (or the whole suite) can be accomplished by running `./test/e2e/run_e2e_local.sh [TestName]` with an optional test name.
+Execute `make e2e-local` in the root of the repository, which will:
+
+- optionally update `test/e2e/assets/chart/zz_chart.go` as needed
+- build local executables used during testing
+  - `bin/e2e-local.test`
+  - `bin/wait`
+  - `bin/cpb`
+  - `bin/catalog`
+  - `bin/olm`
+  - `bin/package-server`
+- build docker file `e2e.Dockerfile` which includes the local executables in a `kind` image archive `test/e2e-local.image.tar`
+- execute `ginkgo` to run the pre-compiled test package `bin/e2e-local.test` with the `kind` image archive. This runs BDD tests defined in `test/e2e`
+  - these tests are run in a kind cluster that is started fresh each time the test is executed
+
+
+Examples:
+
+- Run all BDD tests (this takes a long time)
+
+  ```bash
+  make e2e-local
+  ```
+
+- Run a specific BDD test using the `TEST` argument to make. Note that this argument uses regular expressions.
+
+  ```bash
+  make e2e-local TEST='API service resource not migrated if not adoptable'
+  ```
+
+- If you have previously created the `bin/e2e-local.test` executable and want a quick way to ensure that your TEST regex argument will work, you can bypass the 
+make file and use `-dryRun` with `-focus` and see if the regex would trigger your specific test(s).
+  
+  ```bash
+  GO111MODULE=on GOFLAGS="-mod=vendor" go run github.com/onsi/ginkgo/ginkgo -dryRun -focus 'API service resource not migrated if not adoptable' bin/e2e-local.test
+  ```
+
+- It is also possible to specify the number of parallel test nodes (i.e. one or more instances of `go test`) to run using the `NODES` argument. Defaults to 1 if not specified
+
+  ```bash
+  make e2e-local NODES=2
+  ```
+
+## Build infrastructure
+
+Note that the make file target `e2e-local` is executed by the github workflow `.github/workflows/e2e-tests.yml` and uses two parallel `go test` processes.