GODRIVER-2445 Clean up Makefile and run all package tests in targets. (#997)

Author: benjirewis

Makefile

@@ -1,48 +1,43 @@
-BSON_PKGS = $(shell etc/list_pkgs.sh ./bson)
-BSON_TEST_PKGS = $(shell etc/list_test_pkgs.sh ./bson)
-EVENT_PKGS = $(shell etc/list_pkgs.sh ./event)
-EVENT_TEST_PKGS = $(shell etc/list_test_pkgs.sh ./event)
-MONGO_PKGS = $(shell etc/list_pkgs.sh ./mongo)
-MONGO_TEST_PKGS = $(shell etc/list_test_pkgs.sh ./mongo)
-UNSTABLE_PKGS = $(shell etc/list_pkgs.sh ./x)
-UNSTABLE_TEST_PKGS = $(shell etc/list_test_pkgs.sh ./x)
-TAG_PKG = $(shell etc/list_pkgs.sh ./tag)
-TAG_TEST_PKG = $(shell etc/list_test_pkgs.sh ./tag)
-EXAMPLES_PKGS = $(shell etc/list_pkgs.sh ./examples)
-EXAMPLES_TEST_PKGS = $(shell etc/list_test_pkgs.sh ./examples)
-PKGS = $(BSON_PKGS) $(EVENT_PKGS) $(MONGO_PKGS) $(UNSTABLE_PKGS) $(TAG_PKG) $(EXAMPLES_PKGS)
-TEST_PKGS = $(BSON_TEST_PKGS) $(EVENT_TEST_PKGS) $(MONGO_TEST_PKGS) $(UNSTABLE_TEST_PKGS) $(TAG_PKG) $(EXAMPLES_TEST_PKGS)
+# We list packages with shell scripts and loop through them to avoid testing with ./...
+# Running go test ./... will run tests in all packages concurrently which can lead to
+# unexpected errors.
+#
+# TODO(GODRIVER-2093): Use ./... to run tests in all packages with parallelism and remove
+# these PKG variables and loops from all make targets.
+PKGS = $(shell etc/list_pkgs.sh)
+TEST_PKGS = $(shell etc/list_test_pkgs.sh)
+
 ATLAS_URIS = "$(ATLAS_FREE)" "$(ATLAS_REPLSET)" "$(ATLAS_SHARD)" "$(ATLAS_TLS11)" "$(ATLAS_TLS12)" "$(ATLAS_FREE_SRV)" "$(ATLAS_REPLSET_SRV)" "$(ATLAS_SHARD_SRV)" "$(ATLAS_TLS11_SRV)" "$(ATLAS_TLS12_SRV)" "$(ATLAS_SERVERLESS)" "$(ATLAS_SERVERLESS_SRV)"
 GODISTS=linux/amd64 linux/386 linux/arm64 linux/arm linux/s390x
-
 TEST_TIMEOUT = 1800
 
+### Utility targets. ###
 .PHONY: default
-default: check-env check-fmt build-examples lint test-cover test-race
+default: add-license build build-examples check-env check-fmt lint test-short
 
-.PHONY: check-env
-check-env:
-	etc/check_env.sh
+.PHONY: add-license
+add-license:
+	# Find all .go files not in the vendor directory and try to write a license notice.
+	find . -path ./vendor -prune -o -type f -name "*.go" -print | xargs ./etc/add_license.sh
+	# Check for any changes made with -G. to ignore permissions changes. Exit with a non-zero
+	# exit code if there is a diff.
+	git diff -G. --quiet
 
-.PHONY: doc
-doc:
-	godoc -http=:6060 -index
+.PHONY: build
+build:
+	go build $(BUILD_TAGS) $(PKGS)
 
 .PHONY: build-examples
 build-examples:
 	go build $(BUILD_TAGS) ./examples/... ./x/mongo/driver/examples/...
 
-.PHONY: build
-build:
-	go build $(BUILD_TAGS) $(filter-out ./core/auth/internal/gssapi,$(PKGS))
-
 .PHONY: build-no-tags
 build-no-tags:
-	go build $(filter-out ./core/auth/internal/gssapi,$(PKGS))
+	go build $(PKGS)
 
 .PHONY: build-tests
 build-tests:
-	for TEST in $(PKGS); do \
+	for TEST in $(TEST_PKGS); do \
 		go test $(BUILD_TAGS) -c $$TEST ; \
 		if [ $$? -ne 0 ]; \
 		then \
@@ -54,6 +49,10 @@ build-tests:
 check-fmt:
 	etc/check_fmt.sh $(PKGS)
 
+.PHONY: doc
+doc:
+	godoc -http=:6060 -index
+
 .PHONY: fmt
 fmt:
 	gofmt -l -s -w $(PKGS)
@@ -68,14 +67,11 @@ lint:
 		eval $$command ; \
 	done
 
-.PHONY: add-license
-add-license:
-	# Find all .go files not in the vendor directory and try to write a license notice.
-	find . -path ./vendor -prune -o -type f -name "*.go" -print | xargs ./etc/add-license.sh
-	# Check for any changes made with -G. to ignore permissions changes. Exit with a non-zero
-	# exit code if there is a diff.
-	git diff -G. --quiet
+.PHONY: update-notices
+update-notices:
+	etc/generate_notices.pl > THIRD-PARTY-NOTICES
 
+### Local testing targets. ###
 .PHONY: test
 test:
 	for TEST in $(TEST_PKGS) ; do \
@@ -85,105 +81,69 @@ test:
 .PHONY: test-cover
 test-cover:
 	for TEST in $(TEST_PKGS) ; do \
-    	go test $(BUILD_TAGS) -timeout $(TEST_TIMEOUT)s -cover $(COVER_ARGS) $$TEST ; \
-    done
+		go test $(BUILD_TAGS) -timeout $(TEST_TIMEOUT)s -cover $(COVER_ARGS) $$TEST ; \
+	done
 
 .PHONY: test-race
 test-race:
 	for TEST in $(TEST_PKGS) ; do \
-    	go test $(BUILD_TAGS) -timeout $(TEST_TIMEOUT)s -race $(COVER_ARGS) $$TEST ; \
-    done
+		go test $(BUILD_TAGS) -timeout $(TEST_TIMEOUT)s -race $$TEST ; \
+	done
 
 .PHONY: test-short
 test-short:
-	go test $(BUILD_TAGS) -timeout 60s -short $(COVER_ARGS) ./...
-
-.PHONY: update-bson-corpus-tests
-update-bson-corpus-tests:
-	etc/update-spec-tests.sh bson-corpus
-
-.PHONY: update-connection-string-tests
-update-connection-string-tests:
-	etc/update-spec-tests.sh connection-string
-
-.PHONY: update-crud-tests
-update-crud-tests:
-	etc/update-spec-tests.sh crud
+	go test $(BUILD_TAGS) -timeout 60s -short $(TEST_PKGS)
 
-.PHONY: update-initial-dns-seedlist-discovery-tests
-update-initial-dns-seedlist-discovery-tests:
-	etc/update-spec-tests.sh initial-dns-seedlist-discovery
-
-.PHONY: update-max-staleness-tests
-update-max-staleness-tests:
-	etc/update-spec-tests.sh max-staleness
-
-.PHONY: update-server-discovery-and-monitoring-tests
-update-server-discovery-and-monitoring-tests:
-	etc/update-spec-tests.sh server-discovery-and-monitoring
-
-.PHONY: update-server-selection-tests
-update-server-selection-tests:
-	etc/update-spec-tests.sh server-selection
-
-.PHONY: update-notices
-update-notices:
-	etc/generate-notices.pl > THIRD-PARTY-NOTICES
+### Evergreen specific targets. ###
+.PHONY: build-aws-ecs-test
+build-aws-ecs-test:
+	go build $(BUILD_TAGS) ./mongo/testaws/main.go
 
-# Evergreen specific targets
 .PHONY: evg-test
 evg-test:
 	for TEST in $(TEST_PKGS); do \
 		go test -exec "env PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH)" $(BUILD_TAGS) -v -timeout $(TEST_TIMEOUT)s $$TEST >> test.suite ; \
 	done
 
-.PHONY: evg-test-auth
-evg-test-auth:
-	go run -tags gssapi ./x/mongo/driver/examples/count/main.go -uri $(MONGODB_URI)
-
 .PHONY: evg-test-atlas
 evg-test-atlas:
 	go run ./mongo/testatlas/main.go $(ATLAS_URIS)
 
-.PHONY: evg-test-ocsp
-evg-test-ocsp:
-	go test -v ./mongo -run TestOCSP $(OCSP_TLS_SHOULD_SUCCEED) >> test.suite
-
-.PHONY: build-aws-ecs-test
-build-aws-ecs-test:
-	go build $(BUILD_TAGS) ./mongo/testaws/main.go
-
 .PHONY: evg-test-atlas-data-lake
 evg-test-atlas-data-lake:
 	ATLAS_DATA_LAKE_INTEGRATION_TEST=true go test -v ./mongo/integration -run TestUnifiedSpecs/atlas-data-lake-testing >> spec_test.suite
 	ATLAS_DATA_LAKE_INTEGRATION_TEST=true go test -v ./mongo/integration -run TestAtlasDataLake >> spec_test.suite
 
-.PHONY: evg-test-versioned-api
-evg-test-versioned-api:
-	for TEST_PKG in ./mongo ./mongo/integration ./mongo/integration/unified; do \
-		go test -exec "env PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH)" $(BUILD_TAGS) -v -timeout $(TEST_TIMEOUT)s $$TEST_PKG >> test.suite ; \
-	done
+.PHONY: evg-test-auth
+evg-test-auth:
+	go run -tags gssapi ./x/mongo/driver/examples/count/main.go -uri $(MONGODB_URI)
+
+.PHONY: evg-test-kmip
+evg-test-kmip:
+	go test -exec "env PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH)" $(BUILD_TAGS) -v -timeout $(TEST_TIMEOUT)s ./mongo/integration -run TestClientSideEncryptionSpec/kmipKMS >> test.suite
+	go test -exec "env PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH)" $(BUILD_TAGS) -v -timeout $(TEST_TIMEOUT)s ./mongo/integration -run TestClientSideEncryptionProse/data_key_and_double_encryption >> test.suite
+	go test -exec "env PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH)" $(BUILD_TAGS) -v -timeout $(TEST_TIMEOUT)s ./mongo/integration -run TestClientSideEncryptionProse/corpus >> test.suite
+	go test -exec "env PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH)" $(BUILD_TAGS) -v -timeout $(TEST_TIMEOUT)s ./mongo/integration -run TestClientSideEncryptionProse/custom_endpoint >> test.suite
+	go test -exec "env PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH)" $(BUILD_TAGS) -v -timeout $(TEST_TIMEOUT)s ./mongo/integration -run TestClientSideEncryptionProse/kms_tls_options_test >> test.suite
+
+.PHONY: evg-test-kms
+evg-test-kms:
+	go test -exec "env PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH)" $(BUILD_TAGS) -v -timeout $(TEST_TIMEOUT)s ./mongo/integration -run TestClientSideEncryptionProse/kms_tls_tests >> test.suite
 
 .PHONY: evg-test-load-balancers
 evg-test-load-balancers:
+	# Load balancer should be tested with all unified tests as well as tests in the following
+	# components: retryable reads, retryable writes, change streams, initial DNS seedlist discovery.
 	go test $(BUILD_TAGS) ./mongo/integration -run TestUnifiedSpecs/retryable-reads -v -timeout $(TEST_TIMEOUT)s >> test.suite
 	go test $(BUILD_TAGS) ./mongo/integration -run TestRetryableWritesSpec -v -timeout $(TEST_TIMEOUT)s >> test.suite
 	go test $(BUILD_TAGS) ./mongo/integration -run TestChangeStreamSpec -v -timeout $(TEST_TIMEOUT)s >> test.suite
 	go test $(BUILD_TAGS) ./mongo/integration -run TestInitialDNSSeedlistDiscoverySpec/load_balanced -v -timeout $(TEST_TIMEOUT)s >> test.suite
 	go test $(BUILD_TAGS) ./mongo/integration -run TestLoadBalancerSupport -v -timeout $(TEST_TIMEOUT)s >> test.suite
 	go test $(BUILD_TAGS) ./mongo/integration/unified -run TestUnifiedSpec -v -timeout $(TEST_TIMEOUT)s >> test.suite
 
-.PHONY: evg-test-kms
-evg-test-kms:
-	go test -exec "env PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH)" $(BUILD_TAGS) -v -timeout $(TEST_TIMEOUT)s ./mongo/integration -run TestClientSideEncryptionProse/kms_tls_tests >> test.suite
-
-.PHONY: evg-test-kmip
-evg-test-kmip:
-	go test -exec "env PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH)" $(BUILD_TAGS) -v -timeout $(TEST_TIMEOUT)s ./mongo/integration -run TestClientSideEncryptionSpec/kmipKMS >> test.suite
-	go test -exec "env PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH)" $(BUILD_TAGS) -v -timeout $(TEST_TIMEOUT)s ./mongo/integration -run TestClientSideEncryptionProse/data_key_and_double_encryption >> test.suite
-	go test -exec "env PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH)" $(BUILD_TAGS) -v -timeout $(TEST_TIMEOUT)s ./mongo/integration -run TestClientSideEncryptionProse/corpus >> test.suite
-	go test -exec "env PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH)" $(BUILD_TAGS) -v -timeout $(TEST_TIMEOUT)s ./mongo/integration -run TestClientSideEncryptionProse/custom_endpoint >> test.suite
-	go test -exec "env PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH)" $(BUILD_TAGS) -v -timeout $(TEST_TIMEOUT)s ./mongo/integration -run TestClientSideEncryptionProse/kms_tls_options_test >> test.suite
+.PHONY: evg-test-ocsp
+evg-test-ocsp:
+	go test -v ./mongo -run TestOCSP $(OCSP_TLS_SHOULD_SUCCEED) >> test.suite
 
 .PHONY: evg-test-serverless
 evg-test-serverless:
@@ -207,14 +167,24 @@ evg-test-serverless:
 	go test $(BUILD_TAGS) ./mongo/integration -run TestCursor -v -timeout $(TEST_TIMEOUT)s >> test.suite
 	go test $(BUILD_TAGS) ./mongo/integration/unified -run TestUnifiedSpec -v -timeout $(TEST_TIMEOUT)s >> test.suite
 
-# benchmark specific targets and support
+.PHONY: evg-test-versioned-api
+evg-test-versioned-api:
+	# Versioned API related tests are in the mongo, integration and unified packages.
+	for TEST_PKG in ./mongo ./mongo/integration ./mongo/integration/unified; do \
+		go test -exec "env PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH)" $(BUILD_TAGS) -v -timeout $(TEST_TIMEOUT)s $$TEST_PKG >> test.suite ; \
+	done
+
+### Benchmark specific targets and support. ###
+.PHONY: benchmark
+benchmark:perf
+	go test $(BUILD_TAGS) -benchmem -bench=. ./benchmark
+
+.PHONY: driver-benchmark
+driver-benchmark:perf
+	@go run cmd/godriver-benchmark/main.go | tee perf.suite
+
 perf:driver-test-data.tar.gz
 	tar -zxf $< $(if $(eq $(UNAME_S),Darwin),-s , --transform=s)/data/perf/
 	@touch $@
 driver-test-data.tar.gz:
 	curl --retry 5 "https://s3.amazonaws.com/boxes.10gen.com/build/driver-test-data.tar.gz" -o driver-test-data.tar.gz --silent --max-time 120
-benchmark:perf
-	go test $(BUILD_TAGS) -benchmem -bench=. ./benchmark
-driver-benchmark:perf
-	@go run cmd/godriver-benchmark/main.go | tee perf.suite
-.PHONY:benchmark driver-benchmark

etc/add-license.sh renamed to etc/add_license.sh

@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# check_fmt gopackages...
+# Runs gofmt on given packages and checks that *_example_test.go files have wrapped lines.
 
 gofmt_out="$(gofmt -l -s "$@")"
 

etc/check_env.sh

@@ -1,5 +1,7 @@
 #!/bin/sh
 # list_pkgs <directory>
+# Lists all Go packages in the given directory.
+
 directory="$1"
 if [ -z "$directory" ]; then
     directory="."

etc/check_fmt.sh

@@ -1,5 +1,7 @@
 #!/bin/sh
-# list_pkgs <directory>
+# list_test_pkgs <directory>
+# Lists all Go packages with _test.go files in the given directory.
+
 directory="$1"
 if [ -z "$directory" ]; then
     directory="."

etc/generate-notices.pl renamed to etc/generate_notices.pl

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
-
-# This script is used to fetch the latest JSON tests for the CRUD spec. It puts the tests in the
-# direcory $reporoot/data/crud. It should be run from the root of the repository.
+# update_spec_tests spec
+# This script is used to fetch the latest tests for the given spec. It puts the tests in the
+# directory data/[specname]. It should be run from the root of the repository.
 
 set -o errexit
 set -o nounset

etc/list_pkgs.sh

@@ -4,6 +4,7 @@
 // not use this file except in compliance with the License. You may obtain
 // a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
 
+//go:build race
 // +build race
 
 // Package israce reports if the Go race detector is enabled.