chore: add e2e coverage (#3425)

Author: fmenezes

.gitignore

@@ -17,6 +17,7 @@ macos-notary.zip
 # Output of the go coverage tool
 *.out
 coverage.html
+cov/
 
 # We don't want to commit the 3rd party notices
 third_party_notices/

Makefile

@@ -2,6 +2,7 @@
 
 GOLANGCI_VERSION=v1.61.0
 COVERAGE?=coverage.out
+GOCOVERDIR?=$(abspath cov)
 
 GIT_SHA?=$(shell git rev-parse HEAD)
 
@@ -94,11 +95,11 @@ fix-lint: ## Fix linting errors
 check: test fix-lint ## Run tests and linters
 
 .PHONY: check-templates
-check-templates:
+check-templates: ## Verify templates
 	go run ./tools/templates-checker
 
 .PHONY: addcopy
-addcopy:
+addcopy: ## Add missing license to files
 	@scripts/add-copy.sh
 
 .PHONY: generate
@@ -109,6 +110,10 @@ gen-code: ## Generate code
 	@echo "==> Generating code"
 	go run ./tools/cli-generator
 
+.PHONY: otel
+otel: ## Generate code
+	go run ./tools/otel $(SPAN) --attr $(ATTRS)
+
 .PHONY: gen-mocks
 gen-mocks: ## Generate mocks
 	@echo "==> Generating mocks"
@@ -128,13 +133,13 @@ build: ## Generate an atlas binary in ./bin
 .PHONY: build-debug
 build-debug: ## Generate a binary in ./bin for debugging atlascli
 	@echo "==> Building $(ATLAS_BINARY_NAME) binary for debugging"
-	go build -gcflags="$(DEBUG_FLAGS)" -ldflags "$(LINKER_FLAGS)" $(BUILD_FLAGS) -o $(ATLAS_DESTINATION) $(ATLAS_SOURCE_FILES)
+	go build -gcflags="$(DEBUG_FLAGS)" -ldflags "$(LINKER_FLAGS)" $(BUILD_FLAGS) -cover -o $(ATLAS_DESTINATION) $(ATLAS_SOURCE_FILES)
 
 .PHONY: e2e-test
-e2e-test: build ## Run E2E tests
+e2e-test: build-debug ## Run E2E tests
 # the target assumes the MCLI_* environment variables are exported
 	@echo "==> Running E2E tests..."
-	$(TEST_CMD) -v -p 1 -parallel $(E2E_PARALLEL) -timeout $(E2E_TIMEOUT) -tags="$(E2E_TAGS)" ./test/e2e... $(E2E_EXTRA_ARGS)
+	GOCOVERDIR=$(GOCOVERDIR) $(TEST_CMD) -v -p 1 -parallel $(E2E_PARALLEL) -v -timeout $(E2E_TIMEOUT) -tags="$(E2E_TAGS)" ./test/e2e... $(E2E_EXTRA_ARGS)
 
 .PHONY: fuzz-normalizer-test
 fuzz-normalizer-test: ## Run fuzz test
@@ -144,7 +149,7 @@ fuzz-normalizer-test: ## Run fuzz test
 .PHONY: unit-test
 unit-test: ## Run unit-tests
 	@echo "==> Running unit tests..."
-	$(TEST_CMD) --tags="$(UNIT_TAGS)" -race -cover -count=1 -coverprofile $(COVERAGE) ./...
+	$(TEST_CMD) --tags="$(UNIT_TAGS)" -race -cover -coverprofile $(COVERAGE) -count=1 ./...
 
 .PHONY: install
 install: ## Install a binary in $GOPATH/bin

build/ci/evergreen.yml

@@ -145,8 +145,94 @@ functions:
         env:
           <<: *go_env
           MCLI_SKIP_UPDATE_CHECK: "yes"
+          DO_NOT_TRACK: "1"
           TEST_CMD: gotestsum --junitfile e2e-tests.xml --format standard-verbose --
         command: make e2e-test
+    - command: archive.targz_pack
+      params:
+        target: src/github.com/mongodb/mongodb-atlas-cli/coverage.tgz
+        source_dir: src/github.com/mongodb/mongodb-atlas-cli/cov
+        include:
+          - '*'
+        exclude_files:
+          - .gitkeep
+    - command: s3.put
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: src/github.com/mongodb/mongodb-atlas-cli/coverage.tgz
+        remote_file: ${project}/dist/${revision}_${created_at}/atlascli/cov/${build_variant}_${task_name}.coverage.tgz
+        bucket: mongodb-mongocli-build
+        permissions: public-read
+        content_type: ${content_type|application/json}
+        display_name: internal-bucket
+  "merge cov":
+    - command: subprocess.exec
+      params:
+        <<: *go_options
+        binary: build/ci/merge-cov.sh
+    - command: archive.targz_pack
+      params:
+        target: src/github.com/mongodb/mongodb-atlas-cli/coverage.tgz
+        source_dir: src/github.com/mongodb/mongodb-atlas-cli/
+        include:
+          - 'coverage.out'
+          - 'cov/merged/*'
+        exclude_files:
+          - .gitkeep
+    - command: shell.exec
+      params:
+        <<: *go_options
+        shell: bash
+        script: |
+          set -Eeou pipefail
+          PERCENTAGE=$(go tool cover -func=coverage.out | grep total: | awk '{print $3}' | sed 's/%//')
+          COUNT=$(ls -1 cov/*.tgz | wc -l)
+          cat <<EOF > expansions.yaml
+          percentage: $PERCENTAGE
+          count: $COUNT
+          EOF
+    - command: s3.put
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: src/github.com/mongodb/mongodb-atlas-cli/expansions.yaml
+        remote_file: ${project}/dist/${revision}_${created_at}/atlascli/expansions.yaml
+        bucket: mongodb-mongocli-build
+        permissions: public-read
+        content_type: ${content_type|application/yaml}
+        display_name: internal-bucket expansions.yaml
+    - command: expansions.update
+      params:
+        ignore_missing_file: true
+        file: src/github.com/mongodb/mongodb-atlas-cli/expansions.yaml
+    - command: subprocess.exec
+      params:
+        <<: *go_options
+        binary: rm
+        args:
+          - -f
+          - expansions.yaml
+    - command: s3.put
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: src/github.com/mongodb/mongodb-atlas-cli/coverage.tgz
+        remote_file: ${project}/dist/${revision}_${created_at}/atlascli/cov/e2e.coverage.tgz
+        bucket: mongodb-mongocli-build
+        permissions: public-read
+        content_type: ${content_type|application/json}
+        display_name: internal-bucket e2e.coverage.tgz
+    - command: s3.put
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: src/github.com/mongodb/mongodb-atlas-cli/coverage.out
+        remote_file: ${project}/dist/${revision}_${created_at}/atlascli/cov/e2e.coverage.out
+        bucket: mongodb-mongocli-build
+        permissions: public-read
+        content_type: ${content_type|application/json}
+        display_name: internal-bucket e2e.coverage.out
   "install gotestsum":
     - command: shell.exec
       type: setup
@@ -309,7 +395,7 @@ functions:
         bucket: mongodb-mongocli-build
         permissions: public-read
         content_type: ${content_type|application/octet-stream}
-        display_name: internal-bucket
+        display_name: internal-bucket terraform.tfstate
     - command: shell.exec
       params:
         <<: *go_options
@@ -426,6 +512,26 @@ functions:
         args:
           - -f
           - expansions.yaml
+  "otel":
+    - command: subprocess.exec
+      params:
+        <<: *go_options
+        include_expansions_in_env:
+          - go_base_path
+          - go_proxy
+          - workdir
+          - project_id
+          - project_identifier
+          - otel_trace_id
+          - otel_parent_id
+          - otel_collector_endpoint
+        env:
+          <<: *go_env
+          SPAN: ${span}
+          ATTRS: ${attr}
+        binary: make
+        args:
+          - otel
 tasks:
   - name: compile
     tags: ["code_health"]
@@ -1601,6 +1707,27 @@ tasks:
           user: atlascli
           identity_file: ${home}/.ssh/id_rsa
           cmd: "powershell -ExecutionPolicy Bypass -File C:\\Users\\atlascli\\win_test.ps1 -goproxy ${go_proxy} -revision ${github_commit}"
+  - name: coverage
+    tags: ["coverage"]
+    depends_on:
+      - name: ".e2e"
+        variant: "* !.cron"
+        status: '*'
+    commands:
+      - command: shell.exec
+        params:
+          <<: *go_options
+          env:
+            <<: *go_env
+            AWS_ACCESS_KEY_ID: ${aws_key}
+            AWS_SECRET_ACCESS_KEY: ${aws_secret}
+          script: |
+            aws s3 sync s3://mongodb-mongocli-build/${project}/dist/${revision}_${created_at}/atlascli/cov/ ./cov/
+      - func: "merge cov"
+      - func: "otel"
+        vars:
+          span: "coverage"
+          attr: "total=${percentage},count=${count}"
 task_groups:
   - name: atlas_deployments_windows_group
     setup_task:
@@ -1734,6 +1861,7 @@ buildvariants:
     allowed_requesters: ["patch", "ad_hoc", "github_pr"]
     tags:
       - cleanup
+      - cron
     run_on:
       - rhel80-small
     expansions:
@@ -1766,6 +1894,7 @@ buildvariants:
       - ubuntu2404-small
     tags:
       - localdev
+      - cron
     allowed_requesters: ["patch", "ad_hoc", "github_pr"]
     expansions:
       <<: *go_linux_version
@@ -1777,6 +1906,7 @@ buildvariants:
       - debian11-small
     tags:
       - localdev
+      - cron
     allowed_requesters: ["patch", "ad_hoc", "github_pr"]
     expansions:
       <<: *go_linux_version
@@ -1788,6 +1918,7 @@ buildvariants:
       - debian12-small
     tags:
       - localdev
+      - cron
     allowed_requesters: ["patch", "ad_hoc", "github_pr"]
     expansions:
       <<: *go_linux_version
@@ -1799,6 +1930,7 @@ buildvariants:
       - amazon2023-latest-small
     tags:
       - localdev
+      - cron
     allowed_requesters: ["patch", "ad_hoc", "github_pr"]
     expansions:
       <<: *go_linux_version
@@ -1809,6 +1941,7 @@ buildvariants:
     allowed_requesters: ["patch", "ad_hoc", "github_pr"]
     tags:
       - localdev
+      - cron
     run_on:
       - rhel90-small
     expansions:
@@ -1836,6 +1969,7 @@ buildvariants:
     allowed_requesters: ["patch", "ad_hoc", "github_pr"]
     tags:
       - localdev
+      - cron
     run_on:
       - rhel90-small
     expansions:
@@ -1847,6 +1981,7 @@ buildvariants:
     allowed_requesters: ["patch", "ad_hoc", "github_pr"]
     tags:
       - localdev
+      - cron
     run_on:
       - rhel80-small
     expansions:
@@ -1858,6 +1993,7 @@ buildvariants:
     allowed_requesters: ["patch", "ad_hoc", "github_pr"]
     tags:
       - localdev
+      - cron
     run_on:
       - macos-14-arm64-docker
     expansions:
@@ -1869,6 +2005,7 @@ buildvariants:
     allowed_requesters: ["patch", "ad_hoc", "github_pr"]
     tags:
       - localdev
+      - cron
     run_on:
       - macos-13-arm64-docker
     expansions:
@@ -1880,21 +2017,30 @@ buildvariants:
     allowed_requesters: ["patch", "ad_hoc", "github_pr"]
     tags:
       - packer
+      - cron
     run_on:
       - rhel80-small
     expansions:
       <<: *go_linux_version
     tasks:
       - name: ".packer .windows"
+  - name: coverage
+    display_name: "Coverage"
+    run_on:
+      - rhel80-small
+    expansions:
+      <<: *go_linux_version
+    tasks:
+      - name: ".coverage"
 patch_aliases:
   - alias: "localdev"
-    variant_tags: ["localdev"]
+    variant_tags: ["localdev cron"]
     task: ".*"
   - alias: "packer"
-    variant_tags: ["packer"]
+    variant_tags: ["packer cron"]
     task: ".*"
   - alias: "cleanup"
-    variant_tags: ["cleanup"]
+    variant_tags: ["cleanup cron"]
     task: ".*"
 github_pr_aliases:
   - variant: "code_health"
@@ -1907,3 +2053,6 @@ git_tag_aliases:
   - git_tag: "atlascli/v*"
     variant: "^release_atlascli"
     task: ".*"
+github_checks_aliases:
+ - variant: ".*"
+   task: ".*"

build/ci/library_owners.json

@@ -53,5 +53,10 @@
 	"sigs.k8s.io/kind": "atlas_kubernetes_team",
 	"golang.org/x/exp": "atlas_kubernetes_team",
 	"github.com/denisbrodbeck/machineid": "apix-2",
-	"github.com/shirou/gopsutil/v4": "apix-2"
+	"github.com/shirou/gopsutil/v4": "apix-2",
+	"go.opentelemetry.io/otel": "apix-2",
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc": "apix-2",
+	"go.opentelemetry.io/otel/sdk": "apix-2",
+	"go.opentelemetry.io/otel/trace": "apix-2",
+	"google.golang.org/grpc": "apix-2"
 }

build/ci/merge-cov.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 MongoDB Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+dirs=""
+for f in cov/*.tgz; do
+    dir="${f/.coverage.tgz/}"
+    dirs="$dirs,$dir"
+    mkdir -p "$dir"
+    tar -xzvf "$f" -C "$dir"
+done
+dirs=${dirs:1}
+mkdir -p cov/merged
+go tool covdata merge -i "$dirs" -o cov/merged
+go tool covdata textfmt -i cov/merged -o coverage.out