feat: migrate to tools directive (#13844)

#### Description

This PR migrates the existing `internal/tools` to the new [recommended
approach for managing tool
dependencies](https://go.dev/wiki/Modules#how-can-i-track-tool-dependencies-for-a-module).

The makefile targets have been updated, including test/build scripts as
well.

All tools have been migrated to the to the new `go tool` approach, with
one exception - `modernize` tool.

The reason why `modernize` has been left out is because of [this
issue](golang/go#73279).

> The issue here is that the modernize command, currently (but not
fundamentally) part of the gopls module, is governed by the gopls
release process, and you can't just upgrade the x/tools package to a
later one, as happens when installing stringer, because the gopls
release branch needs a particular version and x/tools does not present a
stable internal API to gopls. In short, no-one should ever add gopls as
a module dependency, and that means you cannot use go get -tool with
gopls. I will update the documentation to make this clear.

Please check this issue for full details:

- golang/go#73279

Based on the comment above it would appear that even in the [current
internal/tools/go.mod](https://github.com/open-telemetry/opentelemetry-collector/blob/f729e7b55a6b9cf703c74f13d9f48ac8b655ced7/internal/tools/go.mod#L22)
we have a dependency on `gopls`, which we should not.

cc: @mx-psi 

#### Link to tracking issue
Fixes #13721

#### Testing

Manual testing and running the various Makefile targets, including `all`
target.

#### Documentation

N/A

Author: dnaeon

.gitignore

@@ -1,6 +1,5 @@
 bin/
 dist/
-.tools/
 /local
 
 # GoLand IDEA
@@ -37,4 +36,3 @@ go.work*
 # npm (used for markdown-link-check)
 node_modules/*
 package-lock.json
-

Makefile

@@ -13,7 +13,11 @@ ALL_DOC := $(shell find . \( -name "*.md" -o -name "*.yaml" \) \
                                 -type f | sort)
 
 # ALL_MODULES includes ./* dirs (excludes . dir)
-ALL_MODULES := $(shell find . -type f -name "go.mod" -exec dirname {} \; | sort | grep -E '^./' )
+ALL_MODULES := $(shell find . -mindepth 2 \
+				-type f \
+				-name "go.mod" \
+				-not -path "./internal/tools/*" \
+				-exec dirname {} \; | sort )
 
 CMD?=
 
@@ -61,8 +65,8 @@ gotest-with-junit:
 	@$(MAKE) for-all-target TARGET="test-with-junit"
 
 .PHONY: goporto
-goporto: $(PORTO)
-	$(PORTO) -w --include-internal --skip-dirs "^cmd/mdatagen/third_party$$" ./
+goporto:
+	$(GO_TOOL) porto -w --include-internal --skip-dirs "^cmd/mdatagen/third_party$$" ./
 
 .PHONY: for-all
 for-all:
@@ -101,8 +105,8 @@ govulncheck:
 	@$(MAKE) for-all-target TARGET="vulncheck"
 
 .PHONY: addlicense
-addlicense: $(ADDLICENSE)
-	@ADDLICENSEOUT=`$(ADDLICENSE) -s=only -y "" -c "The OpenTelemetry Authors" $(ALL_SRC) 2>&1`; \
+addlicense:
+	@ADDLICENSEOUT=`$(GO_TOOL) addlicense -s=only -y "" -c "The OpenTelemetry Authors" $(ALL_SRC) 2>&1`; \
 		if [ "$$ADDLICENSEOUT" ]; then \
 			echo "$(ADDLICENSE) FAILED => add License errors:\n"; \
 			echo "$$ADDLICENSEOUT\n"; \
@@ -112,7 +116,7 @@ addlicense: $(ADDLICENSE)
 		fi
 
 .PHONY: checklicense
-checklicense: $(ADDLICENSE)
+checklicense:
 	@licRes=$$(for f in $$(find . -type f \( -iname '*.go' -o -iname '*.sh' \) ! -path '**/third_party/*') ; do \
 	           awk '/Copyright The OpenTelemetry Authors|generated|GENERATED/ && NR<=3 { found=1; next } END { if (!found) print FILENAME }' $$f; \
 			   awk '/SPDX-License-Identifier: Apache-2.0|generated|GENERATED/ && NR<=4 { found=1; next } END { if (!found) print FILENAME }' $$f; \
@@ -123,12 +127,12 @@ checklicense: $(ADDLICENSE)
 	   fi
 
 .PHONY: misspell
-misspell: $(MISSPELL)
-	$(MISSPELL) -error $(ALL_DOC)
+misspell:
+	$(GO_TOOL) misspell -error $(ALL_DOC)
 
 .PHONY: misspell-correction
-misspell-correction: $(MISSPELL)
-	$(MISSPELL) -w $(ALL_DOC)
+misspell-correction:
+	$(GO_TOOL) misspell -w $(ALL_DOC)
 
 .PHONY: run
 run: otelcorecol
@@ -159,13 +163,13 @@ otelcorecol:
 	pushd cmd/otelcorecol && CGO_ENABLED=0 $(GOCMD) build -trimpath -o ../../bin/otelcorecol_$(GOOS)_$(GOARCH) -tags "grpcnotrace" ./... && popd
 
 .PHONY: genotelcorecol
-genotelcorecol: install-tools
+genotelcorecol:
 	pushd cmd/builder/ && $(GOCMD) run ./ --skip-compilation --config ../otelcorecol/builder-config.yaml --output-path ../otelcorecol && popd
 	$(MAKE) -C cmd/otelcorecol fmt
 
 .PHONY: actionlint
-actionlint: $(ACTIONLINT)
-	$(ACTIONLINT) -config-file .github/actionlint.yaml -color .github/workflows/*.yml .github/workflows/*.yaml
+actionlint:
+	$(GO_TOOL) actionlint -config-file .github/actionlint.yaml -color .github/workflows/*.yml .github/workflows/*.yaml
 
 .PHONY: ocb
 ocb:
@@ -246,13 +250,10 @@ genproto_sub:
 	@rm -rf $(OPENTELEMETRY_PROTO_SRC_DIR)/*
 	@rm -rf $(OPENTELEMETRY_PROTO_SRC_DIR)/.* > /dev/null 2>&1 || true
 
-remove-pdatagen:
-	rm -f .tools/pdatagen
-
 # Generate structs, functions and tests for pdata package. Must be used after any changes
 # to proto and after running `make genproto`
-genpdata: remove-pdatagen $(PDATAGEN)
-	$(PDATAGEN)
+genpdata:
+	cd internal/cmd/pdatagen && $(GOCMD) run main.go -C $(SRC_ROOT)
 	$(MAKE) -C pdata fmt
 
 INTERNAL_PROTO_SRC_DIRS := exporter/exporterhelper/internal/queue pdata/xpdata/request/internal
@@ -321,17 +322,17 @@ certs-dryrun:
 	@internal/buildscripts/gen-certs.sh -d
 
 .PHONY: checkapi
-checkapi: $(CHECKAPI)
-	$(CHECKAPI) -folder . -config .checkapi.yaml
+checkapi:
+	$(GO_TOOL) checkapi -folder . -config .checkapi.yaml
 
 # Verify existence of READMEs for components specified as default components in the collector.
 .PHONY: checkdoc
-checkdoc: $(CHECKFILE)
-	$(CHECKFILE) --project-path $(CURDIR) --component-rel-path $(COMP_REL_PATH) --module-name $(MOD_NAME) --file-name "README.md"
+checkdoc:
+	$(GO_TOOL) checkfile --project-path $(CURDIR) --component-rel-path $(COMP_REL_PATH) --module-name $(MOD_NAME) --file-name "README.md"
 
 # Construct new API state snapshots
 .PHONY: apidiff-build
-apidiff-build: $(APIDIFF)
+apidiff-build:
 	@$(foreach pkg,$(ALL_PKGS),$(call exec-command,./internal/buildscripts/gen-apidiff.sh -p $(pkg)))
 
 # If we are running in CI, change input directory
@@ -343,33 +344,33 @@ endif
 
 # Compare API state snapshots
 .PHONY: apidiff-compare
-apidiff-compare: $(APIDIFF)
+apidiff-compare:
 	@$(foreach pkg,$(ALL_PKGS),$(call exec-command,./internal/buildscripts/compare-apidiff.sh -p $(pkg)))
 
 .PHONY: multimod-verify
-multimod-verify: $(MULTIMOD)
+multimod-verify:
 	@echo "Validating versions.yaml"
-	$(MULTIMOD) verify
+	$(GO_TOOL) multimod verify
 
 MODSET?=stable
 .PHONY: multimod-prerelease
-multimod-prerelease: $(MULTIMOD)
-	$(MULTIMOD) prerelease -s=true -b=false -v ./versions.yaml -m ${MODSET}
+multimod-prerelease:
+	$(GO_TOOL) multimod prerelease -s=true -b=false -v ./versions.yaml -m ${MODSET}
 	$(MAKE) gotidy
 
 COMMIT?=HEAD
 REMOTE?[email protected]:open-telemetry/opentelemetry-collector.git
 .PHONY: push-tags
-push-tags: $(MULTIMOD)
-	$(MULTIMOD) verify
-	set -e; for tag in `$(MULTIMOD) tag -m ${MODSET} -c ${COMMIT} --print-tags | grep -v "Using" `; do \
+push-tags:
+	$(GO_TOOL) multimod verify
+	set -e; for tag in `$(GO_TOOL) multimod tag -m ${MODSET} -c ${COMMIT} --print-tags | grep -v "Using" `; do \
 		echo "pushing tag $${tag}"; \
 		git push ${REMOTE} $${tag}; \
 	done;
 
 .PHONY: check-changes
-check-changes: $(MULTIMOD)
-	$(MULTIMOD) diff -p $(PREVIOUS_VERSION) -m $(MODSET)
+check-changes:
+	$(GO_TOOL) multimod diff -p $(PREVIOUS_VERSION) -m $(MODSET)
 
 .PHONY: prepare-release
 prepare-release:
@@ -429,29 +430,29 @@ checklinks:
 # error message "failed to sync logger:  sync /dev/stderr: inappropriate ioctl for device"
 # is a known issue but does not affect function.
 .PHONY: crosslink
-crosslink: $(CROSSLINK)
+crosslink:
 	@echo "Executing crosslink"
-	$(CROSSLINK) --root=$(shell pwd) --prune
+	$(GO_TOOL) crosslink --root=$(shell pwd) --prune
 
 FILENAME?=$(shell git branch --show-current)
 .PHONY: chlog-new
-chlog-new: $(CHLOGGEN)
-	$(CHLOGGEN) new --config $(CHLOGGEN_CONFIG) --filename $(FILENAME)
+chlog-new:
+	$(GO_TOOL) chloggen new --config $(CHLOGGEN_CONFIG) --filename $(FILENAME)
 
 .PHONY: chlog-validate
-chlog-validate: $(CHLOGGEN)
-	$(CHLOGGEN) validate --config $(CHLOGGEN_CONFIG)
+chlog-validate:
+	$(GO_TOOL) chloggen validate --config $(CHLOGGEN_CONFIG)
 
 .PHONY: chlog-preview
-chlog-preview: $(CHLOGGEN)
-	$(CHLOGGEN) update --config $(CHLOGGEN_CONFIG) --dry
+chlog-preview:
+	$(GO_TOOL) chloggen update --config $(CHLOGGEN_CONFIG) --dry
 
 .PHONY: chlog-update
-chlog-update: $(CHLOGGEN)
-	$(CHLOGGEN) update --config $(CHLOGGEN_CONFIG) --version $(VERSION)
+chlog-update:
+	$(GO_TOOL) chloggen update --config $(CHLOGGEN_CONFIG) --version $(VERSION)
 
 .PHONY: builder-integration-test
-builder-integration-test: $(ENVSUBST)
+builder-integration-test:
 	cd ./cmd/builder && ./test/test.sh
 
 .PHONY: mdatagen-test
@@ -462,16 +463,16 @@ mdatagen-test:
 	cd cmd/mdatagen && $(GOCMD) test ./...
 
 .PHONY: generate-gh-issue-templates
-generate-gh-issue-templates: $(GITHUBGEN)
-	$(GITHUBGEN) issue-templates
+generate-gh-issue-templates:
+	$(GO_TOOL) githubgen issue-templates
 
 .PHONY: generate-codeowners
-generate-codeowners: $(GITHUBGEN)
-	$(GITHUBGEN) --default-codeowner "open-telemetry/collector-approvers" codeowners
+generate-codeowners:
+	$(GO_TOOL) githubgen --default-codeowner "open-telemetry/collector-approvers" codeowners
 
 .PHONY: gengithub
-gengithub: $(GITHUBGEN) generate-codeowners generate-gh-issue-templates
+gengithub: generate-codeowners generate-gh-issue-templates
 
 .PHONY: gendistributions
-gendistributions: $(GITHUBGEN)
-	$(GITHUBGEN) distributions
+gendistributions:
+	$(GO_TOOL) githubgen distributions

Makefile.Common

@@ -17,101 +17,84 @@ GOTEST_OPT?= -timeout $(GOTEST_TIMEOUT) $(if $(and $(filter windows,$(GOOS)), $(
 SRC_ROOT := $(shell git rev-parse --show-toplevel)
 
 TOOLS_MOD_DIR   := $(SRC_ROOT)/internal/tools
-TOOLS_BIN_DIR   := $(SRC_ROOT)/.tools
-TOOLS_MOD_REGEX := "\s+_\s+\".*\""
-TOOLS_PKG_NAMES := $(shell grep -E $(TOOLS_MOD_REGEX) < $(TOOLS_MOD_DIR)/tools.go | tr -d " _\"" | grep -vE '/v[0-9]+$$')
-TOOLS_BIN_NAMES := $(addprefix $(TOOLS_BIN_DIR)/, $(notdir $(shell echo $(TOOLS_PKG_NAMES))))
+TOOLS_MOD_FILE  := $(TOOLS_MOD_DIR)/go.mod
+GO_TOOL         := $(GOCMD) tool -modfile $(TOOLS_MOD_FILE)
 CHLOGGEN_CONFIG := .chloggen/config.yaml
 # no trailing slash
 JUNIT_OUT_DIR ?= $(TOOLS_MOD_DIR)/testresults
 
-ACTIONLINT   := $(TOOLS_BIN_DIR)/actionlint
-ADDLICENSE   := $(TOOLS_BIN_DIR)/addlicense
-APIDIFF      := $(TOOLS_BIN_DIR)/apidiff
-CHECKAPI     := $(TOOLS_BIN_DIR)/checkapi
-CHECKFILE    := $(TOOLS_BIN_DIR)/checkfile
-CHLOGGEN     := $(TOOLS_BIN_DIR)/chloggen
-CROSSLINK    := $(TOOLS_BIN_DIR)/crosslink
-ENVSUBST     := $(TOOLS_BIN_DIR)/envsubst
-GITHUBGEN    := $(TOOLS_BIN_DIR)/githubgen
-GOFUMPT      := $(TOOLS_BIN_DIR)/gofumpt
-GOIMPORTS    := $(TOOLS_BIN_DIR)/goimports
-GOVULNCHECK  := $(TOOLS_BIN_DIR)/govulncheck
-LINT         := $(TOOLS_BIN_DIR)/golangci-lint
-PDATAGEN     := $(TOOLS_BIN_DIR)/pdatagen
-IMPI         := $(TOOLS_BIN_DIR)/impi
-MISSPELL     := $(TOOLS_BIN_DIR)/misspell
-MULTIMOD     := $(TOOLS_BIN_DIR)/multimod
-PORTO        := $(TOOLS_BIN_DIR)/porto
-GOTESTSUM    := $(TOOLS_BIN_DIR)/gotestsum
-MODERNIZE    := $(TOOLS_BIN_DIR)/modernize
-
-.PHONY: install-tools
-install-tools: $(TOOLS_BIN_NAMES)
-
-$(TOOLS_BIN_DIR):
-	mkdir -p $@
-
-$(TOOLS_BIN_NAMES): $(TOOLS_BIN_DIR) $(TOOLS_MOD_DIR)/go.mod
-	cd $(TOOLS_MOD_DIR) && GOOS="" GOARCH="" $(GOCMD) build -o $@ -trimpath $(filter %/$(notdir $@),$(TOOLS_PKG_NAMES))
-
 .PHONY: test
-test: $(GOTESTSUM)
-	$(GOTESTSUM) --packages="./..." -- $(GOTEST_OPT)
+test:
+	$(GO_TOOL) gotestsum --packages="./..." -- $(GOTEST_OPT)
 
 .PHONY: test-with-cover
-test-with-cover: $(GOTESTSUM)
+test-with-cover:
 	mkdir -p $(PWD)/coverage/unit
-	$(GOTESTSUM) --packages="./..." -- $(GOTEST_OPT) -cover -covermode=atomic -coverpkg $(COVER_PKGS) -args -test.gocoverdir="$(PWD)/coverage/unit"
+	$(GO_TOOL) gotestsum \
+		--packages="./..." -- \
+		$(GOTEST_OPT) -cover -covermode=atomic -coverpkg $(COVER_PKGS) -args -test.gocoverdir="$(PWD)/coverage/unit"
 
 .PHONY: test-with-junit
-test-with-junit: $(GOTESTSUM)
+test-with-junit:
 	mkdir -p $(JUNIT_OUT_DIR)
-	$(GOTESTSUM) --packages="./..." --junitfile $(JUNIT_OUT_DIR)/$(CURR_MOD)-junit.xml -- $(GOTEST_OPT) ./...
+	$(GO_TOOL) gotestsum \
+		--packages="./..." --junitfile $(JUNIT_OUT_DIR)/$(CURR_MOD)-junit.xml -- \
+		$(GOTEST_OPT) ./...
 
 .PHONY: benchmark
-benchmark: $(GOTESTSUM)
-	$(GOTESTSUM) --packages="$(ALL_PKGS)" -- -bench=. -run=notests ./... | tee benchmark.txt
+benchmark:
+	$(GO_TOOL) gotestsum \
+		--packages="$(ALL_PKGS)" -- \
+		-bench=. -run=notests ./... | tee benchmark.txt
 
 .PHONY: fmt
 fmt: common/gofmt common/goimports common/gofumpt
 
+# `modernize' cannot be installed as a Go tool via `go get -tool'.
+#
+# See [1] for more details.
+#
+# [1]: https://github.com/golang/go/issues/73279
 .PHONY: modernize
-modernize: $(MODERNIZE)
-	$(MODERNIZE) -fix -test -v ./...
+modernize:
+	$(GOCMD) run \
+		golang.org/x/tools/gopls/internal/analysis/modernize/cmd/modernize@latest \
+		-fix -test -v ./...
 
 .PHONY: tidy
 tidy:
 	rm -fr go.sum
 	$(GOCMD) mod tidy -compat=1.24.0
 
 .PHONY: lint
-lint: $(LINT)
-	$(LINT) run
+lint:
+	$(GO_TOOL) golangci-lint run
 
 .PHONY: common/gofmt
 common/gofmt:
 	gofmt -w -s ./
 
 .PHONY: common/goimports
-common/goimports: $(GOIMPORTS)
-	$(GOIMPORTS) -w  -local go.opentelemetry.io/collector ./
+common/goimports:
+	$(GO_TOOL) goimports -w -local go.opentelemetry.io/collector ./
 
 .PHONY: common/gofumpt
-common/gofumpt: $(GOFUMPT)
-	$(GOFUMPT) -l -w -extra .
+common/gofumpt:
+	$(GO_TOOL) gofumpt -l -w -extra .
 
 .PHONY: vulncheck
-vulncheck: $(GOVULNCHECK)
-	$(GOVULNCHECK) ./...
+vulncheck:
+	$(GO_TOOL) govulncheck ./...
 
 .PHONY: generate
 generate:
 	$(GOCMD) generate ./...
 
 .PHONY: impi
-impi: $(IMPI)
-	@$(IMPI) --local go.opentelemetry.io/collector --scheme stdThirdPartyLocal ./...
+impi:
+	$(GO_TOOL) impi \
+		--local go.opentelemetry.io/collector \
+		--scheme stdThirdPartyLocal ./...
 
 .PHONY: moddownload
 moddownload:

cmd/builder/test/test.sh

@@ -33,7 +33,8 @@ test_build_config() {
     echo "Starting test '${test}' at $(date)" >> "${out}/test.log"
 
     final_build_config=$(basename "${build_config}")
-    "${WORKSPACE_DIR}/.tools/envsubst" -o "${out}/${final_build_config}" -i <(cat "$build_config" "$replaces")
+    go tool -modfile "${WORKSPACE_DIR}/internal/tools/go.mod" envsubst \
+       -o "${out}/${final_build_config}" -i <(cat "$build_config" "$replaces")
     if ! go run . --config "${out}/${final_build_config}" --output-path "${out}" > "${out}/builder.log" 2>&1; then
         echo "❌ FAIL ${test}. Failed to compile the test ${test}. Build logs:"
         cat "${out}/builder.log"