Revert "feat: migrate to tools directive" (#13899)

Reverts #13844 since it caused
#13897

cc @dnaeon (no worries, we can accept your PR again once we are sure it
won't cause further breakage)

Author: mx-psi

.gitignore

@@ -1,5 +1,6 @@
 bin/
 dist/
+.tools/
 /local
 
 # GoLand IDEA
@@ -36,3 +37,4 @@ go.work*
 # npm (used for markdown-link-check)
 node_modules/*
 package-lock.json
+

Makefile

@@ -13,11 +13,7 @@ ALL_DOC := $(shell find . \( -name "*.md" -o -name "*.yaml" \) \
                                 -type f | sort)
 
 # ALL_MODULES includes ./* dirs (excludes . dir)
-ALL_MODULES := $(shell find . -mindepth 2 \
-				-type f \
-				-name "go.mod" \
-				-not -path "./internal/tools/*" \
-				-exec dirname {} \; | sort )
+ALL_MODULES := $(shell find . -type f -name "go.mod" -exec dirname {} \; | sort | grep -E '^./' )
 
 CMD?=
 
@@ -65,8 +61,8 @@ gotest-with-junit:
 	@$(MAKE) for-all-target TARGET="test-with-junit"
 
 .PHONY: goporto
-goporto:
-	$(GO_TOOL) porto -w --include-internal --skip-dirs "^cmd/mdatagen/third_party$$" ./
+goporto: $(PORTO)
+	$(PORTO) -w --include-internal --skip-dirs "^cmd/mdatagen/third_party$$" ./
 
 .PHONY: for-all
 for-all:
@@ -105,8 +101,8 @@ govulncheck:
 	@$(MAKE) for-all-target TARGET="vulncheck"
 
 .PHONY: addlicense
-addlicense:
-	@ADDLICENSEOUT=`$(GO_TOOL) addlicense -s=only -y "" -c "The OpenTelemetry Authors" $(ALL_SRC) 2>&1`; \
+addlicense: $(ADDLICENSE)
+	@ADDLICENSEOUT=`$(ADDLICENSE) -s=only -y "" -c "The OpenTelemetry Authors" $(ALL_SRC) 2>&1`; \
 		if [ "$$ADDLICENSEOUT" ]; then \
 			echo "$(ADDLICENSE) FAILED => add License errors:\n"; \
 			echo "$$ADDLICENSEOUT\n"; \
@@ -116,7 +112,7 @@ addlicense:
 		fi
 
 .PHONY: checklicense
-checklicense:
+checklicense: $(ADDLICENSE)
 	@licRes=$$(for f in $$(find . -type f \( -iname '*.go' -o -iname '*.sh' \) ! -path '**/third_party/*') ; do \
 	           awk '/Copyright The OpenTelemetry Authors|generated|GENERATED/ && NR<=3 { found=1; next } END { if (!found) print FILENAME }' $$f; \
 			   awk '/SPDX-License-Identifier: Apache-2.0|generated|GENERATED/ && NR<=4 { found=1; next } END { if (!found) print FILENAME }' $$f; \
@@ -127,12 +123,12 @@ checklicense:
 	   fi
 
 .PHONY: misspell
-misspell:
-	$(GO_TOOL) misspell -error $(ALL_DOC)
+misspell: $(MISSPELL)
+	$(MISSPELL) -error $(ALL_DOC)
 
 .PHONY: misspell-correction
-misspell-correction:
-	$(GO_TOOL) misspell -w $(ALL_DOC)
+misspell-correction: $(MISSPELL)
+	$(MISSPELL) -w $(ALL_DOC)
 
 .PHONY: run
 run: otelcorecol
@@ -163,13 +159,13 @@ otelcorecol:
 	pushd cmd/otelcorecol && CGO_ENABLED=0 $(GOCMD) build -trimpath -o ../../bin/otelcorecol_$(GOOS)_$(GOARCH) -tags "grpcnotrace" ./... && popd
 
 .PHONY: genotelcorecol
-genotelcorecol:
+genotelcorecol: install-tools
 	pushd cmd/builder/ && $(GOCMD) run ./ --skip-compilation --config ../otelcorecol/builder-config.yaml --output-path ../otelcorecol && popd
 	$(MAKE) -C cmd/otelcorecol fmt
 
 .PHONY: actionlint
-actionlint:
-	$(GO_TOOL) actionlint -config-file .github/actionlint.yaml -color .github/workflows/*.yml .github/workflows/*.yaml
+actionlint: $(ACTIONLINT)
+	$(ACTIONLINT) -config-file .github/actionlint.yaml -color .github/workflows/*.yml .github/workflows/*.yaml
 
 .PHONY: ocb
 ocb:
@@ -250,10 +246,13 @@ genproto_sub:
 	@rm -rf $(OPENTELEMETRY_PROTO_SRC_DIR)/*
 	@rm -rf $(OPENTELEMETRY_PROTO_SRC_DIR)/.* > /dev/null 2>&1 || true
 
+remove-pdatagen:
+	rm -f .tools/pdatagen
+
 # Generate structs, functions and tests for pdata package. Must be used after any changes
 # to proto and after running `make genproto`
-genpdata:
-	cd internal/cmd/pdatagen && $(GOCMD) run main.go -C $(SRC_ROOT)
+genpdata: remove-pdatagen $(PDATAGEN)
+	$(PDATAGEN)
 	$(MAKE) -C pdata fmt
 
 INTERNAL_PROTO_SRC_DIRS := exporter/exporterhelper/internal/queue pdata/xpdata/request/internal
@@ -322,17 +321,17 @@ certs-dryrun:
 	@internal/buildscripts/gen-certs.sh -d
 
 .PHONY: checkapi
-checkapi:
-	$(GO_TOOL) checkapi -folder . -config .checkapi.yaml
+checkapi: $(CHECKAPI)
+	$(CHECKAPI) -folder . -config .checkapi.yaml
 
 # Verify existence of READMEs for components specified as default components in the collector.
 .PHONY: checkdoc
-checkdoc:
-	$(GO_TOOL) checkfile --project-path $(CURDIR) --component-rel-path $(COMP_REL_PATH) --module-name $(MOD_NAME) --file-name "README.md"
+checkdoc: $(CHECKFILE)
+	$(CHECKFILE) --project-path $(CURDIR) --component-rel-path $(COMP_REL_PATH) --module-name $(MOD_NAME) --file-name "README.md"
 
 # Construct new API state snapshots
 .PHONY: apidiff-build
-apidiff-build:
+apidiff-build: $(APIDIFF)
 	@$(foreach pkg,$(ALL_PKGS),$(call exec-command,./internal/buildscripts/gen-apidiff.sh -p $(pkg)))
 
 # If we are running in CI, change input directory
@@ -344,33 +343,33 @@ endif
 
 # Compare API state snapshots
 .PHONY: apidiff-compare
-apidiff-compare:
+apidiff-compare: $(APIDIFF)
 	@$(foreach pkg,$(ALL_PKGS),$(call exec-command,./internal/buildscripts/compare-apidiff.sh -p $(pkg)))
 
 .PHONY: multimod-verify
-multimod-verify:
+multimod-verify: $(MULTIMOD)
 	@echo "Validating versions.yaml"
-	$(GO_TOOL) multimod verify
+	$(MULTIMOD) verify
 
 MODSET?=stable
 .PHONY: multimod-prerelease
-multimod-prerelease:
-	$(GO_TOOL) multimod prerelease -s=true -b=false -v ./versions.yaml -m ${MODSET}
+multimod-prerelease: $(MULTIMOD)
+	$(MULTIMOD) prerelease -s=true -b=false -v ./versions.yaml -m ${MODSET}
 	$(MAKE) gotidy
 
 COMMIT?=HEAD
 REMOTE?[email protected]:open-telemetry/opentelemetry-collector.git
 .PHONY: push-tags
-push-tags:
-	$(GO_TOOL) multimod verify
-	set -e; for tag in `$(GO_TOOL) multimod tag -m ${MODSET} -c ${COMMIT} --print-tags | grep -v "Using" `; do \
+push-tags: $(MULTIMOD)
+	$(MULTIMOD) verify
+	set -e; for tag in `$(MULTIMOD) tag -m ${MODSET} -c ${COMMIT} --print-tags | grep -v "Using" `; do \
 		echo "pushing tag $${tag}"; \
 		git push ${REMOTE} $${tag}; \
 	done;
 
 .PHONY: check-changes
-check-changes:
-	$(GO_TOOL) multimod diff -p $(PREVIOUS_VERSION) -m $(MODSET)
+check-changes: $(MULTIMOD)
+	$(MULTIMOD) diff -p $(PREVIOUS_VERSION) -m $(MODSET)
 
 .PHONY: prepare-release
 prepare-release:
@@ -430,29 +429,29 @@ checklinks:
 # error message "failed to sync logger:  sync /dev/stderr: inappropriate ioctl for device"
 # is a known issue but does not affect function.
 .PHONY: crosslink
-crosslink:
+crosslink: $(CROSSLINK)
 	@echo "Executing crosslink"
-	$(GO_TOOL) crosslink --root=$(shell pwd) --prune
+	$(CROSSLINK) --root=$(shell pwd) --prune
 
 FILENAME?=$(shell git branch --show-current)
 .PHONY: chlog-new
-chlog-new:
-	$(GO_TOOL) chloggen new --config $(CHLOGGEN_CONFIG) --filename $(FILENAME)
+chlog-new: $(CHLOGGEN)
+	$(CHLOGGEN) new --config $(CHLOGGEN_CONFIG) --filename $(FILENAME)
 
 .PHONY: chlog-validate
-chlog-validate:
-	$(GO_TOOL) chloggen validate --config $(CHLOGGEN_CONFIG)
+chlog-validate: $(CHLOGGEN)
+	$(CHLOGGEN) validate --config $(CHLOGGEN_CONFIG)
 
 .PHONY: chlog-preview
-chlog-preview:
-	$(GO_TOOL) chloggen update --config $(CHLOGGEN_CONFIG) --dry
+chlog-preview: $(CHLOGGEN)
+	$(CHLOGGEN) update --config $(CHLOGGEN_CONFIG) --dry
 
 .PHONY: chlog-update
-chlog-update:
-	$(GO_TOOL) chloggen update --config $(CHLOGGEN_CONFIG) --version $(VERSION)
+chlog-update: $(CHLOGGEN)
+	$(CHLOGGEN) update --config $(CHLOGGEN_CONFIG) --version $(VERSION)
 
 .PHONY: builder-integration-test
-builder-integration-test:
+builder-integration-test: $(ENVSUBST)
 	cd ./cmd/builder && ./test/test.sh
 
 .PHONY: mdatagen-test
@@ -463,16 +462,16 @@ mdatagen-test:
 	cd cmd/mdatagen && $(GOCMD) test ./...
 
 .PHONY: generate-gh-issue-templates
-generate-gh-issue-templates:
-	$(GO_TOOL) githubgen issue-templates
+generate-gh-issue-templates: $(GITHUBGEN)
+	$(GITHUBGEN) issue-templates
 
 .PHONY: generate-codeowners
-generate-codeowners:
-	$(GO_TOOL) githubgen --default-codeowner "open-telemetry/collector-approvers" codeowners
+generate-codeowners: $(GITHUBGEN)
+	$(GITHUBGEN) --default-codeowner "open-telemetry/collector-approvers" codeowners
 
 .PHONY: gengithub
-gengithub: generate-codeowners generate-gh-issue-templates
+gengithub: $(GITHUBGEN) generate-codeowners generate-gh-issue-templates
 
 .PHONY: gendistributions
-gendistributions:
-	$(GO_TOOL) githubgen distributions
+gendistributions: $(GITHUBGEN)
+	$(GITHUBGEN) distributions

Makefile.Common

@@ -17,84 +17,101 @@ GOTEST_OPT?= -timeout $(GOTEST_TIMEOUT) $(if $(and $(filter windows,$(GOOS)), $(
 SRC_ROOT := $(shell git rev-parse --show-toplevel)
 
 TOOLS_MOD_DIR   := $(SRC_ROOT)/internal/tools
-TOOLS_MOD_FILE  := $(TOOLS_MOD_DIR)/go.mod
-GO_TOOL         := $(GOCMD) tool -modfile $(TOOLS_MOD_FILE)
+TOOLS_BIN_DIR   := $(SRC_ROOT)/.tools
+TOOLS_MOD_REGEX := "\s+_\s+\".*\""
+TOOLS_PKG_NAMES := $(shell grep -E $(TOOLS_MOD_REGEX) < $(TOOLS_MOD_DIR)/tools.go | tr -d " _\"" | grep -vE '/v[0-9]+$$')
+TOOLS_BIN_NAMES := $(addprefix $(TOOLS_BIN_DIR)/, $(notdir $(shell echo $(TOOLS_PKG_NAMES))))
 CHLOGGEN_CONFIG := .chloggen/config.yaml
 # no trailing slash
 JUNIT_OUT_DIR ?= $(TOOLS_MOD_DIR)/testresults
 
+ACTIONLINT   := $(TOOLS_BIN_DIR)/actionlint
+ADDLICENSE   := $(TOOLS_BIN_DIR)/addlicense
+APIDIFF      := $(TOOLS_BIN_DIR)/apidiff
+CHECKAPI     := $(TOOLS_BIN_DIR)/checkapi
+CHECKFILE    := $(TOOLS_BIN_DIR)/checkfile
+CHLOGGEN     := $(TOOLS_BIN_DIR)/chloggen
+CROSSLINK    := $(TOOLS_BIN_DIR)/crosslink
+ENVSUBST     := $(TOOLS_BIN_DIR)/envsubst
+GITHUBGEN    := $(TOOLS_BIN_DIR)/githubgen
+GOFUMPT      := $(TOOLS_BIN_DIR)/gofumpt
+GOIMPORTS    := $(TOOLS_BIN_DIR)/goimports
+GOVULNCHECK  := $(TOOLS_BIN_DIR)/govulncheck
+LINT         := $(TOOLS_BIN_DIR)/golangci-lint
+PDATAGEN     := $(TOOLS_BIN_DIR)/pdatagen
+IMPI         := $(TOOLS_BIN_DIR)/impi
+MISSPELL     := $(TOOLS_BIN_DIR)/misspell
+MULTIMOD     := $(TOOLS_BIN_DIR)/multimod
+PORTO        := $(TOOLS_BIN_DIR)/porto
+GOTESTSUM    := $(TOOLS_BIN_DIR)/gotestsum
+MODERNIZE    := $(TOOLS_BIN_DIR)/modernize
+
+.PHONY: install-tools
+install-tools: $(TOOLS_BIN_NAMES)
+
+$(TOOLS_BIN_DIR):
+	mkdir -p $@
+
+$(TOOLS_BIN_NAMES): $(TOOLS_BIN_DIR) $(TOOLS_MOD_DIR)/go.mod
+	cd $(TOOLS_MOD_DIR) && GOOS="" GOARCH="" $(GOCMD) build -o $@ -trimpath $(filter %/$(notdir $@),$(TOOLS_PKG_NAMES))
+
 .PHONY: test
-test:
-	$(GO_TOOL) gotestsum --packages="./..." -- $(GOTEST_OPT)
+test: $(GOTESTSUM)
+	$(GOTESTSUM) --packages="./..." -- $(GOTEST_OPT)
 
 .PHONY: test-with-cover
-test-with-cover:
+test-with-cover: $(GOTESTSUM)
 	mkdir -p $(PWD)/coverage/unit
-	$(GO_TOOL) gotestsum \
-		--packages="./..." -- \
-		$(GOTEST_OPT) -cover -covermode=atomic -coverpkg $(COVER_PKGS) -args -test.gocoverdir="$(PWD)/coverage/unit"
+	$(GOTESTSUM) --packages="./..." -- $(GOTEST_OPT) -cover -covermode=atomic -coverpkg $(COVER_PKGS) -args -test.gocoverdir="$(PWD)/coverage/unit"
 
 .PHONY: test-with-junit
-test-with-junit:
+test-with-junit: $(GOTESTSUM)
 	mkdir -p $(JUNIT_OUT_DIR)
-	$(GO_TOOL) gotestsum \
-		--packages="./..." --junitfile $(JUNIT_OUT_DIR)/$(CURR_MOD)-junit.xml -- \
-		$(GOTEST_OPT) ./...
+	$(GOTESTSUM) --packages="./..." --junitfile $(JUNIT_OUT_DIR)/$(CURR_MOD)-junit.xml -- $(GOTEST_OPT) ./...
 
 .PHONY: benchmark
-benchmark:
-	$(GO_TOOL) gotestsum \
-		--packages="$(ALL_PKGS)" -- \
-		-bench=. -run=notests ./... | tee benchmark.txt
+benchmark: $(GOTESTSUM)
+	$(GOTESTSUM) --packages="$(ALL_PKGS)" -- -bench=. -run=notests ./... | tee benchmark.txt
 
 .PHONY: fmt
 fmt: common/gofmt common/goimports common/gofumpt
 
-# `modernize' cannot be installed as a Go tool via `go get -tool'.
-#
-# See [1] for more details.
-#
-# [1]: https://github.com/golang/go/issues/73279
 .PHONY: modernize
-modernize:
-	$(GOCMD) run \
-		golang.org/x/tools/gopls/internal/analysis/modernize/cmd/modernize@latest \
-		-fix -test -v ./...
+modernize: $(MODERNIZE)
+	$(MODERNIZE) -fix -test -v ./...
 
 .PHONY: tidy
 tidy:
 	rm -fr go.sum
 	$(GOCMD) mod tidy -compat=1.24.0
 
 .PHONY: lint
-lint:
-	$(GO_TOOL) golangci-lint run
+lint: $(LINT)
+	$(LINT) run
 
 .PHONY: common/gofmt
 common/gofmt:
 	gofmt -w -s ./
 
 .PHONY: common/goimports
-common/goimports:
-	$(GO_TOOL) goimports -w -local go.opentelemetry.io/collector ./
+common/goimports: $(GOIMPORTS)
+	$(GOIMPORTS) -w  -local go.opentelemetry.io/collector ./
 
 .PHONY: common/gofumpt
-common/gofumpt:
-	$(GO_TOOL) gofumpt -l -w -extra .
+common/gofumpt: $(GOFUMPT)
+	$(GOFUMPT) -l -w -extra .
 
 .PHONY: vulncheck
-vulncheck:
-	$(GO_TOOL) govulncheck ./...
+vulncheck: $(GOVULNCHECK)
+	$(GOVULNCHECK) ./...
 
 .PHONY: generate
 generate:
 	$(GOCMD) generate ./...
 
 .PHONY: impi
-impi:
-	$(GO_TOOL) impi \
-		--local go.opentelemetry.io/collector \
-		--scheme stdThirdPartyLocal ./...
+impi: $(IMPI)
+	@$(IMPI) --local go.opentelemetry.io/collector --scheme stdThirdPartyLocal ./...
 
 .PHONY: moddownload
 moddownload:

cmd/builder/test/test.sh

@@ -33,8 +33,7 @@ test_build_config() {
     echo "Starting test '${test}' at $(date)" >> "${out}/test.log"
 
     final_build_config=$(basename "${build_config}")
-    go tool -modfile "${WORKSPACE_DIR}/internal/tools/go.mod" envsubst \
-       -o "${out}/${final_build_config}" -i <(cat "$build_config" "$replaces")
+    "${WORKSPACE_DIR}/.tools/envsubst" -o "${out}/${final_build_config}" -i <(cat "$build_config" "$replaces")
     if ! go run . --config "${out}/${final_build_config}" --output-path "${out}" > "${out}/builder.log" 2>&1; then
         echo "❌ FAIL ${test}. Failed to compile the test ${test}. Build logs:"
         cat "${out}/builder.log"