Merge pull request #1254 from buildkite/fix-fix-perms

Build fix-perms in Makefile

Author: DrJosh9000

.buildkite/docker-compose.yml

@@ -2,20 +2,10 @@ version: '3'
 
 services:
   fixperms-tests:
-    image: golang:latest
+    image: golang:1.21
     working_dir: /code
     environment:
       CGO_ENABLED: 0
     volumes:
       - ..:/code:ro
     command: go test -v ./...
-
-  fixperms-build:
-    image: golang:latest
-    working_dir: /code
-    environment:
-      CGO_ENABLED: 0
-    volumes:
-      - ..:/code
-      - /var/lib/buildkite-agent/git-mirrors:/var/lib/buildkite-agent/git-mirrors
-    command: .buildkite/steps/build-fixperms.sh

.buildkite/pipeline.yaml

@@ -22,18 +22,6 @@ steps:
           run: fixperms-tests
           config: .buildkite/docker-compose.yml
 
-  - id: "fixperms-build"
-    name: ":go: fixperms build"
-    agents:
-      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
-    artifact_paths: "build/fix-perms-*"
-    plugins:
-      - docker-compose#v2.1.0:
-          run: fixperms-build
-          config: .buildkite/docker-compose.yml
-      - artifacts#v1.9.0:
-          upload: "builds/fix-perms-*"
-
   - id: "deploy-service-role-stack"
     name: ":aws-iam: :cloudformation:"
     agents:
@@ -43,7 +31,6 @@ steps:
       - "fmt"
       - "lint"
       - "fixperms-tests"
-      - "fixperms-build"
 
   - id: "packer-windows-amd64"
     name: ":packer: :windows:"
@@ -56,7 +43,6 @@ steps:
       - "fmt"
       - "lint"
       - "fixperms-tests"
-      - "fixperms-build"
 
   - id: "launch-windows-amd64"
     name: ":cloudformation: :windows: AMD64 Launch"
@@ -101,7 +87,6 @@ steps:
       - "fmt"
       - "lint"
       - "fixperms-tests"
-      - "fixperms-build"
 
   - id: "launch-linux-amd64"
     name: ":cloudformation: :linux: AMD64 Launch"
@@ -145,7 +130,6 @@ steps:
       - "fmt"
       - "lint"
       - "fixperms-tests"
-      - "fixperms-build"
 
   - id: "launch-linux-arm64"
     name: ":cloudformation: :linux: ARM64 Launch"

.buildkite/steps/build-fixperms.sh

@@ -16,17 +16,12 @@ fi
 
 mkdir -p "build/"
 
-if [[ "$os" == "linux" ]]; then
-  buildkite-agent artifact download "build/fix-perms-linux-${arch}" ./build
-  mv "build/fix-perms-linux-${arch}" packer/linux/conf/buildkite-agent/scripts/fix-buildkite-agent-builds-permissions
-  chmod 755 packer/linux/conf/buildkite-agent/scripts/fix-buildkite-agent-builds-permissions
-fi
-
 # Build a hash of packer files and the agent versions
-packer_files_sha=$(find Makefile "packer/${os}" plugins/ -type f -print0 | xargs -0 sha1sum | awk '{print $1}' | sort | sha1sum | awk '{print $1}')
+packer_files_sha=$(find Makefile "packer/${os}" plugins/ -type f -print0 | xargs -0 sha256sum | awk '{print $1}' | sort | sha256sum | awk '{print $1}')
+internal_files_sha=$(find go.mod go.sum internal/ -type f -print0 | xargs -0 sha256sum | awk '{print $1}' | sort | sha256sum | awk '{print $1}')
 stable_agent_sha=$(curl -Lfs "https://download.buildkite.com/agent/stable/latest/${agent_binary}.sha256")
 unstable_agent_sha=$(curl -Lfs "https://download.buildkite.com/agent/unstable/latest/${agent_binary}.sha256")
-packer_hash=$(echo "$packer_files_sha" "$arch" "$stable_agent_sha" "$unstable_agent_sha" | sha1sum | awk '{print $1}')
+packer_hash=$(echo "$packer_files_sha" "$internal_files_sha" "$arch" "$stable_agent_sha" "$unstable_agent_sha" | sha256sum | awk '{print $1}')
 
 echo "Packer image hash for ${os}/${arch} is ${packer_hash}"
 packer_file="packer-${packer_hash}-${os}-${arch}.output"

.buildkite/steps/packer.sh

@@ -7,6 +7,10 @@ PACKER_VERSION ?= 1.9.4
 PACKER_LINUX_FILES = $(exec find packer/linux)
 PACKER_WINDOWS_FILES = $(exec find packer/windows)
 
+GO_VERSION ?= 1.21
+
+FIXPERMS_FILES = go.mod go.sum $(exec find internal/fixperms)
+
 AWS_REGION ?= us-east-1
 
 ARM64_INSTANCE_TYPE ?= m7g.xlarge
@@ -87,7 +91,7 @@ build/linux-amd64-ami.txt: packer-linux-amd64.output env-AWS_REGION
 	grep -Eo "$(AWS_REGION): (ami-.+)" $< | cut -d' ' -f2 | xargs echo -n > $@
 
 # Build linux packer image
-packer-linux-amd64.output: $(PACKER_LINUX_FILES)
+packer-linux-amd64.output: $(PACKER_LINUX_FILES) build/fix-perms-linux-amd64
 	docker run \
 		-e AWS_DEFAULT_REGION  \
 		-e AWS_PROFILE \
@@ -120,7 +124,7 @@ print-agent-versions:
 	@echo Windows: $(CURRENT_AGENT_VERSION_WINDOWS)
 
 # Build linuxarm64 packer image
-packer-linux-arm64.output: $(PACKER_LINUX_FILES)
+packer-linux-arm64.output: $(PACKER_LINUX_FILES) build/fix-perms-linux-arm64
 	@echo Agent Version: $(CURRENT_AGENT_VERSION_LINUX)
 	docker run \
 		-e AWS_DEFAULT_REGION  \
@@ -169,6 +173,31 @@ packer-windows-amd64.output: $(PACKER_WINDOWS_FILES)
 			-var 'agent_version=$(CURRENT_AGENT_VERSION_WINDOWS)' \
 			buildkite-ami.pkr.hcl | tee $@
 
+# -----------------------------------------
+# fixperms
+
+build/fix-perms-linux-amd64: $(FIXPERMS_FILES)
+	docker run \
+		-e CGO_ENABLED=0 \
+		-e GOOS=linux \
+		-e GOARCH=amd64 \
+		-v "$(PWD):/src" \
+		-w /src \
+		--rm \
+		golang:$(GO_VERSION) \
+			go build -v -buildvcs=false -o "build/fix-perms-linux-amd64" ./internal/fixperms
+
+build/fix-perms-linux-arm64: $(FIXPERMS_FILES)
+	docker run \
+		-e CGO_ENABLED=0 \
+		-e GOOS=linux \
+		-e GOARCH=arm64 \
+		-v "$(PWD):/src" \
+		-w /src \
+		--rm \
+		golang:$(GO_VERSION) \
+			go build -v -buildvcs=false -o "build/fix-perms-linux-arm64" ./internal/fixperms
+
 # -----------------------------------------
 # Cloudformation helpers
 

Makefile

@@ -81,6 +81,11 @@ build {
     source      = "../../plugins"
   }
 
+  provisioner "file" {
+    destination = "/tmp/build"
+    source      = "../../build"
+  }
+
   provisioner "shell" {
     script = "scripts/install-utils.sh"
   }

packer/linux/buildkite-ami.pkr.hcl

@@ -11,6 +11,10 @@ echo "Installing bk elastic stack bin files..."
 sudo chmod +x /tmp/conf/bin/bk-*
 sudo mv /tmp/conf/bin/bk-* /usr/local/bin
 
+echo "Installing fix-buildkite-agent-builds-permissions..."
+sudo chmod +x "/tmp/build/fix-perms-linux-${ARCH}"
+sudo mv "/tmp/build/fix-perms-linux-${ARCH}" /usr/bin/fix-buildkite-agent-builds-permissions
+
 S3_SECRETS_HELPER_VERSION=2.1.6
 echo "Downloading s3-secrets-helper ${S3_SECRETS_HELPER_VERSION}..."
 sudo curl -Lsf -o /usr/local/bin/s3secrets-helper \