feat: inject deps from finch-core (#1679)

inject deps from finch-core

Signed-off-by: Swapnanil-Gupta <swpnlg@amazon.com>

Author: Swapnanil-Gupta

Makefile

@@ -12,6 +12,7 @@ COVERAGE_DIR ?= $(CURDIR)/cov
 REPORT_DIR ?= $(CURDIR)/reports
 RUN_ID ?= $(GITHUB_RUN_ID)
 RUN_ATTEMPT ?= $(GITHUB_RUN_ATTEMPT)
+FINCH_CORE_DIR := $(CURDIR)/deps/finch-core
 
 OUTPUT_DIRECTORIES := $(OUTDIR) $(OS_OUTDIR)
 $(OUTPUT_DIRECTORIES):
@@ -34,9 +35,21 @@ GITCOMMIT ?= $(shell git rev-parse HEAD)$(shell test -z "$(git status --porcelai
 VERSION_INJECTION := -X $(PACKAGE)/pkg/version.Version=$(VERSION)
 VERSION_INJECTION += -X $(PACKAGE)/pkg/version.GitCommit=$(GITCOMMIT)
 VERSION_INJECTION += -X $(PACKAGE)/pkg/version.GitCommit=$(GITCOMMIT)
-LDFLAGS = "-w $(VERSION_INJECTION)"
 MIN_MACOS_VERSION ?= 11.0
 
+# Inject soci version
+-include $(FINCH_CORE_DIR)/deps/soci.conf
+VERSION_INJECTION += -X $(PACKAGE)/pkg/config.SociVersion=$(SOCI_VERSION)
+VERSION_INJECTION += -X $(PACKAGE)/pkg/config.SociAMD64Sha256Sum=$(SOCI_AMD64_SHA256_DIGEST)
+VERSION_INJECTION += -X $(PACKAGE)/pkg/config.SociARM64Sha256Sum=$(SOCI_ARM64_SHA256_DIGEST)
+
+# Inject ecr-cred-helper version
+-include $(FINCH_CORE_DIR)/deps/ecr-cred-helper.conf
+VERSION_INJECTION += -X $(PACKAGE)/pkg/dependency/credhelper.EcrVersion=$(ECR_CRED_HELPER_VERSION)
+VERSION_INJECTION += -X $(PACKAGE)/pkg/dependency/credhelper.EcrAMD64Hash=$(ECR_CRED_HELPER_AMD64_DIGEST)
+VERSION_INJECTION += -X $(PACKAGE)/pkg/dependency/credhelper.EcrARM64Hash=$(ECR_CRED_HELPER_ARM64_DIGEST)
+LDFLAGS = "-w $(VERSION_INJECTION)"
+
 FINCH_DAEMON_LOCATION_ROOT ?= $(FINCH_OS_IMAGE_LOCATION_ROOT)/finch-daemon
 FINCH_DAEMON_LOCATION ?= $(FINCH_DAEMON_LOCATION_ROOT)/finch-daemon
 FINCH_DAEMON_CREDHELPER_LOCATION ?= $(FINCH_DAEMON_LOCATION_ROOT)/docker-credential-finch
@@ -81,9 +94,7 @@ ifeq ($(BUILD_OS),)
 BUILD_OS = $(shell uname -s)
 endif
 
-FINCH_CORE_DIR := $(CURDIR)/deps/finch-core
-
-remote-all: arch-test install.finch-core-dependencies finch finch.yaml networks.yaml config.yaml $(OUTDIR)/finch-daemon/finch@.service
+remote-all: arch-test finch install.finch-core-dependencies finch.yaml networks.yaml config.yaml $(OUTDIR)/finch-daemon/finch@.service
 
 ifeq ($(BUILD_OS), Windows_NT)
 include Makefile.windows
@@ -108,13 +119,13 @@ CONTAINER_RUNTIME_ARCHIVE_AARCH64_DIGEST ?= "sha256:$(AARCH64_256_DIGEST)"
 CONTAINER_RUNTIME_ARCHIVE_X86_64_LOCATION ?= "$(ARTIFACT_BASE_URL)/$(X86_64_ARTIFACT)"
 CONTAINER_RUNTIME_ARCHIVE_X86_64_DIGEST ?= "sha256:$(X86_64_256_DIGEST)"
 
-# For Finch on macOS and Windows, the runc override locations and digests are set
-# based on the values set in deps/finch-core/deps/runc-override.conf
--include $(FINCH_CORE_DIR)/deps/runc-override.conf
-RUNC_OVERRIDE_AARCH64_LOCATION ?= "$(RUNC_ARTIFACT_BASE_URL)/$(RUNC_AARCH64_ARTIFACT)"
-RUNC_OVERRIDE_AARCH64_DIGEST ?= "sha256:$(RUNC_AARCH64_256_DIGEST)"
-RUNC_OVERRIDE_X86_64_LOCATION ?= "$(RUNC_ARTIFACT_BASE_URL)/$(RUNC_X86_64_ARTIFACT)"
-RUNC_OVERRIDE_X86_64_DIGEST ?= "sha256:$(RUNC_X86_64_256_DIGEST)"
+# Inject dependency versions for E2E VM serial tests
+E2E_VM_VERSION_INJECTION := $(VERSION_INJECTION)
+E2E_VM_VERSION_INJECTION += -X $(PACKAGE)/e2e/vm.NerdctlVersion=v$(NERDCTL_VERSION)
+E2E_VM_VERSION_INJECTION += -X $(PACKAGE)/e2e/vm.ContainerdVersion=v$(CONTAINERD_VERSION)
+E2E_VM_VERSION_INJECTION += -X $(PACKAGE)/e2e/vm.BuildKitVersion=v$(BUILDKIT_VERSION)
+E2E_VM_VERSION_INJECTION += -X $(PACKAGE)/e2e/vm.RuncVersion=$(RUNC_VERSION)
+E2E_VM_LDFLAGS := -w $(E2E_VM_VERSION_INJECTION)
 
 .PHONY: finch.yaml
 finch.yaml: $(OS_OUTDIR)/finch.yaml
@@ -349,15 +360,15 @@ test-e2e: test-e2e-vm-serial test-e2e-container
 
 .PHONY: test-e2e-vm-serial
 test-e2e-vm-serial: create-report-dir create-coverage-dir add-credhelper-to-path
-	FINCH_GOCOVERDIR=$(COVERAGE_DIR) go test -coverpkg=./... -ldflags $(LDFLAGS) -timeout 2h ./e2e/vm -test.v -test.gocoverdir=$(COVERAGE_DIR) -ginkgo.v -ginkgo.timeout=2h -ginkgo.flake-attempts=3 -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-vm-serial-report.json --installed="$(INSTALLED)"
+	FINCH_GOCOVERDIR=$(COVERAGE_DIR) go test -coverpkg=./... -ldflags "$(E2E_VM_LDFLAGS)" -timeout 2h ./e2e/vm -test.v -test.gocoverdir=$(COVERAGE_DIR) -ginkgo.v -ginkgo.timeout=2h -ginkgo.flake-attempts=3 -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-vm-serial-report.json --installed="$(INSTALLED)"
 
 .PHONY: test-e2e-container
 test-e2e-container: create-report-dir create-coverage-dir add-credhelper-to-path
 	FINCH_GOCOVERDIR=$(COVERAGE_DIR) go test -coverpkg=./... -ldflags $(LDFLAGS) -timeout 2h ./e2e/container -test.v -test.gocoverdir=$(COVERAGE_DIR) -ginkgo.v -ginkgo.timeout=2h -ginkgo.flake-attempts=3 -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-container-report.json --installed="$(INSTALLED)"
 
 .PHONY: test-e2e-vm
 test-e2e-vm: create-report-dir create-coverage-dir
-	FINCH_GOCOVERDIR=$(COVERAGE_DIR) go test -coverpkg=./... -ldflags $(LDFLAGS) -timeout 2h ./e2e/vm -test.v -test.gocoverdir=$(COVERAGE_DIR) -ginkgo.v -ginkgo.timeout=2h  -ginkgo.focus "updates init-only config values when values are changed after init" -ginkgo.flake-attempts=3 -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-vm-report.json --installed="$(INSTALLED)" --registry="$(REGISTRY)"
+	FINCH_GOCOVERDIR=$(COVERAGE_DIR) go test -coverpkg=./... -ldflags "$(E2E_VM_LDFLAGS)" -timeout 2h ./e2e/vm -test.v -test.gocoverdir=$(COVERAGE_DIR) -ginkgo.v -ginkgo.timeout=2h -ginkgo.flake-attempts=3 -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-vm-report.json --installed="$(INSTALLED)" --registry="$(REGISTRY)"
 
 .PHONY: test-e2e-cov
 test-e2e-cov:
@@ -386,7 +397,9 @@ test-e2e-daemon:
 	DOCKER_HOST=$(DAEMON_DOCKER_HOST) \
 	DOCKER_API_VERSION="v1.41" \
 	TEST_E2E=1 \
-	go test ./e2e -timeout 15m -test.v -ginkgo.v -ginkgo.randomize-all -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-daemon-report.json \
+	go test ./e2e -timeout 30m -test.v -ginkgo.v \
+	  -ginkgo.randomize-all -ginkgo.flake-attempts=3 \
+	  -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-daemon-report.json \
 	  --subject="$(OUTDIR)/bin/$(BINARYNAME)" \
 	  --daemon-context-subject-prefix="$(OUTDIR)/lima/bin/limactl shell finch sudo" \
 	  --daemon-context-subject-env="LIMA_HOME=$(OUTDIR)/lima/data"

Makefile.windows

@@ -32,10 +32,6 @@ $(OS_OUTDIR)/finch.yaml: $(OS_OUTDIR) finch.yaml.d/common.yaml finch.yaml.d/wind
 	sed -i.bak -e "s/<container_runtime_archive_aarch64_digest>/$(CONTAINER_RUNTIME_ARCHIVE_AARCH64_DIGEST)/g" finch.yaml.temp
 	sed -i.bak -e "s|<container_runtime_archive_x86_64_location>|$(CONTAINER_RUNTIME_ARCHIVE_X86_64_LOCATION)|g" finch.yaml.temp
 	sed -i.bak -e "s/<container_runtime_archive_x86_64_digest>/$(CONTAINER_RUNTIME_ARCHIVE_X86_64_DIGEST)/g" finch.yaml.temp
-	sed -i.bak -e "s|<runc_override_aarch64_location>|$(RUNC_OVERRIDE_AARCH64_LOCATION)|g" finch.yaml.temp
-	sed -i.bak -e "s/<runc_override_aarch64_digest>/$(RUNC_OVERRIDE_AARCH64_DIGEST)/g" finch.yaml.temp
-	sed -i.bak -e "s|<runc_override_x86_64_location>|$(RUNC_OVERRIDE_X86_64_LOCATION)|g" finch.yaml.temp
-	sed -i.bak -e "s/<runc_override_x86_64_digest>/$(RUNC_OVERRIDE_X86_64_DIGEST)/g" finch.yaml.temp
 
 	# Replacement was successful, so cleanup .bak
 	@rm finch.yaml.temp.bak

deps/finch-core

@@ -18,12 +18,15 @@ import (
 	"github.com/runfinch/finch/pkg/version"
 )
 
-// TODO: read this from finch-core/deps.
-const (
-	nerdctlVersion    = "v2.2.1"
-	buildKitVersion   = "v0.26.3"
-	containerdVersion = "v2.2.1"
-	runcVersion       = "1.3.3"
+var (
+	// NerdctlVersion is filled via Makefile.
+	NerdctlVersion string
+	// ContainerdVersion is filled via Makefile.
+	ContainerdVersion string
+	// BuildKitVersion is filled via Makefile.
+	BuildKitVersion string
+	// RuncVersion is filled via Makefile.
+	RuncVersion string
 )
 
 type Versions struct {
@@ -38,10 +41,10 @@ type Versions struct {
 var versions = Versions{
 	FinchVersion:      strings.ReplaceAll(version.Version, ".", "\\."),
 	FinchCommit:       strings.ReplaceAll(version.GitCommit, ".", "\\."),
-	NerdctlVersion:    strings.ReplaceAll(nerdctlVersion, ".", "\\."),
-	BuildKitVersion:   strings.ReplaceAll(buildKitVersion, ".", "\\."),
-	ContainerdVersion: strings.ReplaceAll(containerdVersion, ".", "\\."),
-	RuncVersion:       strings.ReplaceAll(runcVersion, ".", "\\."),
+	NerdctlVersion:    strings.ReplaceAll(NerdctlVersion, ".", "\\."),
+	BuildKitVersion:   strings.ReplaceAll(BuildKitVersion, ".", "\\."),
+	ContainerdVersion: strings.ReplaceAll(ContainerdVersion, ".", "\\."),
+	RuncVersion:       strings.ReplaceAll(RuncVersion, ".", "\\."),
 }
 
 // Checks finch version.

e2e/vm/version_remote_test.go

@@ -29,44 +29,6 @@ provision:
   - mode: user
     script: |
       #!/bin/bash
-      
-      # Override runc with specified version after container runtime is installed
-      ARCH=$(uname -m)
-      if [ "$ARCH" = "x86_64" ]; then
-        RUNC_URL="<runc_override_x86_64_location>"
-        RUNC_DIGEST="<runc_override_x86_64_digest>"
-      elif [ "$ARCH" = "aarch64" ]; then
-        RUNC_URL="<runc_override_aarch64_location>"
-        RUNC_DIGEST="<runc_override_aarch64_digest>"
-      else
-        echo "Unsupported architecture: $ARCH"
-        exit 1
-      fi
-      
-      # Download and verify runc override
-      curl -L --fail "$RUNC_URL" -o /tmp/runc-override
-      echo "$RUNC_DIGEST /tmp/runc-override" | sha256sum -c -
-      
-      # Replace the existing runc binary
-      sudo chmod +x /tmp/runc-override
-      sudo cp /tmp/runc-override /usr/local/bin/runc
-      sudo cp /tmp/runc-override /usr/bin/runc
-      sudo rm /tmp/runc-override
-      
-      # Verify runc version is >= 1.3.3
-      INSTALLED_VERSION=$(runc --version | head -n1 | awk '{print $3}')
-      REQUIRED_VERSION="1.3.3"
-      if ! printf '%s\n%s\n' "$REQUIRED_VERSION" "$INSTALLED_VERSION" | sort -V -C; then
-        echo "ERROR: runc version $INSTALLED_VERSION is less than required $REQUIRED_VERSION"
-        exit 1
-      fi
-      echo "SUCCESS: runc version $INSTALLED_VERSION meets requirement >= $REQUIRED_VERSION"
-      
-      # Restart containerd
-      sudo systemctl restart containerd.service
-  - mode: user
-    script: |
-      #!/bin/bash
 
       # Enable SSHing into the VM as root (e.g., in `nerdctlConfigApplier.Apply`).
       sudo cp ~/.ssh/authorized_keys /root/.ssh/

finch.yaml.d/common.yaml

@@ -19,10 +19,16 @@ import (
 	"github.com/runfinch/finch/pkg/system"
 )
 
+var (
+	// SociVersion will be injected via Makefile.
+	SociVersion string
+	// SociAMD64Sha256Sum will be injected via Makefile.
+	SociAMD64Sha256Sum string
+	// SociARM64Sha256Sum will be injected via Makefile.
+	SociARM64Sha256Sum string
+)
+
 const (
-	sociVersion                              = "0.11.1"
-	sociAMD64Sha256Sum                       = "52d72692880150f974a48dbfa44bff01f1a3ba97815658cc16f835e5e7f96d49"
-	sociARM64Sha256Sum                       = "da055b182000dbac3e916a8c731cbc0d7a204791311260c94d960a1160d25b3e"
 	snapshotterProvisioningScriptHeader      = "# snapshotter provisioning script"
 	sociInstallationProvisioningScriptHeader = snapshotterProvisioningScriptHeader + ": soci"
 	sociFileNameFormat                       = "soci-snapshotter-%s-linux-%s.tar.gz"
@@ -247,13 +253,13 @@ func (lca *limaConfigApplier) provisionSnapshotters(limaCfg *limayaml.LimaYAML)
 
 func (lca *limaConfigApplier) provisionSociSnapshotter(limaCfg *limayaml.LimaYAML) {
 	arch := lca.systemDeps.Arch()
-	sociFileName := fmt.Sprintf(sociFileNameFormat, sociVersion, arch)
-	sociDownloadURL := fmt.Sprintf(sociDownloadURLFormat, sociVersion, sociFileName)
-	sociSha256Sum := sociAMD64Sha256Sum
+	sociFileName := fmt.Sprintf(sociFileNameFormat, SociVersion, arch)
+	sociDownloadURL := fmt.Sprintf(sociDownloadURLFormat, SociVersion, sociFileName)
+	sociSha256Sum := SociAMD64Sha256Sum
 	if arch == "arm64" {
-		sociSha256Sum = sociARM64Sha256Sum
+		sociSha256Sum = SociARM64Sha256Sum
 	}
-	sociServiceDownloadURL := fmt.Sprintf(sociServiceDownloadURLFormat, sociVersion)
+	sociServiceDownloadURL := fmt.Sprintf(sociServiceDownloadURLFormat, SociVersion)
 
 	// Platform-specific DOCKER_CONFIG for SOCI service (macOS only)
 	// This is needed as DOCKER_CONFIG for SOCI defaults to ~/.finch/config,

pkg/config/lima_config_applier.go

@@ -110,13 +110,13 @@ func TestDiskLimaConfigApplier_Apply(t *testing.T) {
 				deps.EXPECT().Arch().Return(runtime.GOARCH)
 			},
 			postRunCheck: func(t *testing.T, fs afero.Fs) {
-				sociFileName := fmt.Sprintf(sociFileNameFormat, sociVersion, runtime.GOARCH)
-				sociDownloadURL := fmt.Sprintf(sociDownloadURLFormat, sociVersion, sociFileName)
-				sociShaSum := sociAMD64Sha256Sum
+				sociFileName := fmt.Sprintf(sociFileNameFormat, SociVersion, runtime.GOARCH)
+				sociDownloadURL := fmt.Sprintf(sociDownloadURLFormat, SociVersion, sociFileName)
+				sociShaSum := SociAMD64Sha256Sum
 				if runtime.GOARCH == "arm64" {
-					sociShaSum = sociARM64Sha256Sum
+					sociShaSum = SociARM64Sha256Sum
 				}
-				sociServiceDownloadURL := fmt.Sprintf(sociServiceDownloadURLFormat, sociVersion)
+				sociServiceDownloadURL := fmt.Sprintf(sociServiceDownloadURLFormat, SociVersion)
 				dockerConfigSetup := `	sudo mkdir -p /etc/systemd/system/soci-snapshotter.service.d/
 	printf '[Service]\nEnvironment="DOCKER_CONFIG=$HOME/.finch-vm-config"\n' | ` +
 					`sudo tee /etc/systemd/system/soci-snapshotter.service.d/override.conf`
@@ -289,13 +289,13 @@ func TestDiskLimaConfigApplier_Apply(t *testing.T) {
 				deps.EXPECT().Arch().Return(runtime.GOARCH)
 			},
 			postRunCheck: func(t *testing.T, fs afero.Fs) {
-				sociFileName := fmt.Sprintf(sociFileNameFormat, sociVersion, runtime.GOARCH)
-				sociDownloadURL := fmt.Sprintf(sociDownloadURLFormat, sociVersion, sociFileName)
-				sociShaSum := sociAMD64Sha256Sum
+				sociFileName := fmt.Sprintf(sociFileNameFormat, SociVersion, runtime.GOARCH)
+				sociDownloadURL := fmt.Sprintf(sociDownloadURLFormat, SociVersion, sociFileName)
+				sociShaSum := SociAMD64Sha256Sum
 				if runtime.GOARCH == "arm64" {
-					sociShaSum = sociARM64Sha256Sum
+					sociShaSum = SociARM64Sha256Sum
 				}
-				sociServiceDownloadURL := fmt.Sprintf(sociServiceDownloadURLFormat, sociVersion)
+				sociServiceDownloadURL := fmt.Sprintf(sociServiceDownloadURLFormat, SociVersion)
 				dockerConfigSetup := `	sudo mkdir -p /etc/systemd/system/soci-snapshotter.service.d/
 	printf '[Service]\nEnvironment="DOCKER_CONFIG=$HOME/.finch-vm-config"\n' | ` +
 					`sudo tee /etc/systemd/system/soci-snapshotter.service.d/override.conf`
@@ -363,13 +363,13 @@ func TestDiskLimaConfigApplier_Apply(t *testing.T) {
 				deps.EXPECT().Arch().Return(runtime.GOARCH)
 			},
 			postRunCheck: func(t *testing.T, fs afero.Fs) {
-				sociFileName := fmt.Sprintf(sociFileNameFormat, sociVersion, runtime.GOARCH)
-				sociDownloadURL := fmt.Sprintf(sociDownloadURLFormat, sociVersion, sociFileName)
-				sociShaSum := sociAMD64Sha256Sum
+				sociFileName := fmt.Sprintf(sociFileNameFormat, SociVersion, runtime.GOARCH)
+				sociDownloadURL := fmt.Sprintf(sociDownloadURLFormat, SociVersion, sociFileName)
+				sociShaSum := SociAMD64Sha256Sum
 				if runtime.GOARCH == "arm64" {
-					sociShaSum = sociARM64Sha256Sum
+					sociShaSum = SociARM64Sha256Sum
 				}
-				sociServiceDownloadURL := fmt.Sprintf(sociServiceDownloadURLFormat, sociVersion)
+				sociServiceDownloadURL := fmt.Sprintf(sociServiceDownloadURLFormat, SociVersion)
 				dockerConfigSetup := `	sudo mkdir -p /etc/systemd/system/soci-snapshotter.service.d/
 	printf '[Service]\nEnvironment="DOCKER_CONFIG=$HOME/.finch-vm-config"\n' | ` +
 					`sudo tee /etc/systemd/system/soci-snapshotter.service.d/override.conf`

pkg/config/lima_config_applier_darwin_test.go

@@ -17,6 +17,15 @@ import (
 	"github.com/runfinch/finch/pkg/path"
 )
 
+var (
+	// EcrVersion will be injected via Makefile.
+	EcrVersion string
+	// EcrAMD64Hash will be injected via Makefile.
+	EcrAMD64Hash string
+	// EcrARM64Hash will be injected via Makefile.
+	EcrARM64Hash string
+)
+
 const (
 	description = "Installing Credential Helper"
 	errMsg      = "Failed to finish installing credential helper"
@@ -66,11 +75,8 @@ func newDeps(
 	configs := map[string]helperConfig{}
 	installFolder := filepath.Join(finchDir, "cred-helpers")
 
-	const versionEcr = "0.9.0"
-	const hashARM64 = "sha256:76aa3bb223d4e64dd4456376334273f27830c8d818efe278ab6ea81cb0844420"
-	const hashAMD64 = "sha256:dd6bd933e439ddb33b9f005ad5575705a243d4e1e3d286b6c82928bcb70e949a"
 	credHelperURLEcr := fmt.Sprintf("https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com"+
-		"/%s/linux-%s/docker-credential-ecr-login", versionEcr, arch)
+		"/%s/linux-%s/docker-credential-ecr-login", EcrVersion, arch)
 
 	hcEcr := helperConfig{
 		binaryName:    "docker-credential-ecr-login",
@@ -80,9 +86,9 @@ func newDeps(
 	}
 
 	if arch == "arm64" {
-		hcEcr.hash = hashARM64
+		hcEcr.hash = EcrARM64Hash
 	} else {
-		hcEcr.hash = hashAMD64
+		hcEcr.hash = EcrAMD64Hash
 	}
 
 	configs["ecr-login"] = hcEcr