Merge branch 'aws:main' into misleading-404

Author: xabinapal

.github/workflows/build-and-test.yaml

@@ -70,7 +70,7 @@ jobs:
       - name: Set up golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.55.2
+          version: latest
           args: --timeout=5m
           skip-cache: true
 
@@ -124,7 +124,10 @@ jobs:
 
   buildWindows:
     name: Build Windows Binaries
-    runs-on: windows-2019
+    strategy:
+      matrix:
+        version: [2019, 2022]
+    runs-on: windows-${{matrix.version}}
     steps:
     - name: Set up Go 1.x
       uses: actions/setup-go@v2
@@ -149,11 +152,14 @@ jobs:
         Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
         refreshenv
 
-        choco install make && choco install zip && make build-binaries-windows
+        choco install make && choco install zip && make build-binaries-windows-${{matrix.version}}
 
   buildWindowsDocker:
     name: Build Windows Docker Images
-    runs-on: windows-2019
+    strategy:
+      matrix:
+        version: [2019, 2022]
+    runs-on: windows-${{matrix.version}}
     steps:
     - name: Set up Go 1.x
       uses: actions/setup-go@v2
@@ -178,7 +184,7 @@ jobs:
         Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
         refreshenv
 
-        choco install make && make build-docker-images-windows
+        choco install make && make build-docker-images-windows-${{matrix.version}}
 
   e2e:
     name: E2E Tests
@@ -205,4 +211,4 @@ jobs:
         key: gocache
 
     - name: E2E Tests
-      run: test/k8s-local-cluster-test/run-test -v ${{ matrix.k8sVersion }}
+      run: test/k8s-local-cluster-test/run-test -v ${{ matrix.k8sVersion }}

.github/workflows/release.yaml

@@ -36,8 +36,11 @@ jobs:
 
   releaseWindows:
     name: Release Windows
-    runs-on: windows-2019
     needs: [releaseLinux]
+    strategy:
+      matrix:
+        version: [2019, 2022]
+    runs-on: windows-${{matrix.version}}
     steps:
     - name: Set up Go 1.x
       uses: actions/setup-go@v2
@@ -53,7 +56,7 @@ jobs:
         Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
         refreshenv
 
-        choco install make && choco install zip && make release-windows
+        choco install make && choco install zip && make release-windows-${{matrix.version}}
       env:
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -91,4 +94,4 @@ jobs:
       env:
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        AWS_SESSION_TOKEN: ${{ secrets.AWS_SESSION_TOKEN }}
+        AWS_SESSION_TOKEN: ${{ secrets.AWS_SESSION_TOKEN }}

Makefile

@@ -17,7 +17,16 @@ MAKEFILE_PATH = $(dir $(realpath -s $(firstword $(MAKEFILE_LIST))))
 BUILD_DIR_PATH = ${MAKEFILE_PATH}/build
 BIN_DIR = ${MAKEFILE_PATH}/bin
 SUPPORTED_PLATFORMS_LINUX ?= "linux/amd64,linux/arm64"
-SUPPORTED_PLATFORMS_WINDOWS ?= "windows/amd64"
+
+# Each windows version needs a separate make target because each build 
+# needs to happen on a separate GitHub runner
+# A windows version is specified by major-minor-build-revision. 
+# The build number of the OS must match the build number of the container image
+# The revision does not matter for windows 2019 and 2022.
+# Reference: https://learn.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/version-compatibility
+WINDOWS_2019 ?= "windows-10.0.17763.6189/amd64"
+WINDOWS_2022 ?= "windows-10.0.20348.2582/amd64"
+
 BINARY_NAME ?= "node-termination-handler"
 THIRD_PARTY_LICENSES = "${MAKEFILE_PATH}/THIRD_PARTY_LICENSES.md"
 GOLICENSES = $(BIN_DIR)/go-licenses
@@ -48,18 +57,32 @@ docker-run:
 build-docker-images:
 	${MAKEFILE_PATH}/scripts/build-docker-images -p ${SUPPORTED_PLATFORMS_LINUX} -r ${IMG} -v ${VERSION}
 
-build-docker-images-windows:
-	${MAKEFILE_PATH}/scripts/build-docker-images -p ${SUPPORTED_PLATFORMS_WINDOWS} -r ${IMG} -v ${VERSION}
+build-docker-images-windows-2019:
+	${MAKEFILE_PATH}/scripts/build-docker-images -p ${WINDOWS_2019} -r ${IMG} -v ${VERSION}
+
+build-docker-images-windows-2022:
+	${MAKEFILE_PATH}/scripts/build-docker-images -p ${WINDOWS_2022} -r ${IMG} -v ${VERSION}
+
+ecr-public-login:
+	@ECR_REGISTRY=${ECR_REGISTRY} ${MAKEFILE_PATH}/scripts/ecr-public-login
 
 push-docker-images:
 	${MAKEFILE_PATH}/scripts/retag-docker-images -p ${SUPPORTED_PLATFORMS_LINUX} -v ${VERSION} -o ${IMG} -n ${ECR_REPO}
 	@ECR_REGISTRY=${ECR_REGISTRY} ${MAKEFILE_PATH}/scripts/ecr-public-login
 	${MAKEFILE_PATH}/scripts/push-docker-images -p ${SUPPORTED_PLATFORMS_LINUX} -r ${ECR_REPO} -v ${VERSION} -m
 
-push-docker-images-windows:
-	${MAKEFILE_PATH}/scripts/retag-docker-images -p ${SUPPORTED_PLATFORMS_WINDOWS} -v ${VERSION} -o ${IMG} -n ${ECR_REPO}
+amazon-ecr-credential-helper:
 	bash ${MAKEFILE_PATH}/scripts/install-amazon-ecr-credential-helper $(AMAZON_ECR_CREDENTIAL_HELPER_VERSION)
-	${MAKEFILE_PATH}/scripts/push-docker-images -p ${SUPPORTED_PLATFORMS_WINDOWS} -r ${ECR_REPO} -v ${VERSION} -m
+
+push-docker-images-windows-2019:
+	${MAKEFILE_PATH}/scripts/retag-docker-images -p ${WINDOWS_2019} -v ${VERSION} -o ${IMG} -n ${ECR_REPO}
+	bash ${MAKEFILE_PATH}/scripts/install-amazon-ecr-credential-helper $(AMAZON_ECR_CREDENTIAL_HELPER_VERSION)
+	${MAKEFILE_PATH}/scripts/push-docker-images -p ${WINDOWS_2019} -r ${ECR_REPO} -v ${VERSION} -m
+
+push-docker-images-windows-2022:
+	${MAKEFILE_PATH}/scripts/retag-docker-images -p ${WINDOWS_2022} -v ${VERSION} -o ${IMG} -n ${ECR_REPO}
+	bash ${MAKEFILE_PATH}/scripts/install-amazon-ecr-credential-helper $(AMAZON_ECR_CREDENTIAL_HELPER_VERSION)
+	${MAKEFILE_PATH}/scripts/push-docker-images -p ${WINDOWS_2022} -r ${ECR_REPO} -v ${VERSION} -m
 
 push-helm-chart:
 	@ECR_REGISTRY=${ECR_REGISTRY} ${MAKEFILE_PATH}/scripts/helm-login
@@ -122,8 +145,11 @@ helm-validate-chart-versions:
 build-binaries:
 	${MAKEFILE_PATH}/scripts/build-binaries -p ${SUPPORTED_PLATFORMS_LINUX} -v ${VERSION}
 
-build-binaries-windows:
-	${MAKEFILE_PATH}/scripts/build-binaries -p ${SUPPORTED_PLATFORMS_WINDOWS} -v ${VERSION}
+build-binaries-windows-2019:
+	${MAKEFILE_PATH}/scripts/build-binaries -p ${WINDOWS_2019} -v ${VERSION}
+
+build-binaries-windows-2022:
+	${MAKEFILE_PATH}/scripts/build-binaries -p ${WINDOWS_2022} -v ${VERSION}
 
 upload-resources-to-github:
 	${MAKEFILE_PATH}/scripts/upload-resources-to-github
@@ -165,7 +191,9 @@ eks-cluster-test:
 
 release: build-binaries build-docker-images push-docker-images generate-k8s-yaml upload-resources-to-github
 
-release-windows: build-binaries-windows build-docker-images-windows push-docker-images-windows
+release-windows-2019: build-binaries-windows-2019 build-docker-images-windows-2019 push-docker-images-windows-2019
+
+release-windows-2022: build-binaries-windows-2022 build-docker-images-windows-2022 push-docker-images-windows-2022
 
 test: spellcheck shellcheck unit-test e2e-test compatibility-test license-test go-linter helm-version-sync-test helm-lint
 

README.md

@@ -83,8 +83,8 @@ When using the EC2 Console or EC2 API to terminate the instance, a state-change
 | Spot Instance Termination Notifications (ITN) |       ✅        |        ✅        |
 |               Scheduled Events                |       ✅        |        ✅        |
 |       Instance Rebalance Recommendation       |       ✅        |        ✅        |
+|        ASG Termination Lifecycle Hooks        |       ✅        |        ✅        |
 |        AZ Rebalance Recommendation            |       ❌        |        ✅        |
-|        ASG Termination Lifecycle Hooks        |       ❌        |        ✅        |
 |         Instance State Change Events          |       ❌        |        ✅        |
 
 ### Kubernetes Compatibility

cmd/node-termination-handler.go

@@ -16,6 +16,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"github.com/aws/aws-node-termination-handler/pkg/monitor/asglifecycle"
 	"os"
 	"os/signal"
 	"strings"
@@ -53,6 +54,7 @@ import (
 const (
 	scheduledMaintenance    = "Scheduled Maintenance"
 	spotITN                 = "Spot ITN"
+	asgLifecycle            = "ASG Lifecycle"
 	rebalanceRecommendation = "Rebalance Recommendation"
 	sqsEvents               = "SQS Event"
 	timeFormat              = "2006/01/02 15:04:05"
@@ -168,8 +170,9 @@ func main() {
 			err = handleRebootUncordon(nthConfig.NodeName, interruptionEventStore, *node)
 			if err != nil {
 				log.Warn().Err(err).Msgf("Unable to complete the uncordon after reboot workflow on startup, retrying")
+				return false, nil
 			}
-			return false, nil
+			return true, nil
 		})
 		if err != nil {
 			log.Warn().Err(err).Msgf("All retries failed, unable to complete the uncordon after reboot workflow")
@@ -188,6 +191,10 @@ func main() {
 			imdsSpotMonitor := spotitn.NewSpotInterruptionMonitor(imds, interruptionChan, cancelChan, nthConfig.NodeName)
 			monitoringFns[spotITN] = imdsSpotMonitor
 		}
+		if nthConfig.EnableASGLifecycleDraining {
+			asgLifecycleMonitor := asglifecycle.NewASGLifecycleMonitor(imds, interruptionChan, cancelChan, nthConfig.NodeName)
+			monitoringFns[asgLifecycle] = asgLifecycleMonitor
+		}
 		if nthConfig.EnableScheduledEventDraining {
 			imdsScheduledEventMonitor := scheduledevent.NewScheduledEventMonitor(imds, interruptionChan, cancelChan, nthConfig.NodeName)
 			monitoringFns[scheduledMaintenance] = imdsScheduledEventMonitor

config/helm/aws-node-termination-handler/templates/daemonset.linux.yaml

@@ -143,6 +143,8 @@ spec:
             {{- end }}
             - name: ENABLE_SPOT_INTERRUPTION_DRAINING
               value: {{ .Values.enableSpotInterruptionDraining | quote }}
+            - name: ENABLE_ASG_LIFECYCLE_DRAINING
+              value: {{ .Values.enableASGLifecycleDraining | quote }}
             - name: ENABLE_SCHEDULED_EVENT_DRAINING
               value: {{ .Values.enableScheduledEventDraining | quote }}
             - name: ENABLE_REBALANCE_MONITORING

config/helm/aws-node-termination-handler/templates/daemonset.windows.yaml

@@ -143,6 +143,8 @@ spec:
             {{- end }}
             - name: ENABLE_SPOT_INTERRUPTION_DRAINING
               value: {{ .Values.enableSpotInterruptionDraining | quote }}
+            - name: ENABLE_ASG_LIFECYCLE_DRAINING
+              value: {{ .Values.enableASGLifecycleDraining | quote }}
             - name: ENABLE_SCHEDULED_EVENT_DRAINING
               value: {{ .Values.enableScheduledEventDraining | quote }}
             - name: ENABLE_REBALANCE_MONITORING

config/helm/aws-node-termination-handler/values.yaml

@@ -270,6 +270,9 @@ metadataTries: 3
 # enableSpotInterruptionDraining If false, do not drain nodes when the spot interruption termination notice is received. Only used in IMDS mode.
 enableSpotInterruptionDraining: true
 
+# enableASGLifecycleDraining If false, do not drain nodes when ASG target lifecycle state Terminated is received. Only used in IMDS mode.
+enableASGLifecycleDraining: true
+
 # enableScheduledEventDraining If false, do not drain nodes before the maintenance window starts for an EC2 instance scheduled event. Only used in IMDS mode.
 enableScheduledEventDraining: true
 

go.mod

@@ -7,19 +7,19 @@ toolchain go1.22.2
 require (
 	github.com/Masterminds/sprig/v3 v3.2.3
 	github.com/aws/aws-sdk-go v1.55.4
-	github.com/prometheus/client_golang v1.18.0
+	github.com/prometheus/client_golang v1.20.1
 	github.com/rs/zerolog v1.29.0
 	go.opentelemetry.io/contrib/instrumentation/runtime v0.47.0
-	go.opentelemetry.io/otel v1.22.0
-	go.opentelemetry.io/otel/exporters/prometheus v0.45.0
-	go.opentelemetry.io/otel/metric v1.22.0
-	go.opentelemetry.io/otel/sdk/metric v1.22.0
+	go.opentelemetry.io/otel v1.29.0
+	go.opentelemetry.io/otel/exporters/prometheus v0.51.0
+	go.opentelemetry.io/otel/metric v1.29.0
+	go.opentelemetry.io/otel/sdk/metric v1.29.0
 	go.uber.org/multierr v1.11.0
-	golang.org/x/sys v0.18.0
-	k8s.io/api v0.30.3
-	k8s.io/apimachinery v0.30.3
-	k8s.io/client-go v0.30.3
-	k8s.io/kubectl v0.30.3
+	golang.org/x/sys v0.24.0
+	k8s.io/api v0.30.4
+	k8s.io/apimachinery v0.30.4
+	k8s.io/client-go v0.30.4
+	k8s.io/kubectl v0.30.4
 )
 
 require (
@@ -28,26 +28,26 @@ require (
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/chai2010/gettext-go v1.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
-	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
-	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/go-openapi/swag v0.22.4 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/btree v1.0.1 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/google/uuid v1.3.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
 	github.com/huandu/xstrings v1.3.3 // indirect
@@ -56,55 +56,54 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
-	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
-	github.com/moby/spdystream v0.2.0 // indirect
-	github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect
+	github.com/moby/spdystream v0.4.0 // indirect
+	github.com/moby/term v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_model v0.5.0 // indirect
-	github.com/prometheus/common v0.45.0 // indirect
-	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/spf13/cast v1.3.1 // indirect
-	github.com/spf13/cobra v1.7.0 // indirect
+	github.com/spf13/cobra v1.8.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/xlab/treeprint v1.2.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.22.0 // indirect
-	go.opentelemetry.io/otel/trace v1.22.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.29.0 // indirect
+	go.opentelemetry.io/otel/trace v1.29.0 // indirect
 	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
-	golang.org/x/crypto v0.21.0 // indirect
-	golang.org/x/net v0.23.0 // indirect
-	golang.org/x/oauth2 v0.12.0 // indirect
-	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/term v0.18.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/crypto v0.24.0 // indirect
+	golang.org/x/net v0.26.0 // indirect
+	golang.org/x/oauth2 v0.21.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/term v0.21.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.33.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/cli-runtime v0.30.3 // indirect
-	k8s.io/component-base v0.30.3 // indirect
-	k8s.io/klog/v2 v2.120.1 // indirect
+	k8s.io/cli-runtime v0.30.4 // indirect
+	k8s.io/component-base v0.30.4 // indirect
+	k8s.io/klog/v2 v2.130.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
-	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
+	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
 	sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
 )