From 7672127b4f83f459cf30677c5c1f5805f4ee7103 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Thu, 12 Jun 2025 16:03:04 -0700 Subject: [PATCH 1/7] add ip masq merger to pipeline yaml --- .pipelines/pipeline.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index 8ab29da7e6..7b735653c7 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -121,6 +121,10 @@ stages: arch: amd64 name: azure-ipam os: windows + azure_ip_masq_merger_linux_amd64: + arch: amd64 + name: azure-ip-masq-merger + os: linux cni_linux_amd64: arch: amd64 name: cni @@ -166,6 +170,10 @@ stages: arch: arm64 name: azure-ipam os: linux + azure_ip_masq_merger_linux_arm64: + arch: arm64 + name: azure-ip-masq-merger + os: linux cni_linux_arm64: arch: arm64 name: cni @@ -217,6 +225,9 @@ stages: npm: name: npm platforms: linux/amd64 linux/arm64 windows/amd64 + azure_ip_masq_merger: + name: azure-ip-masq-merger + platforms: linux/amd64 linux/arm64 steps: - template: containers/manifest-template.yaml parameters: From 9f902bbbd27c0560dbb38d37d4fe3e6da96fb340 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 13 Jun 2025 11:49:35 -0700 Subject: [PATCH 2/7] add azure ip masq merger to signed binaries (acn official build) pipeline todo: add azureIpMasqMergerVersion pipeline version somewhere --- .pipelines/run-pipeline.yaml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.pipelines/run-pipeline.yaml b/.pipelines/run-pipeline.yaml index 851bf9289f..b6ae6f1528 100644 --- a/.pipelines/run-pipeline.yaml +++ b/.pipelines/run-pipeline.yaml @@ -39,6 +39,7 @@ stages: IMAGE_REPO_PATH: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.imageRepositoryPath'] ] AZURE_IPAM_VERSION: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.azureIpamVersion'] ] + AZURE_IP_MASQ_MERGER_VERSION: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.azureIpMasqMergerVersion'] ] CNI_VERSION: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.cniVersion'] ] CNS_VERSION: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.cnsVersion'] ] IPV6_HP_BPF_VERSION: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.ipv6HpBpfVersion'] ] @@ -63,6 +64,12 @@ stages: archiveVersion: $(AZURE_IPAM_VERSION) imageTag: $(Build.BuildNumber) packageWithDropGZ: True + azure_ip_masq_merger: + name: azure-ip-masq-merger + extraArgs: '' + archiveName: azure-ip-masq-merger + archiveVersion: $(AZURE_IP_MASQ_MERGER_VERSION) + imageTag: $(Build.BuildNumber) cni: name: cni extraArgs: '--build-arg CNI_AI_PATH=$(CNI_AI_PATH) --build-arg CNI_AI_ID=$(CNI_AI_ID)' @@ -141,6 +148,12 @@ stages: extraArgs: '' imageTag: $(Build.BuildNumber) packageWithDropGZ: True + azure_ip_masq_merger: + name: azure-ip-masq-merger + extraArgs: '' + archiveName: azure-ip-masq-merger + archiveVersion: $(AZURE_IP_MASQ_MERGER_VERSION) + imageTag: $(Build.BuildNumber) cni: name: cni extraArgs: '--build-arg CNI_AI_PATH=$(CNI_AI_PATH) --build-arg CNI_AI_ID=$(CNI_AI_ID)' @@ -177,6 +190,7 @@ stages: IMAGE_REPO_PATH: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.imageRepositoryPath'] ] AZURE_IPAM_VERSION: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.azureIpamVersion'] ] + AZURE_IP_MASQ_MERGER_VERSION: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.azureIpMasqMergerVersion'] ] CNI_VERSION: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.cniVersion'] ] CNS_VERSION: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.cnsVersion'] ] IPV6_HP_BPF_VERSION: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.ipv6HpBpfVersion'] ] @@ -186,6 +200,9 @@ stages: IPAM_LINUX_ARM64_REF: $(IMAGE_REPO_PATH)/linux-arm64/azure-ipam:$(Build.BuildNumber) IPAM_WINDOWS_AMD64_REF: $(IMAGE_REPO_PATH)/windows-amd64/azure-ipam:$(Build.BuildNumber) + IP_MASQ_MERGER_LINUX_AMD64_REF: $(IMAGE_REPO_PATH)/linux-amd64/azure-ip-masq-merger:$(Build.BuildNumber) + IP_MASQ_MERGER_LINUX_ARM64_REF: $(IMAGE_REPO_PATH)/linux-arm64/azure-ip-masq-merger:$(Build.BuildNumber) + CNI_LINUX_AMD64_REF: $(IMAGE_REPO_PATH)/linux-amd64/cni:$(Build.BuildNumber) CNI_LINUX_ARM64_REF: $(IMAGE_REPO_PATH)/linux-arm64/cni:$(Build.BuildNumber) CNI_WINDOWS_AMD64_REF: $(IMAGE_REPO_PATH)/windows-amd64/cni:$(Build.BuildNumber) @@ -216,6 +233,15 @@ stages: imageReference: $(IPAM_LINUX_ARM64_REF) - platform: windows/amd64 imageReference: $(IPAM_WINDOWS_AMD64_REF) + - job: azure_ip_masq_merger + templateContext: + name: azure-ip-masq-merger + image_tag: $(AZURE_IP_MASQ_MERGER_VERSION) + platforms: + - platform: linux/amd64 + imageReference: $(IP_MASQ_MERGER_LINUX_AMD64_REF) + - platform: linux/arm64 + imageReference: $(IP_MASQ_MERGER_LINUX_ARM64_REF) - job: cni templateContext: name: cni From 1f0741bde1041f7881a0bd63eee2dc9f96ace8d2 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 13 Jun 2025 13:25:41 -0700 Subject: [PATCH 3/7] add script and changes to signed pipeline --- .../azure-ip-masq-merger.Dockerfile | 7 +++++++ .pipelines/build/ob-prepare.steps.yaml | 5 ++++- .../build/scripts/azure-ip-masq-merger.sh | 18 ++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 .pipelines/build/dockerfiles/azure-ip-masq-merger.Dockerfile create mode 100644 .pipelines/build/scripts/azure-ip-masq-merger.sh diff --git a/.pipelines/build/dockerfiles/azure-ip-masq-merger.Dockerfile b/.pipelines/build/dockerfiles/azure-ip-masq-merger.Dockerfile new file mode 100644 index 0000000000..1e14819139 --- /dev/null +++ b/.pipelines/build/dockerfiles/azure-ip-masq-merger.Dockerfile @@ -0,0 +1,7 @@ +ARG ARCH + +FROM --platform=linux/${ARCH} mcr.microsoft.com/azurelinux/distroless/minimal:3.0 AS linux +ARG ARTIFACT_DIR + +COPY ${ARTIFACT_DIR}/bin/azure-ip-masq-merger /azure-ip-masq-merger +ENTRYPOINT ["/azure-ip-masq-merger"] diff --git a/.pipelines/build/ob-prepare.steps.yaml b/.pipelines/build/ob-prepare.steps.yaml index bfd6f004b5..802b9f73b1 100644 --- a/.pipelines/build/ob-prepare.steps.yaml +++ b/.pipelines/build/ob-prepare.steps.yaml @@ -1,4 +1,3 @@ - steps: - template: utils/rename-dockerfile-references.steps.yaml parameters: @@ -59,6 +58,10 @@ steps: echo "##vso[task.setvariable variable=azureIpamVersion;isOutput=true]$AZUREIPAMVERSION" echo "azureIpamVersion: $AZUREIPAMVERSION" + AZUREIPMASQMERGERVERSION=$(make azure-ip-masq-merger-version) + echo "##vso[task.setvariable variable=azureIpMasqMergerVersion;isOutput=true]$AZUREIPMASQMERGERVERSION" + echo "azureIpMasqMergerVersion: $AZUREIPMASQMERGERVERSION" + CNIVERSION=$(make cni-version) echo "##vso[task.setvariable variable=cniVersion;isOutput=true]$CNIVERSION" echo "cniVersion: $CNIVERSION" diff --git a/.pipelines/build/scripts/azure-ip-masq-merger.sh b/.pipelines/build/scripts/azure-ip-masq-merger.sh new file mode 100644 index 0000000000..03c2c4efbd --- /dev/null +++ b/.pipelines/build/scripts/azure-ip-masq-merger.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -eux + +[[ $OS =~ windows ]] && { echo "azure-ip-masq-merger is not supported on Windows"; exit 1; } +FILE_EXT='' + +export CGO_ENABLED=0 + +mkdir -p "$OUT_DIR"/bin +mkdir -p "$OUT_DIR"/files + +pushd "$REPO_ROOT"/azure-ip-masq-merger + GOOS="$OS" go build -v -a -trimpath \ + -o "$OUT_DIR"/bin/azure-ip-masq-merger"$FILE_EXT" \ + -ldflags "-X github.com/Azure/azure-container-networking/azure-ip-masq-merger/internal/buildinfo.Version=$AZURE_IP_MASQ_MERGER_VERSION -X main.version=$AZURE_IP_MASQ_MERGER_VERSION" \ + -gcflags="-dwarflocationlists=true" \ + . +popd From bd622aa286dd5a90cf3f5a977225712d6f44f0fb Mon Sep 17 00:00:00 2001 From: QxBytes Date: Mon, 16 Jun 2025 12:27:05 -0700 Subject: [PATCH 4/7] add README to azure ip masq merger --- azure-ip-masq-merger/README.md | 69 ++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 azure-ip-masq-merger/README.md diff --git a/azure-ip-masq-merger/README.md b/azure-ip-masq-merger/README.md new file mode 100644 index 0000000000..fd10afd70a --- /dev/null +++ b/azure-ip-masq-merger/README.md @@ -0,0 +1,69 @@ +# azure-ip-masq-merger + +`azure-ip-masq-merger` is a utility for merging multiple ip-masq-agent configuration files into a single, valid configuration for use in Kubernetes clusters. + +## Description + +The goal of this program is to periodically scan a directory for configuration fragments (YAML or JSON files starting with `ip-masq`), validate and merge them, and write the resulting configuration to a target directory for consumption. This allows us to combine non-masquerade CIDRs and related options between multiple files, for example if we had one ip masq config managed by the cloud provider and another supplied by the user. + +## Usage + +Follow the steps below to build and run the program: + +1. Build the binary using `make`: + ```bash + make azure-ip-masq-merger + ``` + or make an image: + ```bash + make azure-ip-masq-merger-image + ``` + +2. Deploy or copy the binary to your node(s). + +3. Prepare your configuration fragments in the input directory (see below for defaults). Each file should be named with the prefix `ip-masq` and contain valid YAML or JSON for the ip-masq-agent config. + +4. Start the program with: + ```bash + ./azure-ip-masq-merger --input=/etc/config/ --output=/etc/merged-config/ + ``` + - The `--input` flag specifies the directory to scan for config fragments. Default: `/etc/config/` + - The `--output` flag specifies where to write the merged config. Default: `/etc/merged-config/` + +5. The merged configuration will be written to the output directory as `ip-masq-agent`. If no valid configs are found, any existing merged config will be removed. + +## Manual Testing + +You can test the merger locally by creating sample config files in your input directory and running the merger. + +## Configuration File Format + +Each config fragment should be a YAML or JSON file that may have the following fields: +```yaml +nonMasqueradeCIDRs: + - 10.0.0.0/8 + - 192.168.0.0/16 +masqLinkLocal: true +masqLinkLocalIPv6: false +``` +- `nonMasqueradeCIDRs`: List of CIDRs that should not be masqueraded. Appended between configs. +- `masqLinkLocal`: Boolean to enable/disable masquerading of link-local addresses. OR'd between configs. +- `masqLinkLocalIPv6`: Boolean to enable/disable masquerading of IPv6 link-local addresses. OR'd between configs. + +## Debugging + +Logs are output to standard error. Increase verbosity with the `-v` flag: +```bash +./azure-ip-masq-merger -v 2 +``` + +## Development + +To run tests: +```bash +go test ./... +``` +or at the repository level: +```bash +make test-azure-ip-masq-merger +``` From 72cc8c676538c2ebdd397cea8bf834f68da32670 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Mon, 16 Jun 2025 12:27:55 -0700 Subject: [PATCH 5/7] modify dockerfile to be consistent with binary naming tested merged config live cluster with tcpdump --- azure-ip-masq-merger/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azure-ip-masq-merger/Dockerfile b/azure-ip-masq-merger/Dockerfile index bab957aceb..03fcd718dd 100644 --- a/azure-ip-masq-merger/Dockerfile +++ b/azure-ip-masq-merger/Dockerfile @@ -17,10 +17,10 @@ RUN GOOS=$OS CGO_ENABLED=0 go build -a -o /go/bin/ip-masq-merger -trimpath -ldfl # TODO: Replace with scratch later FROM --platform=linux/${ARCH} mcr.microsoft.com/azurelinux/base/core@sha256:b46476be0b5c9691ad20f78871819950c01433bdfad81d72c61618f4a6202b25 AS linux -COPY --from=azure-ip-masq-merger /go/bin/ip-masq-merger ip-masq-merger -ENTRYPOINT [ "/ip-masq-merger" ] +COPY --from=azure-ip-masq-merger /go/bin/ip-masq-merger azure-ip-masq-merger +ENTRYPOINT [ "/azure-ip-masq-merger" ] # skopeo inspect docker://mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image:v1.0.0 --format "{{.Name}}@{{.Digest}}" FROM mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image@sha256:b4c9637e032f667c52d1eccfa31ad8c63f1b035e8639f3f48a510536bf34032b as windows -COPY --from=azure-ip-masq-merger /go/bin/ip-masq-merger ip-masq-merger.exe -ENTRYPOINT [ "/ip-masq-merger.exe" ] +COPY --from=azure-ip-masq-merger /go/bin/ip-masq-merger azure-ip-masq-merger.exe +ENTRYPOINT [ "/azure-ip-masq-merger.exe" ] From d9541a4649de98dc6b950c87baa8b678967f9efb Mon Sep 17 00:00:00 2001 From: QxBytes Date: Wed, 18 Jun 2025 10:35:35 -0700 Subject: [PATCH 6/7] update dockerfiles --- .../build/dockerfiles/azure-ip-masq-merger.Dockerfile | 2 +- azure-ip-masq-merger/Dockerfile | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.pipelines/build/dockerfiles/azure-ip-masq-merger.Dockerfile b/.pipelines/build/dockerfiles/azure-ip-masq-merger.Dockerfile index 1e14819139..733231cefe 100644 --- a/.pipelines/build/dockerfiles/azure-ip-masq-merger.Dockerfile +++ b/.pipelines/build/dockerfiles/azure-ip-masq-merger.Dockerfile @@ -1,6 +1,6 @@ ARG ARCH -FROM --platform=linux/${ARCH} mcr.microsoft.com/azurelinux/distroless/minimal:3.0 AS linux +FROM scratch AS linux ARG ARTIFACT_DIR COPY ${ARTIFACT_DIR}/bin/azure-ip-masq-merger /azure-ip-masq-merger diff --git a/azure-ip-masq-merger/Dockerfile b/azure-ip-masq-merger/Dockerfile index 03fcd718dd..89cbf7f8c6 100644 --- a/azure-ip-masq-merger/Dockerfile +++ b/azure-ip-masq-merger/Dockerfile @@ -15,12 +15,6 @@ WORKDIR /azure-ip-masq-merger COPY ./azure-ip-masq-merger . RUN GOOS=$OS CGO_ENABLED=0 go build -a -o /go/bin/ip-masq-merger -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" . -# TODO: Replace with scratch later -FROM --platform=linux/${ARCH} mcr.microsoft.com/azurelinux/base/core@sha256:b46476be0b5c9691ad20f78871819950c01433bdfad81d72c61618f4a6202b25 AS linux +FROM scratch AS linux COPY --from=azure-ip-masq-merger /go/bin/ip-masq-merger azure-ip-masq-merger ENTRYPOINT [ "/azure-ip-masq-merger" ] - -# skopeo inspect docker://mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image:v1.0.0 --format "{{.Name}}@{{.Digest}}" -FROM mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image@sha256:b4c9637e032f667c52d1eccfa31ad8c63f1b035e8639f3f48a510536bf34032b as windows -COPY --from=azure-ip-masq-merger /go/bin/ip-masq-merger azure-ip-masq-merger.exe -ENTRYPOINT [ "/azure-ip-masq-merger.exe" ] From 97759a584d775a6a8a32bd85a0ab82b3930c05e0 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Mon, 23 Jun 2025 09:50:14 -0700 Subject: [PATCH 7/7] remove unused dockerfile components --- azure-ip-masq-merger/Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/azure-ip-masq-merger/Dockerfile b/azure-ip-masq-merger/Dockerfile index 89cbf7f8c6..fc0592ae69 100644 --- a/azure-ip-masq-merger/Dockerfile +++ b/azure-ip-masq-merger/Dockerfile @@ -1,13 +1,9 @@ ARG ARCH -ARG OS_VERSION ARG OS # skopeo inspect docker://mcr.microsoft.com/oss/go/microsoft/golang:1.23.2-azurelinux3.0 --format "{{.Name}}@{{.Digest}}" FROM --platform=linux/${ARCH} mcr.microsoft.com/oss/go/microsoft/golang@sha256:f1f0cbd464ae4cd9d41176d47f1f9fe16a6965425871f817587314e3a04576ec AS go -# skopeo inspect docker://mcr.microsoft.com/azurelinux/base/core:3.0 --format "{{.Name}}@{{.Digest}}" -FROM --platform=linux/${ARCH} mcr.microsoft.com/azurelinux/base/core@sha256:b46476be0b5c9691ad20f78871819950c01433bdfad81d72c61618f4a6202b25 AS mariner-core - FROM go AS azure-ip-masq-merger ARG OS ARG VERSION