diff --git a/.gitattributes b/.gitattributes index 980d8c0774d..7ccd8e1ac2b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,2 @@ # Specify generated cluster templates as generated files -**/cluster-template-*.yaml linguist-generated +**/cluster-template-*.yaml linguist-generated \ No newline at end of file diff --git a/.github/workflows/bulwark-gitleaks-pr-validation.yaml b/.github/workflows/bulwark-gitleaks-pr-validation.yaml new file mode 100644 index 00000000000..6e50bc2a1e2 --- /dev/null +++ b/.github/workflows/bulwark-gitleaks-pr-validation.yaml @@ -0,0 +1,36 @@ +name: BulwarkGitLeaks +on: [pull_request] + +concurrency: + group: gitleaks-${{ github.ref }} + cancel-in-progress: true + +jobs: + gitleaks-pr-scan: + runs-on: ubuntu-latest + container: + image: gcr.io/spectro-dev-public/bulwark/gitleaks:latest + env: + REPO: ${{ github.event.repository.name }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITLEAKS_CONFIG: /workspace/config.toml + steps: + + - name: run-bulwark-gitleaks-scan + shell: sh + env: + BRANCH: ${{ github.head_ref || github.ref_name }} + run: /workspace/bulwark -name CodeSASTGitLeaks -target $REPO -tags "branch:$BRANCH,options:--log-opts origin..HEAD" + + - name: check-result + shell: sh + run: | + resultPath=./$REPO/gitleaks.json + cat $resultPath | grep -v \"Match\"\: | grep -v \"Secret\"\: + total_failed_tests=`cat $resultPath | grep \"Fingerprint\"\: | wc -l` + if [ "$total_failed_tests" -gt 0 ]; then + echo "GitLeaks validation check failed with above findings..." + exit 1 + else + echo "GitLeaks validation check passed" + fi \ No newline at end of file diff --git a/.github/workflows/bulwark-gosec-pr-scan.yaml b/.github/workflows/bulwark-gosec-pr-scan.yaml new file mode 100644 index 00000000000..bbc50ca193a --- /dev/null +++ b/.github/workflows/bulwark-gosec-pr-scan.yaml @@ -0,0 +1,35 @@ +name: BulwarkGoSec +on: [pull_request] + +concurrency: + group: gosec-${{ github.ref }} + cancel-in-progress: true + +jobs: + gosec-pr-scan: + runs-on: ubuntu-latest + container: + image: gcr.io/spectro-dev-public/bulwark/gosec:latest + steps: + + - name: run-gosec-scan + shell: sh + env: + BRANCH: ${{ github.head_ref || github.ref_name }} + GO111MODULE: on + run: | + /workspace/bulwark -name CodeSASTGoSec -verbose -target $REPO -tags "branch:$BRANCH,rules:all" + + - name: check-result + shell: sh + run: | + resultPath=$REPO-result.json + issues=$(cat $resultPath | jq -r '.Stats.found') + echo "Found ${issues} issues" + if [ "$issues" -gt 0 ]; then + echo "GoSec SAST scan failed with below findings..." + cat $resultPath + exit 1 + else + echo "GoSec SAST scan passed" + fi diff --git a/.github/workflows/golicense-pr-validation.yaml b/.github/workflows/golicense-pr-validation.yaml new file mode 100644 index 00000000000..129eea9f918 --- /dev/null +++ b/.github/workflows/golicense-pr-validation.yaml @@ -0,0 +1,31 @@ +name: GoLicenses +on: [pull_request] + +concurrency: + group: golicenses-${{ github.ref }} + cancel-in-progress: true + +jobs: + golicense-pr-scan: + runs-on: ubuntu-latest + steps: + - name: install-git + run: sudo apt-get install -y git + + - name: install-golicenses + run: GOBIN=/usr/local/bin go install github.com/google/go-licenses@latest + + - name: checkout + uses: actions/checkout@v3 + + - name: set-github-access + run: | + /usr/bin/git config --global --add url."https://${{ secrets.GH_TOKEN }}:x-oauth-basic@github".insteadOf ssh://git@github + /usr/bin/git config --global --add url."https://${{ secrets.GH_TOKEN }}:x-oauth-basic@github".insteadOf https://github + /usr/bin/git config --global --add url."https://${{ secrets.GH_TOKEN }}:x-oauth-basic@github".insteadOf git@github + + - name: golicense-scan + run: | + go-licenses check --ignore github.com/spectrocloud ./ +# go-licenses check --ignore github.com/spectrocloud ./hack/tools +# go-licenses check --ignore github.com/spectrocloud ./spate/xk6-spate \ No newline at end of file diff --git a/.github/workflows/govulncheck-pr-validation.yaml b/.github/workflows/govulncheck-pr-validation.yaml new file mode 100644 index 00000000000..711dd704be1 --- /dev/null +++ b/.github/workflows/govulncheck-pr-validation.yaml @@ -0,0 +1,33 @@ +name: GoVulnCheck +on: [pull_request] + +concurrency: + group: govulncheck-${{ github.ref }} + cancel-in-progress: true + +jobs: + govulncheck-pr-scan: + runs-on: ubuntu-latest + container: + image: gcr.io/spectro-images-public/golang:alpine + steps: + - name: install-govulncheck + run: GOBIN=/usr/local/bin go install golang.org/x/vuln/cmd/govulncheck@latest + + - name: checkout + uses: actions/checkout@v3 + + - name: set-github-access + run: | + /usr/bin/git config --global --add url."https://${{ secrets.GH_TOKEN }}:x-oauth-basic@github".insteadOf ssh://git@github + /usr/bin/git config --global --add url."https://${{ secrets.GH_TOKEN }}:x-oauth-basic@github".insteadOf https://github + /usr/bin/git config --global --add url."https://${{ secrets.GH_TOKEN }}:x-oauth-basic@github".insteadOf git@github + + - name: govulncheck-scan + run: | + go version + govulncheck -mode source ./ + govulncheck -mode source ./hack/tools + + + diff --git a/.github/workflows/spectro-release.yaml b/.github/workflows/spectro-release.yaml new file mode 100644 index 00000000000..4830447fa47 --- /dev/null +++ b/.github/workflows/spectro-release.yaml @@ -0,0 +1,90 @@ +name: Spectro Release +run-name: Release for Cluster API Azure ${{ github.event.inputs.release_version }} +on: + workflow_dispatch: + inputs: + release_version: + description: 'Cluster API Version to Build' + required: true + default: '0.0.0' + rel_type: + type: choice + description: Type of release + options: + - release + - rc +jobs: + builder: + # edge-runner machine group is a bunch of machines in US Datacenter + runs-on: ubuntu-latest + # Initialize all secrets required for the job + # Ensure that the credentials are provided as encrypted secrets + env: + SPECTRO_VERSION: ${{ github.event.inputs.release_version }} + LEGACY_REGISTRY: gcr.io/spectro-images-public/release/cluster-api-azure + FIPS_REGISTRY: gcr.io/spectro-images-public/release-fips/cluster-api-azure + steps: + - + uses: mukunku/tag-exists-action@v1.2.0 + id: checkTag + with: + tag: v${{ github.event.inputs.release_version }}-spectro + - + if: ${{ steps.checkTag.outputs.exists == 'true' }} + run: | + echo "Tag already exists for v${{ github.event.inputs.release_version }}-spectro..." + exit 1 + - + if: ${{ github.event.inputs.rel_type == 'rc' }} + run: | + echo "LEGACY_REGISTRY=gcr.io/spectro-dev-public/release/cluster-api-azure" >> $GITHUB_ENV + echo "FIPS_REGISTRY=gcr.io/spectro-dev-public/release-fips/cluster-api-azure" >> $GITHUB_ENV + - + name: Install SSH key for remote docker build + uses: shimataro/ssh-key-action@v2 + with: + key: ${{ secrets.SSH_KEY }} + known_hosts: ${{ secrets.KNOWN_HOSTS }} + if_key_exists: fail + - + uses: actions/checkout@v3 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Login to private registry + uses: docker/login-action@v1 + with: + registry: ${{ secrets.REGISTRY_URL }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + - + name: Build Image + env: + REGISTRY: ${{ env.LEGACY_REGISTRY }} + run: | + make docker-build-all + make docker-push-all + - + name: Build Image - FIPS Mode + env: + REGISTRY: ${{ env.FIPS_REGISTRY }} + FIPS_ENABLE: yes + ALL_ARCH: amd64 + run: | + make docker-build-all + make docker-push-all + - + name: Create Release + if: ${{ github.event.inputs.rel_type == 'release' }} + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: v${{ github.event.inputs.release_version }}-spectro + release_name: Release v${{ github.event.inputs.release_version }}-spectro + body: | + Release version v${{ github.event.inputs.release_version }}-spectro + draft: false + prerelease: false diff --git a/Dockerfile b/Dockerfile index 9dffa8d2064..4b2b9b42a38 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,3 @@ -# syntax=docker/dockerfile:1.4 - # Copyright 2019 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,17 +13,26 @@ # limitations under the License. # Build the manager binary -FROM golang:1.17 as builder -WORKDIR /workspace - +ARG BUILDER_GOLANG_VERSION +# First stage: build the executable. +FROM --platform=$TARGETPLATFORM gcr.io/spectro-images-public/golang:${BUILDER_GOLANG_VERSION}-alpine as toolchain # Run this with docker build --build_arg $(go env GOPROXY) to override the goproxy ARG goproxy=https://proxy.golang.org ENV GOPROXY=$goproxy +# FIPS +ARG CRYPTO_LIB +ENV GOEXPERIMENT=${CRYPTO_LIB:+boringcrypto} + +FROM toolchain as builder +WORKDIR /workspace + +RUN apk update +RUN apk add git gcc g++ curl binutils-gold + # Copy the Go Modules manifests COPY go.mod go.mod COPY go.sum go.sum - # Cache deps before building and copying source so that we don't need to re-download as much # and so that source changes don't invalidate our downloaded layer RUN --mount=type=cache,target=/go/pkg/mod \ @@ -34,22 +41,33 @@ RUN --mount=type=cache,target=/go/pkg/mod \ # Copy the sources COPY ./ ./ +# Build +ARG package=. +ARG ARCH + # Cache the go build into the the Go’s compiler cache folder so we take benefits of compiler caching across docker build calls RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ - go build . + if [ ${CRYPTO_LIB} ]; \ + then \ + GOARCH=${ARCH} go-build-fips.sh . ;\ + else \ + GOARCH=${ARCH} go-build-static.sh . ;\ + fi -# Build -ARG package=. -ARG ARCH -ARG ldflags # Do not force rebuild of up-to-date packages (do not use -a) and use the compiler cache folder RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ - CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \ - go build -ldflags "${ldflags} -extldflags '-static'" \ - -o manager ${package} + if [ ${CRYPTO_LIB} ]; \ + then \ + GOARCH=${ARCH} go-build-fips.sh -a -o manager ${package};\ + else \ + GOARCH=${ARCH} go-build-static.sh -a -o manager ${package} ;\ + fi +RUN if [ "${CRYPTO_LIB}" ]; then assert-static.sh manager; fi +RUN if [ "${CRYPTO_LIB}" ]; then assert-fips.sh manager; fi +RUN scan-govulncheck.sh manager # Production image FROM gcr.io/distroless/static:nonroot diff --git a/Makefile b/Makefile index b943699b48b..849f773d1f7 100644 --- a/Makefile +++ b/Makefile @@ -15,13 +15,15 @@ # If you update this file, please follow # https://suva.sh/posts/well-documented-makefiles + # Ensure Make is run with bash shell as some syntax below is bash-specific SHELL:=/usr/bin/env bash .DEFAULT_GOAL:=help GOPATH := $(shell go env GOPATH) -GOARCH := $(shell go env GOARCH) +#GOARCH := $(shell go env GOARCH) +GOARCH := amd64 GOOS := $(shell go env GOOS) GOPROXY := $(shell go env GOPROXY) ifeq ($(GOPROXY),) @@ -120,14 +122,28 @@ KUBE_APISERVER=$(TOOLS_BIN_DIR)/kube-apiserver ETCD=$(TOOLS_BIN_DIR)/etcd # Define Docker related variables. Releases should modify and double check these vars. -REGISTRY ?= gcr.io/$(shell gcloud config get-value project) +# Fips Flags +FIPS_ENABLE ?= "" +BUILDER_GOLANG_VERSION ?= 1.21 +BUILD_ARGS = --build-arg CRYPTO_LIB=${FIPS_ENABLE} --build-arg BUILDER_GOLANG_VERSION=${BUILDER_GOLANG_VERSION} + +RELEASE_LOC := release +ifeq ($(FIPS_ENABLE),yes) + RELEASE_LOC := release-fips +endif + +SPECTRO_VERSION ?= 4.0.0-dev +TAG ?= v1.3.2-spectro-${SPECTRO_VERSION} +ARCH ?= amd64 +# ALL_ARCH = amd64 arm arm64 ppc64le s390x +ALL_ARCH = arm64 amd64 + +REGISTRY ?= gcr.io/spectro-dev-public/$(USER)/${RELEASE_LOC} + STAGING_REGISTRY := gcr.io/k8s-staging-cluster-api-azure PROD_REGISTRY := registry.k8s.io/cluster-api-azure IMAGE_NAME ?= cluster-api-azure-controller CONTROLLER_IMG ?= $(REGISTRY)/$(IMAGE_NAME) -TAG ?= dev -ARCH ?= $(GOARCH) -ALL_ARCH = amd64 arm arm64 ppc64le s390x # Allow overriding manifest generation destination directory MANIFEST_ROOT ?= config @@ -339,8 +355,8 @@ docker-pull-prerequisites: ## Pull prerequisites for building controller-manager .PHONY: docker-build docker-build: docker-pull-prerequisites ## Build the docker image for controller-manager. - DOCKER_BUILDKIT=1 docker build --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg ldflags="$(LDFLAGS)" . -t $(CONTROLLER_IMG)-$(ARCH):$(TAG) - $(MAKE) set-manifest-image MANIFEST_IMG=$(CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./config/default/manager_image_patch.yaml" + docker buildx build --load --platform linux/${ARCH} ${BUILD_ARGS} --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg ldflags="$(LDFLAGS)" . -t $(CONTROLLER_IMG)-$(ARCH):$(TAG) + $(MAKE) set-manifest-image MANIFEST_IMG=$(CONTROLLER_IMG) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./config/default/manager_image_patch.yaml" $(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./config/default/manager_pull_policy.yaml" .PHONY: docker-push diff --git a/api/v1beta1/azurecluster_default.go b/api/v1beta1/azurecluster_default.go index 71ff0f20282..a00b963c09d 100644 --- a/api/v1beta1/azurecluster_default.go +++ b/api/v1beta1/azurecluster_default.go @@ -18,7 +18,7 @@ package v1beta1 import ( "fmt" - + "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-02-01/network" "k8s.io/utils/pointer" ) @@ -106,7 +106,7 @@ func (c *AzureCluster) setSubnetDefaults() { var nodeSubnetFound bool var nodeSubnetCounter int for i, subnet := range c.Spec.NetworkSpec.Subnets { - if subnet.Role == SubnetNode { + if subnet.Role == SubnetNode || subnet.Role == SubnetAll { nodeSubnetCounter++ nodeSubnetFound = true if subnet.Name == "" { @@ -202,7 +202,7 @@ func (c *AzureCluster) SetNodeOutboundLBDefaults() { var needsOutboundLB bool for _, subnet := range c.Spec.NetworkSpec.Subnets { - if subnet.Role == SubnetNode && !subnet.IsNatGatewayEnabled() { + if (subnet.Role == SubnetNode || subnet.Role == SubnetAll) && !subnet.IsNatGatewayEnabled() { needsOutboundLB = true break } @@ -304,6 +304,9 @@ func (lb *LoadBalancerClassSpec) setAPIServerLBDefaults() { if lb.SKU == "" { lb.SKU = SKUStandard } + if lb.IPAllocationMethod == "" { + lb.IPAllocationMethod = string(network.IPAllocationMethodDynamic) + } if lb.IdleTimeoutInMinutes == nil { lb.IdleTimeoutInMinutes = pointer.Int32Ptr(DefaultOutboundRuleIdleTimeoutInMinutes) } diff --git a/api/v1beta1/azurecluster_validation.go b/api/v1beta1/azurecluster_validation.go index e6e5a5254f5..ade8f0f400d 100644 --- a/api/v1beta1/azurecluster_validation.go +++ b/api/v1beta1/azurecluster_validation.go @@ -135,7 +135,7 @@ func validateNetworkSpec(networkSpec NetworkSpec, old NetworkSpec, fldPath *fiel var oneSubnetWithoutNatGateway bool for _, subnet := range networkSpec.Subnets { - if subnet.Role == SubnetNode && !subnet.IsNatGatewayEnabled() { + if (subnet.Role == SubnetNode || subnet.Role == SubnetAll) && !subnet.IsNatGatewayEnabled() { oneSubnetWithoutNatGateway = true break } @@ -171,20 +171,26 @@ func validateSubnets(subnets Subnets, vnet VnetSpec, fldPath *field.Path) field. "control-plane": false, "node": false, } + subnetAllRoleSpecified := false for i, subnet := range subnets { if err := validateSubnetName(subnet.Name, fldPath.Index(i).Child("name")); err != nil { allErrs = append(allErrs, err) } - if _, ok := subnetNames[subnet.Name]; ok { - allErrs = append(allErrs, field.Duplicate(fldPath, subnet.Name)) - } - subnetNames[subnet.Name] = true - for role := range requiredSubnetRoles { - if role == string(subnet.Role) { - requiredSubnetRoles[role] = true + if subnet.Role == SubnetAll { + subnetAllRoleSpecified = true + } else { + if _, ok := subnetNames[subnet.Name]; ok { + allErrs = append(allErrs, field.Duplicate(fldPath, subnet.Name)) + } + subnetNames[subnet.Name] = true + for role := range requiredSubnetRoles { + if role == string(subnet.Role) { + requiredSubnetRoles[role] = true + } } } + for _, rule := range subnet.SecurityGroup.SecurityRules { if err := validateSecurityRule( rule, @@ -195,10 +201,12 @@ func validateSubnets(subnets Subnets, vnet VnetSpec, fldPath *field.Path) field. } allErrs = append(allErrs, validateSubnetCIDR(subnet.CIDRBlocks, vnet.CIDRBlocks, fldPath.Index(i).Child("cidrBlocks"))...) } - for k, v := range requiredSubnetRoles { - if !v { - allErrs = append(allErrs, field.Required(fldPath, - fmt.Sprintf("required role %s not included in provided subnets", k))) + if !subnetAllRoleSpecified { + for k, v := range requiredSubnetRoles { + if !v { + allErrs = append(allErrs, field.Required(fldPath, + fmt.Sprintf("required role %s not included in provided subnets", k))) + } } } return allErrs @@ -339,7 +347,7 @@ func validateAPIServerLB(lb LoadBalancerSpec, old LoadBalancerSpec, cidrs []stri fldPath.Child("frontendIPConfigs").Index(0).Child("privateIP")); err != nil { allErrs = append(allErrs, err) } - if len(old.FrontendIPs) != 0 && old.FrontendIPs[0].PrivateIPAddress != lb.FrontendIPs[0].PrivateIPAddress { + if lb.IPAllocationMethod == "Static" && len(old.FrontendIPs) != 0 && old.FrontendIPs[0].PrivateIPAddress == "" && old.FrontendIPs[0].PrivateIPAddress != lb.FrontendIPs[0].PrivateIPAddress { allErrs = append(allErrs, field.Forbidden(fldPath.Child("name"), "API Server load balancer private IP should not be modified after AzureCluster creation.")) } } diff --git a/api/v1beta1/azureclustertemplate_default.go b/api/v1beta1/azureclustertemplate_default.go index fb5d42f82da..3bf3d207086 100644 --- a/api/v1beta1/azureclustertemplate_default.go +++ b/api/v1beta1/azureclustertemplate_default.go @@ -65,7 +65,7 @@ func (c *AzureClusterTemplate) setSubnetsTemplateDefaults() { var nodeSubnetFound bool var nodeSubnetCounter int for i, subnet := range c.Spec.Template.Spec.NetworkSpec.Subnets { - if subnet.Role == SubnetNode { + if subnet.Role == SubnetNode || subnet.Role == SubnetAll { nodeSubnetCounter++ nodeSubnetFound = true subnet.SubnetClassSpec.setDefaults(fmt.Sprintf(DefaultNodeSubnetCIDRPattern, nodeSubnetCounter)) @@ -93,7 +93,7 @@ func (c *AzureClusterTemplate) setNodeOutboundLBDefaults() { var needsOutboundLB bool for _, subnet := range c.Spec.Template.Spec.NetworkSpec.Subnets { - if subnet.Role == SubnetNode && !subnet.IsNatGatewayEnabled() { + if (subnet.Role == SubnetNode || subnet.Role == SubnetAll) && !subnet.IsNatGatewayEnabled() { needsOutboundLB = true break } diff --git a/api/v1beta1/azureclustertemplate_validation.go b/api/v1beta1/azureclustertemplate_validation.go index aaafc1454b4..661bf408bea 100644 --- a/api/v1beta1/azureclustertemplate_validation.go +++ b/api/v1beta1/azureclustertemplate_validation.go @@ -58,7 +58,7 @@ func (c *AzureClusterTemplate) validateClusterTemplateSpec() field.ErrorList { var oneSubnetWithoutNatGateway bool networkSpec := c.Spec.Template.Spec.NetworkSpec for _, subnet := range networkSpec.Subnets { - if subnet.Role == SubnetNode && !subnet.IsNatGatewayEnabled() { + if (subnet.Role == SubnetNode || subnet.Role == SubnetAll) && !subnet.IsNatGatewayEnabled() { oneSubnetWithoutNatGateway = true break } diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index 7907eecf493..62aaa36fe58 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -28,6 +28,8 @@ const ( Node string = "node" // Bastion subnet label. Bastion string = "bastion" + // All subnet label. + All string = "all" ) // Futures is a slice of Future. @@ -552,6 +554,9 @@ const ( // SubnetBastion defines a Bastion subnet role. SubnetBastion = SubnetRole(Bastion) + + // SubnetAll defines a role that can be used for both Kubernetes control plane node and Kubernetes workload node. + SubnetAll = SubnetRole(All) ) // SubnetSpec configures an Azure subnet. @@ -582,17 +587,17 @@ type SubnetSpec struct { // GetControlPlaneSubnet returns the cluster control plane subnet. func (n *NetworkSpec) GetControlPlaneSubnet() (SubnetSpec, error) { for _, sn := range n.Subnets { - if sn.Role == SubnetControlPlane { + if sn.Role == SubnetControlPlane || sn.Role == SubnetAll { return sn, nil } } - return SubnetSpec{}, errors.Errorf("no subnet found with role %s", SubnetControlPlane) + return SubnetSpec{}, errors.Errorf("no subnet found with role %s or %s", SubnetControlPlane, SubnetAll) } // UpdateControlPlaneSubnet updates the cluster control plane subnet. func (n *NetworkSpec) UpdateControlPlaneSubnet(subnet SubnetSpec) { for i, sn := range n.Subnets { - if sn.Role == SubnetControlPlane { + if sn.Role == SubnetControlPlane || sn.Role == SubnetAll { n.Subnets[i] = subnet } } @@ -601,7 +606,7 @@ func (n *NetworkSpec) UpdateControlPlaneSubnet(subnet SubnetSpec) { // UpdateNodeSubnet updates the cluster node subnet. func (n *NetworkSpec) UpdateNodeSubnet(subnet SubnetSpec) { for i, sn := range n.Subnets { - if sn.Role == SubnetNode { + if sn.Role == SubnetNode || sn.Role == SubnetAll { n.Subnets[i] = subnet } } diff --git a/api/v1beta1/types_class.go b/api/v1beta1/types_class.go index bace21a7b88..26675a02d74 100644 --- a/api/v1beta1/types_class.go +++ b/api/v1beta1/types_class.go @@ -73,7 +73,7 @@ type VnetClassSpec struct { // SubnetClassSpec defines the SubnetSpec properties that may be shared across several Azure clusters. type SubnetClassSpec struct { // Role defines the subnet role (eg. Node, ControlPlane) - // +kubebuilder:validation:Enum=node;control-plane;bastion + // +kubebuilder:validation:Enum=node;control-plane;bastion;all Role SubnetRole `json:"role"` // CIDRBlocks defines the subnet's address space, specified as one or more address prefixes in CIDR notation. @@ -87,6 +87,8 @@ type LoadBalancerClassSpec struct { SKU SKU `json:"sku,omitempty"` // +optional Type LBType `json:"type,omitempty"` + // +optional + IPAllocationMethod string `json:"ipAllocationMethod,omitempty"` // IdleTimeoutInMinutes specifies the timeout for the TCP idle connection. // +optional IdleTimeoutInMinutes *int32 `json:"idleTimeoutInMinutes,omitempty"` diff --git a/api/v1beta1/types_template.go b/api/v1beta1/types_template.go index 1b5a20481a3..128fcba79cb 100644 --- a/api/v1beta1/types_template.go +++ b/api/v1beta1/types_template.go @@ -58,17 +58,17 @@ type NetworkTemplateSpec struct { // GetControlPlaneSubnetTemplate returns the cluster control plane subnet template. func (n *NetworkTemplateSpec) GetControlPlaneSubnetTemplate() (SubnetTemplateSpec, error) { for _, sn := range n.Subnets { - if sn.Role == SubnetControlPlane { + if sn.Role == SubnetControlPlane || sn.Role == SubnetAll { return sn, nil } } - return SubnetTemplateSpec{}, errors.Errorf("no subnet template found with role %s", SubnetControlPlane) + return SubnetTemplateSpec{}, errors.Errorf("no subnet template found with role %s or %s", SubnetControlPlane, SubnetAll) } // UpdateControlPlaneSubnet updates the cluster control plane subnet. func (n *NetworkTemplateSpec) UpdateControlPlaneSubnetTemplate(subnet SubnetTemplateSpec) { for i, sn := range n.Subnets { - if sn.Role == SubnetControlPlane { + if sn.Role == SubnetControlPlane || sn.Role == SubnetAll { n.Subnets[i] = subnet } } diff --git a/azure/const.go b/azure/const.go index 9c4ac179ab2..c45b0d1d461 100644 --- a/azure/const.go +++ b/azure/const.go @@ -28,4 +28,8 @@ const ( // See https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ // for annotation formatting rules. RGTagsLastAppliedAnnotation = "sigs.k8s.io/cluster-api-provider-azure-last-applied-tags-rg" + + // ReplicasManagedByAutoscalerAnnotation is set to true in the corresponding capi machine pool + // when an external autoscaler manages the node count of the associated machine pool. + ReplicasManagedByAutoscalerAnnotation = "cluster.x-k8s.io/replicas-managed-by-autoscaler" ) diff --git a/azure/converters/managedagentpool.go b/azure/converters/managedagentpool.go index 370ef823cfc..95e9d06756e 100644 --- a/azure/converters/managedagentpool.go +++ b/azure/converters/managedagentpool.go @@ -51,7 +51,7 @@ func AgentPoolToContainerServiceAgentPool(pool azure.AgentPoolSpec) containerser return containerservice.AgentPool{ ManagedClusterAgentPoolProfileProperties: &containerservice.ManagedClusterAgentPoolProfileProperties{ VMSize: &pool.SKU, - OsType: containerservice.OSTypeLinux, + OsType: containerservice.OSType(to.String(pool.OSType)), OsDiskSizeGB: &pool.OSDiskSizeGB, Count: &pool.Replicas, Type: containerservice.AgentPoolTypeVirtualMachineScaleSets, diff --git a/azure/defaults.go b/azure/defaults.go index 53c83781e2c..bfd4b531129 100644 --- a/azure/defaults.go +++ b/azure/defaults.go @@ -164,7 +164,10 @@ func GeneratePrivateDNSZoneName(clusterName string) string { } // GeneratePrivateFQDN generates the FQDN for a private API Server based on the private DNS zone name. -func GeneratePrivateFQDN(zoneName string) string { +func GeneratePrivateFQDN(zoneName, clusterName string, isPrivateDNSZoneName bool) string { + if isPrivateDNSZoneName { + return fmt.Sprintf("%s.%s.%s", PrivateAPIServerHostname, clusterName, zoneName) + } return fmt.Sprintf("%s.%s", PrivateAPIServerHostname, zoneName) } diff --git a/azure/scope/cluster.go b/azure/scope/cluster.go index 4f74556af36..3ba3d131bc7 100644 --- a/azure/scope/cluster.go +++ b/azure/scope/cluster.go @@ -54,6 +54,7 @@ type ClusterScopeParams struct { Client client.Client Cluster *clusterv1.Cluster AzureCluster *infrav1.AzureCluster + Cache *ClusterCache } // NewClusterScope creates a new Scope from the supplied parameters. @@ -85,6 +86,10 @@ func NewClusterScope(ctx context.Context, params ClusterScopeParams) (*ClusterSc } } + if params.Cache == nil { + params.Cache = &ClusterCache{} + } + helper, err := patch.NewHelper(params.AzureCluster, params.Client) if err != nil { return nil, errors.Errorf("failed to init patch helper: %v", err) @@ -96,6 +101,7 @@ func NewClusterScope(ctx context.Context, params ClusterScopeParams) (*ClusterSc Cluster: params.Cluster, AzureCluster: params.AzureCluster, patchHelper: helper, + cache: params.Cache, }, nil } @@ -103,12 +109,18 @@ func NewClusterScope(ctx context.Context, params ClusterScopeParams) (*ClusterSc type ClusterScope struct { Client client.Client patchHelper *patch.Helper + cache *ClusterCache AzureClients Cluster *clusterv1.Cluster AzureCluster *infrav1.AzureCluster } +// ClusterCache stores ClusterCache data locally so we don't have to hit the API multiple times within the same reconcile loop. +type ClusterCache struct { + isVnetManaged *bool +} + // BaseURI returns the Azure ResourceManagerEndpoint. func (s *ClusterScope) BaseURI() string { return s.ResourceManagerEndpoint @@ -187,6 +199,7 @@ func (s *ClusterScope) LBSpecs() []azure.ResourceSpecGetter { Type: s.APIServerLB().Type, SKU: infrav1.SKUStandard, Role: infrav1.APIServerRole, + IPAllocationMethod: s.APIServerLB().IPAllocationMethod, BackendPoolName: s.APIServerLBPoolName(s.APIServerLB().Name), IdleTimeoutInMinutes: s.APIServerLB().IdleTimeoutInMinutes, AdditionalTags: s.AdditionalTags(), @@ -428,7 +441,7 @@ func (s *ClusterScope) PrivateDNSSpec() (zoneSpec azure.ResourceSpecGetter, link records := make([]azure.ResourceSpecGetter, 1) records[0] = privatedns.RecordSpec{ Record: infrav1.AddressRecord{ - Hostname: azure.PrivateAPIServerHostname, + Hostname: s.getHostName(), IP: s.APIServerPrivateIP(), }, ZoneName: s.GetPrivateDNSZoneName(), @@ -441,6 +454,13 @@ func (s *ClusterScope) PrivateDNSSpec() (zoneSpec azure.ResourceSpecGetter, link return nil, nil, nil } +func (s *ClusterScope) getHostName() string { + if s.AzureCluster.Spec.NetworkSpec.PrivateDNSZoneName != "" { + return fmt.Sprintf("%s.%s", azure.PrivateAPIServerHostname, s.AzureCluster.Name) + } + return azure.PrivateAPIServerHostname +} + // IsAzureBastionEnabled returns true if the azure bastion is enabled. func (s *ClusterScope) IsAzureBastionEnabled() bool { return s.AzureCluster.Spec.BastionSpec.AzureBastion != nil @@ -477,7 +497,12 @@ func (s *ClusterScope) Vnet() *infrav1.VnetSpec { // IsVnetManaged returns true if the vnet is managed. func (s *ClusterScope) IsVnetManaged() bool { - return s.Vnet().ID == "" || s.Vnet().Tags.HasOwned(s.ClusterName()) + if s.cache.isVnetManaged != nil { + return to.Bool(s.cache.isVnetManaged) + } + isVnetManaged := s.Vnet().ID == "" || s.Vnet().Tags.HasOwned(s.ClusterName()) + s.cache.isVnetManaged = to.BoolPtr(isVnetManaged) + return isVnetManaged } // IsIPv6Enabled returns true if IPv6 is enabled. @@ -754,7 +779,11 @@ func (s *ClusterScope) APIServerPort() int32 { // APIServerHost returns the hostname used to reach the API server. func (s *ClusterScope) APIServerHost() string { if s.IsAPIServerPrivate() { - return azure.GeneratePrivateFQDN(s.GetPrivateDNSZoneName()) + var isPrivateDNSZoneName bool + if len(s.AzureCluster.Spec.NetworkSpec.PrivateDNSZoneName) > 0 { + isPrivateDNSZoneName = true + } + return azure.GeneratePrivateFQDN(s.GetPrivateDNSZoneName(), s.AzureCluster.Name, isPrivateDNSZoneName) } return s.APIServerPublicIP().DNSName } diff --git a/azure/scope/cluster_test.go b/azure/scope/cluster_test.go index 05ed0c85f89..f407b9b3641 100644 --- a/azure/scope/cluster_test.go +++ b/azure/scope/cluster_test.go @@ -606,6 +606,7 @@ func TestRouteTableSpecs(t *testing.T) { }, }, }, + cache: &ClusterCache{}, }, want: nil, }, @@ -641,6 +642,7 @@ func TestRouteTableSpecs(t *testing.T) { }, }, }, + cache: &ClusterCache{}, }, want: []azure.ResourceSpecGetter{ &routetables.RouteTableSpec{ @@ -688,6 +690,7 @@ func TestNatGatewaySpecs(t *testing.T) { }, }, }, + cache: &ClusterCache{}, }, want: nil, }, @@ -730,6 +733,7 @@ func TestNatGatewaySpecs(t *testing.T) { }, }, }, + cache: &ClusterCache{}, }, want: []azure.ResourceSpecGetter{ &natgateways.NatGatewaySpec{ @@ -800,6 +804,7 @@ func TestNatGatewaySpecs(t *testing.T) { }, }, }, + cache: &ClusterCache{}, }, want: []azure.ResourceSpecGetter{ &natgateways.NatGatewaySpec{ @@ -869,6 +874,7 @@ func TestNatGatewaySpecs(t *testing.T) { }, }, }, + cache: &ClusterCache{}, }, want: []azure.ResourceSpecGetter{ &natgateways.NatGatewaySpec{ @@ -946,6 +952,7 @@ func TestNSGSpecs(t *testing.T) { }, }, }, + cache: &ClusterCache{}, }, want: []azure.ResourceSpecGetter{ &securitygroups.NSGSpec{ @@ -991,6 +998,7 @@ func TestSubnetSpecs(t *testing.T) { }, }, }, + cache: &ClusterCache{}, }, want: []azure.ResourceSpecGetter{}, }, @@ -1052,6 +1060,7 @@ func TestSubnetSpecs(t *testing.T) { }, }, }, + cache: &ClusterCache{}, }, want: []azure.ResourceSpecGetter{ &subnets.SubnetSpec{ @@ -1154,6 +1163,7 @@ func TestSubnetSpecs(t *testing.T) { }, }, }, + cache: &ClusterCache{}, }, want: []azure.ResourceSpecGetter{ &subnets.SubnetSpec{ @@ -1196,6 +1206,122 @@ func TestSubnetSpecs(t *testing.T) { } } +func TestIsVnetManaged(t *testing.T) { + tests := []struct { + name string + clusterScope ClusterScope + want bool + }{ + { + name: "VNET ID is empty", + clusterScope: ClusterScope{ + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster", + }, + }, + AzureCluster: &infrav1.AzureCluster{ + Spec: infrav1.AzureClusterSpec{ + NetworkSpec: infrav1.NetworkSpec{ + Vnet: infrav1.VnetSpec{ + ID: "", + }, + }, + }, + }, + cache: &ClusterCache{}, + }, + want: true, + }, + { + name: "Wrong tags", + clusterScope: ClusterScope{ + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster", + }, + }, + AzureCluster: &infrav1.AzureCluster{ + Spec: infrav1.AzureClusterSpec{ + NetworkSpec: infrav1.NetworkSpec{ + Vnet: infrav1.VnetSpec{ + ID: "my-id", + VnetClassSpec: infrav1.VnetClassSpec{Tags: map[string]string{ + "key": "value", + }}, + }, + }, + }, + }, + cache: &ClusterCache{}, + }, + want: false, + }, + { + name: "Has owning tags", + clusterScope: ClusterScope{ + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster", + }, + }, + AzureCluster: &infrav1.AzureCluster{ + Spec: infrav1.AzureClusterSpec{ + NetworkSpec: infrav1.NetworkSpec{ + Vnet: infrav1.VnetSpec{ + ID: "my-id", + VnetClassSpec: infrav1.VnetClassSpec{Tags: map[string]string{ + "sigs.k8s.io_cluster-api-provider-azure_cluster_my-cluster": "owned", + }}, + }, + }, + }, + }, + cache: &ClusterCache{}, + }, + want: true, + }, + { + name: "Has cached value of false", + clusterScope: ClusterScope{ + AzureCluster: &infrav1.AzureCluster{ + Spec: infrav1.AzureClusterSpec{}, + }, + cache: &ClusterCache{ + isVnetManaged: to.BoolPtr(false), + }, + }, + want: false, + }, + { + name: "Has cached value of true", + clusterScope: ClusterScope{ + AzureCluster: &infrav1.AzureCluster{ + Spec: infrav1.AzureClusterSpec{}, + }, + cache: &ClusterCache{ + isVnetManaged: to.BoolPtr(true), + }, + }, + want: true, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := tt.clusterScope.IsVnetManaged() + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("IsVnetManaged() = \n%t, want \n%t", got, tt.want) + } + if to.Bool(tt.clusterScope.cache.isVnetManaged) != got { + t.Errorf("IsVnetManaged() = \n%t, cache = \n%t", got, to.Bool(tt.clusterScope.cache.isVnetManaged)) + } + }) + } +} + func TestAzureBastionSpec(t *testing.T) { tests := []struct { name string @@ -1302,6 +1428,7 @@ func TestAzureBastionSpec(t *testing.T) { }, }, }, + cache: &ClusterCache{}, }, want: &bastionhosts.AzureBastionSpec{ Name: "fake-azure-bastion-1", diff --git a/azure/scope/machine.go b/azure/scope/machine.go index be5b837705e..dcae55eea11 100644 --- a/azure/scope/machine.go +++ b/azure/scope/machine.go @@ -348,9 +348,9 @@ func (m *MachineScope) Subnet() infrav1.SubnetSpec { // AvailabilityZone returns the AzureMachine Availability Zone. // Priority for selecting the AZ is -// 1) Machine.Spec.FailureDomain -// 2) AzureMachine.Spec.FailureDomain (This is to support deprecated AZ) -// 3) No AZ +// 1. Machine.Spec.FailureDomain +// 2. AzureMachine.Spec.FailureDomain (This is to support deprecated AZ) +// 3. No AZ func (m *MachineScope) AvailabilityZone() string { if m.Machine.Spec.FailureDomain != nil { return *m.Machine.Spec.FailureDomain @@ -653,14 +653,21 @@ func (m *MachineScope) SetSubnetName() error { subnetName := "" subnets := m.Subnets() var subnetCount int + subnetAllSpecified := false for _, subnet := range subnets { if string(subnet.Role) == m.Role() { subnetCount++ subnetName = subnet.Name } + if subnet.Role == infrav1.SubnetAll { + subnetAllSpecified = true + subnetName = subnet.Name + } } - if subnetCount == 0 || subnetCount > 1 || subnetName == "" { - return errors.New("a subnet name must be specified when no subnets are specified or more than 1 subnet of the same role exist") + if !subnetAllSpecified { + if subnetCount == 0 || subnetCount > 1 || subnetName == "" { + return errors.New("a subnet name must be specified when no subnets are specified or more than 1 subnet of the same role exist") + } } m.AzureMachine.Spec.SubnetName = subnetName diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go index a3e8cd4c2dc..b9680ab79e6 100644 --- a/azure/scope/managedcontrolplane.go +++ b/azure/scope/managedcontrolplane.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "golang.org/x/mod/semver" corev1 "k8s.io/api/core/v1" @@ -51,6 +52,7 @@ type ManagedControlPlaneScopeParams struct { Cluster *clusterv1.Cluster ControlPlane *infrav1exp.AzureManagedControlPlane ManagedMachinePools []ManagedMachinePool + Cache *ManagedControlPlaneCache } // NewManagedControlPlaneScope creates a new Scope from the supplied parameters. @@ -82,6 +84,10 @@ func NewManagedControlPlaneScope(ctx context.Context, params ManagedControlPlane } } + if params.Cache == nil { + params.Cache = &ManagedControlPlaneCache{} + } + helper, err := patch.NewHelper(params.ControlPlane, params.Client) if err != nil { return nil, errors.Wrap(err, "failed to init patch helper") @@ -94,14 +100,17 @@ func NewManagedControlPlaneScope(ctx context.Context, params ManagedControlPlane ControlPlane: params.ControlPlane, ManagedMachinePools: params.ManagedMachinePools, patchHelper: helper, + cache: params.Cache, }, nil } // ManagedControlPlaneScope defines the basic context for an actuator to operate upon. type ManagedControlPlaneScope struct { - Client client.Client - patchHelper *patch.Helper - kubeConfigData []byte + Client client.Client + patchHelper *patch.Helper + adminKubeConfigData []byte + userKubeConfigData []byte + cache *ManagedControlPlaneCache AzureClients Cluster *clusterv1.Cluster @@ -109,6 +118,11 @@ type ManagedControlPlaneScope struct { ManagedMachinePools []ManagedMachinePool } +// ManagedControlPlaneCache stores ManagedControlPlane data locally so we don't have to hit the API multiple times within the same reconcile loop. +type ManagedControlPlaneCache struct { + isVnetManaged *bool +} + // ResourceGroup returns the managed control plane's resource group. func (s *ManagedControlPlaneScope) ResourceGroup() string { if s.ControlPlane == nil { @@ -197,8 +211,12 @@ func (s *ManagedControlPlaneScope) Close(ctx context.Context) error { // Vnet returns the cluster Vnet. func (s *ManagedControlPlaneScope) Vnet() *infrav1.VnetSpec { + resourceGroupName := s.ControlPlane.Spec.ResourceGroupName + if s.ControlPlane.Spec.VirtualNetwork.ResourceGroupName != "" { + resourceGroupName = s.ControlPlane.Spec.VirtualNetwork.ResourceGroupName + } return &infrav1.VnetSpec{ - ResourceGroup: s.ControlPlane.Spec.ResourceGroupName, + ResourceGroup: resourceGroupName, Name: s.ControlPlane.Spec.VirtualNetwork.Name, VnetClassSpec: infrav1.VnetClassSpec{ CIDRBlocks: []string{s.ControlPlane.Spec.VirtualNetwork.CIDRBlock}, @@ -328,7 +346,20 @@ func (s *ManagedControlPlaneScope) IsIPv6Enabled() bool { // IsVnetManaged returns true if the vnet is managed. func (s *ManagedControlPlaneScope) IsVnetManaged() bool { - return true + if s.cache.isVnetManaged != nil { + return to.Bool(s.cache.isVnetManaged) + } + // TODO refactor `IsVnetManaged` so that it is able to use an upstream context + // see https://github.com/kubernetes-sigs/cluster-api-provider-azure/issues/2581 + ctx := context.Background() + ctx, log, done := tele.StartSpanWithLogger(ctx, "scope.ManagedControlPlaneScope.IsVnetManaged") + defer done() + isManaged, err := virtualnetworks.New(s).IsManaged(ctx) + if err != nil { + log.Error(err, "Unable to determine if ManagedControlPlaneScope VNET is managed by capz", "AzureManagedCluster", s.ClusterName()) + } + s.cache.isVnetManaged = to.BoolPtr(isManaged) + return isManaged } // APIServerLBName returns the API Server LB spec. @@ -383,6 +414,43 @@ func (s *ManagedControlPlaneScope) ManagedClusterAnnotations() map[string]string return s.ControlPlane.Annotations } +// IsLocalAcountsDisabled checks if local accounts have been disabled. +func (s *ManagedControlPlaneScope) IsLocalAcountsDisabled() bool { + if s.IsAadEnabled() && + s.ControlPlane.Spec.DisableLocalAccounts != nil && + *s.ControlPlane.Spec.DisableLocalAccounts { + return true + } + return false +} + +// IsAadEnabled checks if aad is enabled. +func (s *ManagedControlPlaneScope) IsAadEnabled() bool { + if s.ControlPlane.Spec.AADProfile != nil && s.ControlPlane.Spec.AADProfile.Managed { + return true + } + return false +} + +// SetAutoUpgradeVersionStatus sets the auto upgrade version in status +func (s *ManagedControlPlaneScope) SetAutoUpgradeVersionStatus(version string) { + s.ControlPlane.Status.AutoUpgradeVersion = version +} + +// IsManagedVersionUpgrade checks if version is auto managed by AKS. +func (s *ManagedControlPlaneScope) IsManagedVersionUpgrade() bool { + return isManagedVersionUpgrade(s.ControlPlane) +} + +func isManagedVersionUpgrade(managedControlPlane *infrav1exp.AzureManagedControlPlane) bool { + if managedControlPlane.Spec.AutoUpgradeProfile != nil && + (managedControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel != infrav1exp.UpgradeChannelNone && + managedControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel != infrav1exp.UpgradeChannelNodeImage) { + return true + } + return false +} + // ManagedClusterSpec returns the managed cluster spec. func (s *ManagedControlPlaneScope) ManagedClusterSpec(ctx context.Context) azure.ResourceSpecGetter { managedClusterSpec := managedclusters.ManagedClusterSpec{ @@ -397,13 +465,31 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec(ctx context.Context) azure DNSServiceIP: s.ControlPlane.Spec.DNSServiceIP, VnetSubnetID: azure.SubnetID( s.ControlPlane.Spec.SubscriptionID, - s.ControlPlane.Spec.ResourceGroupName, + s.VNetSpec().ResourceGroupName(), s.ControlPlane.Spec.VirtualNetwork.Name, s.ControlPlane.Spec.VirtualNetwork.Subnet.Name, ), GetAllAgentPools: s.GetAllAgentPoolSpecs, } + if s.ControlPlane.Spec.OutboundType != nil { + managedClusterSpec.OutboundType = s.ControlPlane.Spec.OutboundType + } + + if s.ControlPlane.Spec.DNSPrefix != nil { + managedClusterSpec.DNSPrefix = s.ControlPlane.Spec.DNSPrefix + } else { + managedClusterSpec.DNSPrefix = &s.ControlPlane.Name + } + + if s.ControlPlane.Spec.FqdnSubdomain != nil { + managedClusterSpec.FqdnSubdomain = s.ControlPlane.Spec.FqdnSubdomain + } + + if s.ControlPlane.Spec.DockerBridgeCidr != nil { + managedClusterSpec.DockerBridgeCidr = s.ControlPlane.Spec.DockerBridgeCidr + } + if s.ControlPlane.Spec.NetworkPlugin != nil { managedClusterSpec.NetworkPlugin = *s.ControlPlane.Spec.NetworkPlugin } @@ -429,6 +515,9 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec(ctx context.Context) azure EnableAzureRBAC: s.ControlPlane.Spec.AADProfile.Managed, AdminGroupObjectIDs: s.ControlPlane.Spec.AADProfile.AdminGroupObjectIDs, } + if s.ControlPlane.Spec.DisableLocalAccounts != nil { + managedClusterSpec.DisableLocalAccounts = s.ControlPlane.Spec.DisableLocalAccounts + } } if s.ControlPlane.Spec.AddonProfiles != nil { @@ -466,6 +555,20 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec(ctx context.Context) azure } } + if s.ControlPlane.Spec.UserAssignedIdentities != nil { + managedClusterSpec.UserAssignedIdentities = []managedclusters.UserAssignedIdentity{ + { + ProviderID: s.ControlPlane.Spec.UserAssignedIdentities[0].ProviderID, + }, + } + } + + if s.ControlPlane.Spec.AutoUpgradeProfile != nil { + managedClusterSpec.AutoUpgradeProfile = &managedclusters.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: s.ControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel, + } + } + return &managedClusterSpec } @@ -517,14 +620,24 @@ func (s *ManagedControlPlaneScope) MakeEmptyKubeConfigSecret() corev1.Secret { } } -// GetKubeConfigData returns a []byte that contains kubeconfig. -func (s *ManagedControlPlaneScope) GetKubeConfigData() []byte { - return s.kubeConfigData +// GetAdminKubeConfigData returns admin kubeconfig. +func (s *ManagedControlPlaneScope) GetAdminKubeConfigData() []byte { + return s.adminKubeConfigData +} + +// SetAdminKubeConfigData sets adminKubeconfig data. +func (s *ManagedControlPlaneScope) SetAdminKubeConfigData(kubeConfigData []byte) { + s.adminKubeConfigData = kubeConfigData +} + +// GetUserKubeConfigData returns user kubeconfig, required when using AAD with AKS cluster. +func (s *ManagedControlPlaneScope) GetUserKubeConfigData() []byte { + return s.userKubeConfigData } -// SetKubeConfigData sets kubeconfig data. -func (s *ManagedControlPlaneScope) SetKubeConfigData(kubeConfigData []byte) { - s.kubeConfigData = kubeConfigData +// SetUserKubeConfigData sets userKubeconfig data. +func (s *ManagedControlPlaneScope) SetUserKubeConfigData(kubeConfigData []byte) { + s.userKubeConfigData = kubeConfigData } // SetLongRunningOperationState will set the future on the AzureManagedControlPlane status to allow the resource to continue diff --git a/azure/scope/managedcontrolplane_test.go b/azure/scope/managedcontrolplane_test.go index ebfebeac241..b8db1b0d4d5 100644 --- a/azure/scope/managedcontrolplane_test.go +++ b/azure/scope/managedcontrolplane_test.go @@ -21,10 +21,10 @@ import ( "testing" "github.com/Azure/go-autorest/autorest" - "github.com/Azure/go-autorest/autorest/to" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + pointer "k8s.io/utils/ptr" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/services/managedclusters" infrav1 "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" @@ -80,6 +80,7 @@ func TestManagedControlPlaneScope_PoolVersion(t *testing.T) { Mode: "System", Cluster: "cluster1", VnetSubnetID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups//providers/Microsoft.Network/virtualNetworks//subnets/", + OSType: pointer.To("Linux"), }, }, }, @@ -118,9 +119,10 @@ func TestManagedControlPlaneScope_PoolVersion(t *testing.T) { SKU: "Standard_D2s_v3", Mode: "System", Replicas: 1, - Version: to.StringPtr("1.21.1"), + Version: pointer.To("1.21.1"), Cluster: "cluster1", VnetSubnetID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups//providers/Microsoft.Network/virtualNetworks//subnets/", + OSType: pointer.To("Linux"), }, }, }, @@ -267,3 +269,742 @@ func TestManagedControlPlaneScope_AddonProfiles(t *testing.T) { }) } } + +func TestManagedControlPlaneScope_AutoUpgradeProfile(t *testing.T) { + scheme := runtime.NewScheme() + _ = capiv1exp.AddToScheme(scheme) + _ = infrav1.AddToScheme(scheme) + + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected *managedclusters.ManagedClusterAutoUpgradeProfile + }{ + { + Name: "Without AutoUpgradeProfile", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: nil, + }, + { + Name: "With AutoUpgradeProfile UpgradeChannelNodeImage", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AutoUpgradeProfile: &infrav1.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: infrav1.UpgradeChannelNodeImage, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: &managedclusters.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: infrav1.UpgradeChannelNodeImage, + }, + }, + } + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(c.Input.ControlPlane).Build() + c.Input.Client = fakeClient + s, err := NewManagedControlPlaneScope(context.TODO(), c.Input) + g.Expect(err).To(Succeed()) + managedClusterGetter := s.ManagedClusterSpec(context.TODO()) + managedCluster, ok := managedClusterGetter.(*managedclusters.ManagedClusterSpec) + g.Expect(ok).To(BeTrue()) + g.Expect(managedCluster.AutoUpgradeProfile).To(Equal(c.Expected)) + }) + } +} + +func TestManagedControlPlaneScope_AADProfile(t *testing.T) { + scheme := runtime.NewScheme() + _ = capiv1exp.AddToScheme(scheme) + _ = infrav1.AddToScheme(scheme) + + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected *managedclusters.AADProfile + }{ + { + Name: "Without AADProfile", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: nil, + }, + { + Name: "With AADProfile", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: &managedclusters.AADProfile{ + Managed: true, + EnableAzureRBAC: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + }, + } + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(c.Input.ControlPlane).Build() + c.Input.Client = fakeClient + s, err := NewManagedControlPlaneScope(context.TODO(), c.Input) + g.Expect(err).To(Succeed()) + managedClusterGetter := s.ManagedClusterSpec(context.TODO()) + managedCluster, ok := managedClusterGetter.(*managedclusters.ManagedClusterSpec) + g.Expect(ok).To(BeTrue()) + g.Expect(managedCluster.AADProfile).To(Equal(c.Expected)) + }) + } +} + +func TestManagedControlPlaneScope_DisableLocalAccounts(t *testing.T) { + scheme := runtime.NewScheme() + _ = capiv1exp.AddToScheme(scheme) + _ = infrav1.AddToScheme(scheme) + + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected *bool + }{ + { + Name: "Without DisableLocalAccounts", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: nil, + }, + { + Name: "Without AAdProfile and With DisableLocalAccounts", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + DisableLocalAccounts: pointer.To(true), + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: nil, + }, + { + Name: "With AAdProfile and With DisableLocalAccounts", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: pointer.To(true), + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: pointer.To(true), + }, + } + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(c.Input.ControlPlane).Build() + c.Input.Client = fakeClient + s, err := NewManagedControlPlaneScope(context.TODO(), c.Input) + g.Expect(err).To(Succeed()) + managedClusterGetter := s.ManagedClusterSpec(context.TODO()) + managedCluster, ok := managedClusterGetter.(*managedclusters.ManagedClusterSpec) + g.Expect(ok).To(BeTrue()) + g.Expect(managedCluster.DisableLocalAccounts).To(Equal(c.Expected)) + }) + } +} + +func TestIsAaDEnabled(t *testing.T) { + scheme := runtime.NewScheme() + _ = capiv1exp.AddToScheme(scheme) + _ = infrav1.AddToScheme(scheme) + + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected bool + }{ + { + Name: "AAD is not enabled", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: false, + }, + { + Name: "AAdProfile and With DisableLocalAccounts", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: pointer.To(true), + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: true, + }, + } + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(c.Input.ControlPlane).Build() + c.Input.Client = fakeClient + s, err := NewManagedControlPlaneScope(context.TODO(), c.Input) + g.Expect(err).To(Succeed()) + aadEnabled := s.IsAadEnabled() + g.Expect(aadEnabled).To(Equal(c.Expected)) + }) + } +} + +func TestIsLocalAcountsDisabled(t *testing.T) { + scheme := runtime.NewScheme() + _ = capiv1exp.AddToScheme(scheme) + _ = infrav1.AddToScheme(scheme) + + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected bool + }{ + { + Name: "DisbaleLocalAccount is not enabled", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: false, + }, + { + Name: "With AAdProfile and Without DisableLocalAccounts", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: false, + }, + { + Name: "With AAdProfile and With DisableLocalAccounts", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: pointer.To(true), + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: true, + }, + } + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(c.Input.ControlPlane).Build() + c.Input.Client = fakeClient + s, err := NewManagedControlPlaneScope(context.TODO(), c.Input) + g.Expect(err).To(Succeed()) + localAccountsDisabled := s.IsLocalAcountsDisabled() + g.Expect(localAccountsDisabled).To(Equal(c.Expected)) + }) + } +} + +func Test_IsManagedVersionUpgrade(t *testing.T) { + scheme := runtime.NewScheme() + _ = capiv1exp.AddToScheme(scheme) + _ = infrav1.AddToScheme(scheme) + + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected bool + }{ + { + Name: "Upgrade channel not set", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: false, + }, + { + Name: "Upgradechannel is set rapid", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + AutoUpgradeProfile: &infrav1.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: infrav1.UpgradeChannelRapid, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: true, + }, + { + Name: "Upgradechannel is set patch", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + AutoUpgradeProfile: &infrav1.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: infrav1.UpgradeChannelPatch, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: true, + }, + { + Name: "Upgradechannel is set stable", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + AutoUpgradeProfile: &infrav1.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: infrav1.UpgradeChannelStable, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: true, + }, + { + Name: "Upgradechannel is set none", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + AutoUpgradeProfile: &infrav1.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: infrav1.UpgradeChannelNone, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: false, + }, + { + Name: "Upgradechannel is set node-image", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + AutoUpgradeProfile: &infrav1.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: infrav1.UpgradeChannelNodeImage, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: false, + }, + } + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(c.Input.ControlPlane).Build() + c.Input.Client = fakeClient + s, err := NewManagedControlPlaneScope(context.TODO(), c.Input) + g.Expect(err).To(Succeed()) + autoUpgrade := s.IsManagedVersionUpgrade() + g.Expect(autoUpgrade).To(Equal(c.Expected)) + }) + } +} diff --git a/azure/scope/managedmachinepool.go b/azure/scope/managedmachinepool.go index 801d609ad01..1da1d3070df 100644 --- a/azure/scope/managedmachinepool.go +++ b/azure/scope/managedmachinepool.go @@ -23,11 +23,13 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" + "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/util/futures" "sigs.k8s.io/cluster-api-provider-azure/util/tele" + "sigs.k8s.io/cluster-api-provider-azure/util/versions" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" clusterv1exp "sigs.k8s.io/cluster-api/exp/api/v1beta1" "sigs.k8s.io/cluster-api/util/conditions" @@ -69,21 +71,28 @@ func NewManagedMachinePoolScope(ctx context.Context, params ManagedMachinePoolSc return nil, errors.Wrap(err, "failed to init patch helper") } + capiMachinePoolPatchHelper, err := patch.NewHelper(params.MachinePool, params.Client) + if err != nil { + return nil, errors.Wrap(err, "failed to init patch helper") + } + return &ManagedMachinePoolScope{ - Client: params.Client, - Cluster: params.Cluster, - ControlPlane: params.ControlPlane, - MachinePool: params.MachinePool, - InfraMachinePool: params.InfraMachinePool, - patchHelper: helper, - ManagedClusterScoper: params.ManagedControlPlaneScope, + Client: params.Client, + Cluster: params.Cluster, + ControlPlane: params.ControlPlane, + MachinePool: params.MachinePool, + InfraMachinePool: params.InfraMachinePool, + patchHelper: helper, + capiMachinePoolPatchHelper: capiMachinePoolPatchHelper, + ManagedClusterScoper: params.ManagedControlPlaneScope, }, nil } // ManagedMachinePoolScope defines the basic context for an actuator to operate upon. type ManagedMachinePoolScope struct { - Client client.Client - patchHelper *patch.Helper + Client client.Client + patchHelper *patch.Helper + capiMachinePoolPatchHelper *patch.Helper azure.ManagedClusterScoper Cluster *clusterv1.Cluster @@ -128,17 +137,23 @@ func (s *ManagedMachinePoolScope) AgentPoolSpec() azure.AgentPoolSpec { func buildAgentPoolSpec(managedControlPlane *infrav1exp.AzureManagedControlPlane, machinePool *clusterv1exp.MachinePool, managedMachinePool *infrav1exp.AzureManagedMachinePool) azure.AgentPoolSpec { - var normalizedVersion *string - if machinePool.Spec.Template.Spec.Version != nil { - v := strings.TrimPrefix(*machinePool.Spec.Template.Spec.Version, "v") - normalizedVersion = &v - } + normalizedVersion := getManagedMachinePoolVersion(managedControlPlane, machinePool) replicas := int32(1) if machinePool.Spec.Replicas != nil { replicas = *machinePool.Spec.Replicas } + resourceGroupName := managedControlPlane.Spec.ResourceGroupName + if managedControlPlane.Spec.VirtualNetwork.ResourceGroupName != "" { + resourceGroupName = managedControlPlane.Spec.VirtualNetwork.ResourceGroupName + } + + subnetName := managedControlPlane.Spec.VirtualNetwork.Subnet.Name + if managedMachinePool.Spec.SubnetName != "" { + subnetName = managedMachinePool.Spec.SubnetName + } + agentPoolSpec := azure.AgentPoolSpec{ Name: to.String(managedMachinePool.Spec.Name), ResourceGroup: managedControlPlane.Spec.ResourceGroupName, @@ -148,17 +163,24 @@ func buildAgentPoolSpec(managedControlPlane *infrav1exp.AzureManagedControlPlane Version: normalizedVersion, VnetSubnetID: azure.SubnetID( managedControlPlane.Spec.SubscriptionID, - managedControlPlane.Spec.ResourceGroupName, + resourceGroupName, managedControlPlane.Spec.VirtualNetwork.Name, - managedControlPlane.Spec.VirtualNetwork.Subnet.Name, + subnetName, ), Mode: managedMachinePool.Spec.Mode, MaxPods: managedMachinePool.Spec.MaxPods, AvailabilityZones: managedMachinePool.Spec.AvailabilityZones, OsDiskType: managedMachinePool.Spec.OsDiskType, + OSType: managedMachinePool.Spec.OSType, EnableUltraSSD: managedMachinePool.Spec.EnableUltraSSD, } + // Set OSType to Linux by default if not specified + if managedMachinePool.Spec.OSType == nil || *managedMachinePool.Spec.OSType == "" { + osType := azure.LinuxOS + agentPoolSpec.OSType = &osType + } + if managedMachinePool.Spec.OSDiskSizeGB != nil { agentPoolSpec.OSDiskSizeGB = *managedMachinePool.Spec.OSDiskSizeGB } @@ -230,7 +252,7 @@ func (s *ManagedMachinePoolScope) UpdateDeleteStatus(condition clusterv1.Conditi } } -// UpdatePutStatus updates a condition on the AzureManagedControlPlane status after a PUT operation. +// UpdatePutStatus updates a condition on the AzureManagedMachinePool status after a PUT operation. func (s *ManagedMachinePoolScope) UpdatePutStatus(condition clusterv1.ConditionType, service string, err error) { switch { case err == nil: @@ -242,7 +264,7 @@ func (s *ManagedMachinePoolScope) UpdatePutStatus(condition clusterv1.ConditionT } } -// UpdatePatchStatus updates a condition on the AzureManagedControlPlane status after a PATCH operation. +// UpdatePatchStatus updates a condition on the AzureManagedMachinePool status after a PATCH operation. func (s *ManagedMachinePoolScope) UpdatePatchStatus(condition clusterv1.ConditionType, service string, err error) { switch { case err == nil: @@ -253,3 +275,47 @@ func (s *ManagedMachinePoolScope) UpdatePatchStatus(condition clusterv1.Conditio conditions.MarkFalse(s.InfraMachinePool, condition, infrav1.FailedReason, clusterv1.ConditionSeverityError, "%s failed to update. err: %s", service, err.Error()) } } + +// PatchCAPIMachinePoolObject persists the capi machinepool configuration and status. +func (s *ManagedMachinePoolScope) PatchCAPIMachinePoolObject(ctx context.Context) error { + return s.capiMachinePoolPatchHelper.Patch( + ctx, + s.MachinePool, + ) +} + +// UpdateCAPIMachinePoolReplicas updates the associated MachinePool replica count. +func (s *ManagedMachinePoolScope) UpdateCAPIMachinePoolReplicas(ctx context.Context, replicas *int32) { + s.MachinePool.Spec.Replicas = replicas +} + +// UpdateCAPIMachinePoolAnnotations updates the associated MachinePool annotation. +func (s *ManagedMachinePoolScope) UpdateCAPIMachinePoolAnnotations(ctx context.Context, key, value string) { + s.MachinePool.Annotations[key] = value +} + +// GetCAPIMachinePoolAnnotations gets the associated MachinePool annotation. +func (s *ManagedMachinePoolScope) GetCAPIMachinePoolAnnotation(ctx context.Context, key string) string { + return s.MachinePool.Annotations[key] +} + +// IsManagedVersionUpgrade checks if version is auto managed by AKS. +func (s *ManagedMachinePoolScope) IsManagedAutoUpgrade() bool { + return isManagedVersionUpgrade(s.ControlPlane) +} + +func getManagedMachinePoolVersion(managedControlPlane *infrav1exp.AzureManagedControlPlane, + machinePool *clusterv1exp.MachinePool) *string { + var v, av string + if machinePool != nil { + v = ptr.Deref(machinePool.Spec.Template.Spec.Version, "") + } + if managedControlPlane != nil { + av = managedControlPlane.Status.AutoUpgradeVersion + } + higherVersion, err := versions.GetHigherK8sVersion(v, av) + if err != nil { + return nil + } + return ptr.To(strings.TrimPrefix(higherVersion, "v")) +} diff --git a/azure/scope/managedmachinepool_test.go b/azure/scope/managedmachinepool_test.go index 907ca55a31a..dba02a8837c 100644 --- a/azure/scope/managedmachinepool_test.go +++ b/azure/scope/managedmachinepool_test.go @@ -25,6 +25,7 @@ import ( . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" "sigs.k8s.io/cluster-api-provider-azure/azure" infrav1 "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -66,7 +67,7 @@ func TestManagedMachinePoolScope_Autoscaling(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ - + OSType: ptr.To("Linux"), Name: "pool0", SKU: "Standard_D2s_v3", Replicas: 1, @@ -108,6 +109,7 @@ func TestManagedMachinePoolScope_Autoscaling(t *testing.T) { MinCount: to.Int32Ptr(2), MaxCount: to.Int32Ptr(10), VnetSubnetID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups//providers/Microsoft.Network/virtualNetworks//subnets/", + OSType: ptr.To("Linux"), }, }, } @@ -160,7 +162,7 @@ func TestManagedMachinePoolScope_NodeLabels(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ - + OSType: ptr.To("Linux"), Name: "pool0", SKU: "Standard_D2s_v3", Replicas: 1, @@ -195,6 +197,7 @@ func TestManagedMachinePoolScope_NodeLabels(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ + OSType: ptr.To("Linux"), Name: "pool1", SKU: "Standard_D2s_v3", Mode: "System", @@ -256,6 +259,7 @@ func TestManagedMachinePoolScope_MaxPods(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ + OSType: ptr.To("Linux"), Name: "pool0", SKU: "Standard_D2s_v3", Replicas: 1, @@ -288,6 +292,7 @@ func TestManagedMachinePoolScope_MaxPods(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ + OSType: ptr.To("Linux"), Name: "pool1", SKU: "Standard_D2s_v3", Mode: "System", @@ -347,7 +352,7 @@ func TestManagedMachinePoolScope_Taints(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ - + OSType: ptr.To("Linux"), Name: "pool0", SKU: "Standard_D2s_v3", Replicas: 1, @@ -386,6 +391,7 @@ func TestManagedMachinePoolScope_Taints(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ + OSType: ptr.To("Linux"), Name: "pool1", SKU: "Standard_D2s_v3", Mode: "User", @@ -445,6 +451,7 @@ func TestManagedMachinePoolScope_OSDiskType(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ + OSType: ptr.To("Linux"), Name: "pool0", SKU: "Standard_D2s_v3", Replicas: 1, @@ -477,6 +484,7 @@ func TestManagedMachinePoolScope_OSDiskType(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ + OSType: ptr.To("Linux"), Name: "pool1", SKU: "Standard_D2s_v3", Mode: "User", @@ -502,6 +510,122 @@ func TestManagedMachinePoolScope_OSDiskType(t *testing.T) { } } +func Test_getManagedMachinePoolVersion(t *testing.T) { + cases := []struct { + Name string + managedControlPlane *infrav1.AzureManagedControlPlane + machinePool *capiv1exp.MachinePool + Expected *string + }{ + { + Name: "Empty configs", + managedControlPlane: nil, + machinePool: nil, + Expected: nil, + }, + { + Name: "Empty mp", + managedControlPlane: &infrav1.AzureManagedControlPlane{}, + machinePool: nil, + Expected: nil, + }, + { + Name: "Only machine pool is available", + managedControlPlane: nil, + machinePool: &capiv1exp.MachinePool{ + Spec: capiv1exp.MachinePoolSpec{ + Template: clusterv1.MachineTemplateSpec{ + Spec: clusterv1.MachineSpec{ + Version: ptr.To("v1.15.0"), + }, + }, + }, + }, + Expected: ptr.To("1.15.0"), + }, + { + Name: "Only machine pool is available and cp is nil", + managedControlPlane: nil, + machinePool: &capiv1exp.MachinePool{ + Spec: capiv1exp.MachinePoolSpec{ + Template: clusterv1.MachineTemplateSpec{ + Spec: clusterv1.MachineSpec{ + Version: ptr.To("v1.15.0"), + }, + }, + }, + }, + Expected: ptr.To("1.15.0"), + }, + { + Name: "mcp.status.autoUpgradeVersion > mp.spec.template.spec.version", + managedControlPlane: &infrav1.AzureManagedControlPlane{ + Status: infrav1.AzureManagedControlPlaneStatus{ + AutoUpgradeVersion: "1.20.3", + }, + }, + machinePool: &capiv1exp.MachinePool{ + Spec: capiv1exp.MachinePoolSpec{ + Template: clusterv1.MachineTemplateSpec{ + Spec: clusterv1.MachineSpec{ + Version: ptr.To("v1.15.0"), + }, + }, + }, + }, + Expected: ptr.To("1.20.3"), + }, + { + Name: "suffix + mcp.status.autoUpgradeVersion > mp.spec.template.spec.version", + managedControlPlane: &infrav1.AzureManagedControlPlane{ + Status: infrav1.AzureManagedControlPlaneStatus{ + AutoUpgradeVersion: "v1.20.3", + }, + }, + machinePool: &capiv1exp.MachinePool{ + Spec: capiv1exp.MachinePoolSpec{ + Template: clusterv1.MachineTemplateSpec{ + Spec: clusterv1.MachineSpec{ + Version: ptr.To("v1.15.0"), + }, + }, + }, + }, + Expected: ptr.To("1.20.3"), + }, + { + Name: "mcp.status.autoUpgradeVersion < mp.spec.template.spec.version", + managedControlPlane: &infrav1.AzureManagedControlPlane{ + Status: infrav1.AzureManagedControlPlaneStatus{ + AutoUpgradeVersion: "v1.20.3", + }, + }, + machinePool: &capiv1exp.MachinePool{ + Spec: capiv1exp.MachinePoolSpec{ + Template: clusterv1.MachineTemplateSpec{ + Spec: clusterv1.MachineSpec{ + Version: ptr.To("v1.21.0"), + }, + }, + }, + }, + Expected: ptr.To("1.21.0"), + }, + } + + for _, c := range cases { + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + v := getManagedMachinePoolVersion(c.managedControlPlane, c.machinePool) + if c.Expected != nil { + g.Expect(*v).To(Equal(*c.Expected)) + } else { + g.Expect(v).To(Equal(c.Expected)) + } + }) + } +} + func getAzureMachinePool(name string, mode infrav1.NodePoolMode) *infrav1.AzureManagedMachinePool { return &infrav1.AzureManagedMachinePool{ ObjectMeta: metav1.ObjectMeta{ diff --git a/azure/services/agentpools/agentpools.go b/azure/services/agentpools/agentpools.go index 549aba02d0a..4664d80851c 100644 --- a/azure/services/agentpools/agentpools.go +++ b/azure/services/agentpools/agentpools.go @@ -22,13 +22,16 @@ import ( "time" "github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2021-05-01/containerservice" + "github.com/Azure/go-autorest/autorest/to" "github.com/google/go-cmp/cmp" "github.com/pkg/errors" - infrav1alpha4 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + "k8s.io/utils/ptr" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/converters" "sigs.k8s.io/cluster-api-provider-azure/util/maps" "sigs.k8s.io/cluster-api-provider-azure/util/tele" + "sigs.k8s.io/cluster-api-provider-azure/util/versions" ) const serviceName = "agentpools" @@ -43,6 +46,10 @@ type ManagedMachinePoolScope interface { SetAgentPoolProviderIDList([]string) SetAgentPoolReplicas(int32) SetAgentPoolReady(bool) + UpdateCAPIMachinePoolReplicas(ctx context.Context, replicas *int32) + UpdateCAPIMachinePoolAnnotations(ctx context.Context, key, value string) + GetCAPIMachinePoolAnnotation(ctx context.Context, key string) string + IsManagedAutoUpgrade() bool } // Service provides operations on Azure resources. @@ -74,7 +81,6 @@ func (s *Service) Reconcile(ctx context.Context) error { agentPoolSpec := s.scope.AgentPoolSpec() profile := converters.AgentPoolToContainerServiceAgentPool(agentPoolSpec) - existingPool, err := s.Client.Get(ctx, agentPoolSpec.ResourceGroup, agentPoolSpec.Cluster, agentPoolSpec.Name) if err != nil && !azure.ResourceNotFound(err) { return errors.Wrap(err, "failed to get existing agent pool") @@ -87,6 +93,9 @@ func (s *Service) Reconcile(ctx context.Context) error { customHeaders := maps.FilterByKeyPrefix(s.scope.AgentPoolAnnotations(), azure.CustomHeaderPrefix) if isCreate := azure.ResourceNotFound(err); isCreate { + if s.scope.IsManagedAutoUpgrade() { + profile.OrchestratorVersion = nil + } err = s.Client.CreateOrUpdate(ctx, agentPoolSpec.ResourceGroup, agentPoolSpec.Cluster, agentPoolSpec.Name, profile, customHeaders) if err != nil && azure.ResourceNotFound(err) { @@ -96,11 +105,19 @@ func (s *Service) Reconcile(ctx context.Context) error { } } else { ps := *existingPool.ManagedClusterAgentPoolProfileProperties.ProvisioningState - if ps != string(infrav1alpha4.Canceled) && ps != string(infrav1alpha4.Failed) && ps != string(infrav1alpha4.Succeeded) { + if ps != string(infrav1.Canceled) && ps != string(infrav1.Failed) && ps != string(infrav1.Succeeded) { msg := fmt.Sprintf("Unable to update existing agent pool in non terminal state. Agent pool must be in one of the following provisioning states: canceled, failed, or succeeded. Actual state: %s", ps) log.V(2).Info(msg) return azure.WithTransientError(errors.New(msg), 20*time.Second) } + // Get the higher version out of the existing and new version + profileOrchestratorVersion, err := versions.GetHigherK8sVersion( + ptr.Deref(existingPool.OrchestratorVersion, ""), + ptr.Deref(profile.OrchestratorVersion, "")) + + if err != nil { + return errors.Wrap(err, "error while calculating k8s version") + } // Normalize individual agent pools to diff in case we need to update existingProfile := containerservice.AgentPool{ @@ -111,20 +128,41 @@ func (s *Service) Reconcile(ctx context.Context) error { EnableAutoScaling: existingPool.EnableAutoScaling, MinCount: existingPool.MinCount, MaxCount: existingPool.MaxCount, + OsType: existingPool.OsType, }, } normalizedProfile := containerservice.AgentPool{ ManagedClusterAgentPoolProfileProperties: &containerservice.ManagedClusterAgentPoolProfileProperties{ Count: profile.Count, - OrchestratorVersion: profile.OrchestratorVersion, + OrchestratorVersion: ptr.To(profileOrchestratorVersion), Mode: profile.Mode, EnableAutoScaling: profile.EnableAutoScaling, MinCount: profile.MinCount, MaxCount: profile.MaxCount, + OsType: profile.OsType, }, } + // When autoscaling is set, the count of the nodes differ based on the autoscaler and should not depend on the + // count present in MachinePool or AzureManagedMachinePool, hence we should not make an update API call based + // on difference in count. + if to.Bool(profile.EnableAutoScaling) && existingProfile.Count != nil { + if to.Bool(profile.EnableAutoScaling) && s.scope.GetCAPIMachinePoolAnnotation(ctx, azure.ReplicasManagedByAutoscalerAnnotation) != "true" { + s.scope.UpdateCAPIMachinePoolAnnotations(ctx, azure.ReplicasManagedByAutoscalerAnnotation, "true") + } + + if to.Int32(existingProfile.Count) != to.Int32(normalizedProfile.Count) { + s.scope.UpdateCAPIMachinePoolReplicas(ctx, existingProfile.Count) + } + normalizedProfile.Count = existingProfile.Count + } + + // set ReplicasManagedByAutoscalerAnnotation to false as it is disabled by the user. + if !to.Bool(profile.EnableAutoScaling) && s.scope.GetCAPIMachinePoolAnnotation(ctx, azure.ReplicasManagedByAutoscalerAnnotation) == "true" { + s.scope.UpdateCAPIMachinePoolAnnotations(ctx, azure.ReplicasManagedByAutoscalerAnnotation, "false") + } + // Diff and check if we require an update diff := cmp.Diff(normalizedProfile, existingProfile) if diff != "" { diff --git a/azure/services/async/async.go b/azure/services/async/async.go index 93d10100114..e8afd248d78 100644 --- a/azure/services/async/async.go +++ b/azure/services/async/async.go @@ -66,26 +66,31 @@ func processOngoingOperation(ctx context.Context, scope FutureScope, client Futu } isDone, err := client.IsDone(ctx, sdkFuture) - if err != nil { - return nil, errors.Wrap(err, "failed checking if the operation was complete") - } - + // Assume that if isDone is true, then we successfully checked that the + // operation was complete even if err is non-nil. Assume the error in that + // case is unrelated and will be captured in Result below. if !isDone { + if err != nil { + return nil, errors.Wrap(err, "failed checking if the operation was complete") + } // Operation is still in progress, update conditions and requeue. log.V(2).Info("long running operation is still ongoing", "service", serviceName, "resource", resourceName) return nil, azure.WithTransientError(azure.NewOperationNotDoneError(future), retryAfter(sdkFuture)) } + if err != nil { + log.V(2).Error(err, "error checking long running operation status after it finished") + } + + // Once the operation is done, we can delete the long running operation state. + // If the operation failed, this will allow it to be retried during the next reconciliation. + // If the resource is not found, we also reset the long-running operation state so we can attempt to create it again. + // This can happen if the resource was deleted by another process before we could get the result. + scope.DeleteLongRunningOperationState(resourceName, serviceName) // Resource has been created/deleted/updated. log.V(2).Info("long running operation has completed", "service", serviceName, "resource", resourceName) result, err = client.Result(ctx, sdkFuture, future.Type) - if err == nil || azure.ResourceNotFound(err) { - // Once we have the result, we can delete the long running operation state. - // If the resource is not found, we also reset the long-running operation state so we can attempt to create it again. - // This can happen if the resource was deleted by another process before we could get the result. - scope.DeleteLongRunningOperationState(resourceName, serviceName) - } - return result, err + return client.Result(ctx, sdkFuture, future.Type) } // CreateResource implements the logic for creating a resource Asynchronously. @@ -151,7 +156,9 @@ func (s *Service) DeleteResource(ctx context.Context, spec azure.ResourceSpecGet future := s.Scope.GetLongRunningOperationState(resourceName, serviceName) if future != nil { _, err := processOngoingOperation(ctx, s.Scope, s.Deleter, resourceName, serviceName) - return err + if err != nil && future.Type == infrav1.DeleteFuture { + return err + } } // No long running operation is active, so delete the resource. diff --git a/azure/services/loadbalancers/loadbalancers.go b/azure/services/loadbalancers/loadbalancers.go index 49aa430ee78..9de73a60cd9 100644 --- a/azure/services/loadbalancers/loadbalancers.go +++ b/azure/services/loadbalancers/loadbalancers.go @@ -18,7 +18,8 @@ package loadbalancers import ( "context" - + "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-02-01/network" + "github.com/pkg/errors" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/services/async" @@ -78,10 +79,26 @@ func (s *Service) Reconcile(ctx context.Context) error { // Order of precedence (highest -> lowest) is: error that is not an operationNotDoneError (i.e. error creating) -> operationNotDoneError (i.e. creating in progress) -> no error (i.e. created) var result error for _, lbSpec := range specs { - if _, err := s.CreateResource(ctx, lbSpec, serviceName); err != nil { + if lb, err := s.CreateResource(ctx, lbSpec, serviceName); err != nil { if !azure.IsOperationNotDoneError(err) || result == nil { result = err } + } else { + loadBalancer, ok := lb.(network.LoadBalancer) + if !ok { + // Return out of loop since this would be an unexpected fatal error + result = errors.Errorf("created resource %T is not a network.LoadBalancer", lb) + break + } + if lbSpec.ResourceName() == s.Scope.APIServerLB().Name { + lbIPConfig := loadBalancer.FrontendIPConfigurations + if (lbIPConfig != nil) && len(*lbIPConfig) > 0 && + (*lbIPConfig)[0].PrivateIPAddress != nil && + *(*lbIPConfig)[0].PrivateIPAddress != "" { + s.Scope.APIServerLB().FrontendIPs[0].PrivateIPAddress = *(*lbIPConfig)[0].PrivateIPAddress + } + } + } } diff --git a/azure/services/loadbalancers/spec.go b/azure/services/loadbalancers/spec.go index 3948b56c0ee..da3ffcc2a48 100644 --- a/azure/services/loadbalancers/spec.go +++ b/azure/services/loadbalancers/spec.go @@ -35,6 +35,7 @@ type LBSpec struct { Role string Type infrav1.LBType SKU infrav1.SKU + IPAllocationMethod string VNetName string VNetResourceGroup string SubnetName string @@ -163,13 +164,17 @@ func getFrontendIPConfigs(lbSpec LBSpec) ([]network.FrontendIPConfiguration, []n frontendIDs := make([]network.SubResource, 0) for _, ipConfig := range lbSpec.FrontendIPConfigs { var properties network.FrontendIPConfigurationPropertiesFormat + var privateIPAddress string + if lbSpec.IPAllocationMethod == "Static" { + privateIPAddress = ipConfig.PrivateIPAddress + } if lbSpec.Type == infrav1.Internal { properties = network.FrontendIPConfigurationPropertiesFormat{ - PrivateIPAllocationMethod: network.IPAllocationMethodStatic, + PrivateIPAllocationMethod: network.IPAllocationMethod(lbSpec.IPAllocationMethod), Subnet: &network.Subnet{ ID: to.StringPtr(azure.SubnetID(lbSpec.SubscriptionID, lbSpec.VNetResourceGroup, lbSpec.VNetName, lbSpec.SubnetName)), }, - PrivateIPAddress: to.StringPtr(ipConfig.PrivateIPAddress), + PrivateIPAddress: to.StringPtr(privateIPAddress), } } else { properties = network.FrontendIPConfigurationPropertiesFormat{ diff --git a/azure/services/managedclusters/client.go b/azure/services/managedclusters/client.go index 73ab07b1448..6f6ca0ccb7d 100644 --- a/azure/services/managedclusters/client.go +++ b/azure/services/managedclusters/client.go @@ -33,6 +33,7 @@ import ( // CredentialGetter is a helper interface for getting managed cluster credentials. type CredentialGetter interface { GetCredentials(context.Context, string, string) ([]byte, error) + GetUserCredentials(context.Context, string, string) ([]byte, error) } // azureClient contains the Azure go-sdk Client. @@ -79,6 +80,23 @@ func (ac *azureClient) GetCredentials(ctx context.Context, resourceGroupName, na return *(*credentialList.Kubeconfigs)[0].Value, nil } +// GetUserCredentials fetches the user kubeconfig for a managed cluster. +func (ac *azureClient) GetUserCredentials(ctx context.Context, resourceGroupName, name string) ([]byte, error) { + ctx, _, done := tele.StartSpanWithLogger(ctx, "managedclusters.azureClient.GetCredentials") + defer done() + + credentialList, err := ac.managedclusters.ListClusterUserCredentials(ctx, resourceGroupName, name, "") + if err != nil { + return nil, err + } + + if credentialList.Kubeconfigs == nil || len(*credentialList.Kubeconfigs) < 1 { + return nil, errors.New("no user kubeconfigs available for the managed cluster") + } + + return *(*credentialList.Kubeconfigs)[0].Value, nil +} + // CreateOrUpdateAsync creates or updates a managed cluster. // It sends a PUT request to Azure and if accepted without error, the func will return a Future which can be used to track the ongoing // progress of the operation. diff --git a/azure/services/managedclusters/managedclusters.go b/azure/services/managedclusters/managedclusters.go index b9662600a67..4902558a5e2 100644 --- a/azure/services/managedclusters/managedclusters.go +++ b/azure/services/managedclusters/managedclusters.go @@ -18,20 +18,26 @@ package managedclusters import ( "context" + "fmt" "github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2021-05-01/containerservice" "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/tools/clientcmd" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/services/async" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/token" "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" "sigs.k8s.io/cluster-api-provider-azure/util/tele" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ) -const serviceName = "managedcluster" +const ( + serviceName = "managedcluster" + aadResourceId = "6dae42f8-4368-4678-94ff-3960e28e3630" +) // ManagedClusterScope defines the scope interface for a managed cluster. type ManagedClusterScope interface { @@ -40,8 +46,14 @@ type ManagedClusterScope interface { ManagedClusterSpec(context.Context) azure.ResourceSpecGetter SetControlPlaneEndpoint(clusterv1.APIEndpoint) MakeEmptyKubeConfigSecret() corev1.Secret - GetKubeConfigData() []byte - SetKubeConfigData([]byte) + GetAdminKubeConfigData() []byte + SetAdminKubeConfigData([]byte) + GetUserKubeConfigData() []byte + SetUserKubeConfigData([]byte) + IsAadEnabled() bool + IsLocalAcountsDisabled() bool + SetAutoUpgradeVersionStatus(version string) + IsManagedVersionUpgrade() bool } // Service provides operations on azure resources. @@ -94,11 +106,18 @@ func (s *Service) Reconcile(ctx context.Context) error { // Update kubeconfig data // Always fetch credentials in case of rotation - kubeConfigData, err := s.GetCredentials(ctx, managedClusterSpec.ResourceGroupName(), managedClusterSpec.ResourceName()) + adminKubeConfigData, userKubeConfigData, err := s.ReconcileKubeconfig(ctx, managedClusterSpec) if err != nil { - return errors.Wrap(err, "failed to get credentials for managed cluster") + return errors.Wrap(err, "error while reconciling adminKubeConfigData") + } + + if s.Scope.IsManagedVersionUpgrade() && managedCluster.KubernetesVersion != nil { + kubernetesVersion := fmt.Sprintf("v%s", *managedCluster.KubernetesVersion) + s.Scope.SetAutoUpgradeVersionStatus(kubernetesVersion) } - s.Scope.SetKubeConfigData(kubeConfigData) + + s.Scope.SetAdminKubeConfigData(adminKubeConfigData) + s.Scope.SetUserKubeConfigData(userKubeConfigData) } s.Scope.UpdatePutStatus(infrav1.ManagedClusterRunningCondition, serviceName, resultErr) return resultErr @@ -126,3 +145,70 @@ func (s *Service) Delete(ctx context.Context) error { func (s *Service) IsManaged(ctx context.Context) (bool, error) { return true, nil } + +func (s *Service) ReconcileKubeconfig(ctx context.Context, managedClusterSpec azure.ResourceSpecGetter) ([]byte, []byte, error) { + var ( + userKubeConfigData []byte + adminKubeConfigData []byte + err error + ) + + if s.Scope.IsAadEnabled() { + if userKubeConfigData, err = s.GetUserKubeConfigData(ctx, managedClusterSpec); err != nil { + return nil, nil, errors.Wrap(err, "error while trying to get user kubeconfig") + } + } + + if s.Scope.IsLocalAcountsDisabled() { + userKubeconfigWithToken, err := s.GetUserKubeConfigWithToken(userKubeConfigData, ctx, managedClusterSpec) + if err != nil { + return nil, nil, errors.Wrap(err, "error while trying to get user kubeconfig with token") + } + return userKubeconfigWithToken, userKubeConfigData, nil + } + + adminKubeConfigData, err = s.GetCredentials(ctx, managedClusterSpec.ResourceGroupName(), managedClusterSpec.ResourceName()) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to get credentials for managed cluster") + } + return adminKubeConfigData, userKubeConfigData, nil +} + +func (s *Service) GetUserKubeConfigData(ctx context.Context, managedClusterSpec azure.ResourceSpecGetter) ([]byte, error) { + kubeConfigData, err := s.GetUserCredentials(ctx, managedClusterSpec.ResourceGroupName(), managedClusterSpec.ResourceName()) + if err != nil { + return nil, errors.Wrap(err, "failed to get credentials for managed cluster") + } + return kubeConfigData, nil +} + +func (s *Service) GetUserKubeConfigWithToken(userKubeConfigData []byte, ctx context.Context, managedClusterSpec azure.ResourceSpecGetter) ([]byte, error) { + + tokenClient, err := token.NewClient(s.Scope) + if err != nil { + return nil, errors.Wrap(err, "error while getting aad token client") + } + + token, err := tokenClient.GetAzureActiveDirectoryToken(ctx, aadResourceId) + if err != nil { + return nil, errors.Wrap(err, "error while getting aad token for user kubeconfig") + } + + return s.CreateUserKubeconfigWithToken(token, userKubeConfigData) +} + +func (s *Service) CreateUserKubeconfigWithToken(token string, userKubeConfigData []byte) ([]byte, error) { + config, err := clientcmd.Load(userKubeConfigData) + if err != nil { + return nil, errors.Wrap(err, "error while trying to unmarshal new user kubeconfig with token") + } + for _, auth := range config.AuthInfos { + auth.Token = token + auth.Exec = nil + } + kubeconfig, err := clientcmd.Write(*config) + if err != nil { + return nil, errors.Wrap(err, "error while trying to marshal new user kubeconfig with token") + } + return kubeconfig, nil +} diff --git a/azure/services/managedclusters/managedclusters_test.go b/azure/services/managedclusters/managedclusters_test.go index af875091dd6..549e380902f 100644 --- a/azure/services/managedclusters/managedclusters_test.go +++ b/azure/services/managedclusters/managedclusters_test.go @@ -33,6 +33,14 @@ import ( ) var fakeManagedClusterSpec = &ManagedClusterSpec{Name: "my-managedcluster", ResourceGroup: "my-rg"} +var fakeManagedClusterSpecWithAAD = &ManagedClusterSpec{ + Name: "my-managedcluster", + ResourceGroup: "my-rg", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"000000-000000-000000-000000"}, + }, +} func TestReconcile(t *testing.T) { testcases := []struct { @@ -60,6 +68,7 @@ func TestReconcile(t *testing.T) { name: "create managed cluster succeeds", expectedError: "", expect: func(m *mock_managedclusters.MockCredentialGetterMockRecorder, s *mock_managedclusters.MockManagedClusterScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { + var userKubeConfigData []byte s.ManagedClusterSpec(gomockinternal.AContext()).Return(fakeManagedClusterSpec) r.CreateResource(gomockinternal.AContext(), fakeManagedClusterSpec, serviceName).Return(containerservice.ManagedCluster{ ManagedClusterProperties: &containerservice.ManagedClusterProperties{ @@ -71,14 +80,74 @@ func TestReconcile(t *testing.T) { Host: "my-managedcluster-fqdn", Port: 443, }) + s.IsAadEnabled().Return(false) + s.IsLocalAcountsDisabled().Return(false) + m.GetCredentials(gomockinternal.AContext(), "my-rg", "my-managedcluster").Return([]byte("credentials"), nil) + s.SetAdminKubeConfigData([]byte("credentials")) + s.SetUserKubeConfigData(userKubeConfigData) + s.IsManagedVersionUpgrade().Return(false) + s.UpdatePutStatus(infrav1.ManagedClusterRunningCondition, serviceName, nil) + }, + }, + { + name: "create managed cluster succeeds, update autoupgrade status", + expectedError: "", + expect: func(m *mock_managedclusters.MockCredentialGetterMockRecorder, s *mock_managedclusters.MockManagedClusterScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { + var userKubeConfigData []byte + s.ManagedClusterSpec(gomockinternal.AContext()).Return(fakeManagedClusterSpec) + r.CreateResource(gomockinternal.AContext(), fakeManagedClusterSpec, serviceName).Return(containerservice.ManagedCluster{ + ManagedClusterProperties: &containerservice.ManagedClusterProperties{ + Fqdn: pointer.String("my-managedcluster-fqdn"), + ProvisioningState: pointer.String("Succeeded"), + KubernetesVersion: pointer.String("1.27.3"), + }, + }, nil) + s.SetControlPlaneEndpoint(clusterv1.APIEndpoint{ + Host: "my-managedcluster-fqdn", + Port: 443, + }) + s.IsAadEnabled().Return(false) + s.IsLocalAcountsDisabled().Return(false) + m.GetCredentials(gomockinternal.AContext(), "my-rg", "my-managedcluster").Return([]byte("credentials"), nil) + s.SetAdminKubeConfigData([]byte("credentials")) + s.SetUserKubeConfigData(userKubeConfigData) + s.IsManagedVersionUpgrade().Return(true) + s.SetAutoUpgradeVersionStatus("v1.27.3") + s.UpdatePutStatus(infrav1.ManagedClusterRunningCondition, serviceName, nil) + }, + }, + { + name: "create managed cluster succeeds with user kubeconfig", + expectedError: "", + expect: func(m *mock_managedclusters.MockCredentialGetterMockRecorder, s *mock_managedclusters.MockManagedClusterScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { + s.ManagedClusterSpec(gomockinternal.AContext()).Return(fakeManagedClusterSpecWithAAD) + r.CreateResource(gomockinternal.AContext(), fakeManagedClusterSpecWithAAD, serviceName).Return(containerservice.ManagedCluster{ + ManagedClusterProperties: &containerservice.ManagedClusterProperties{ + Fqdn: pointer.String("my-managedcluster-fqdn"), + ProvisioningState: pointer.String("Succeeded"), + AadProfile: &containerservice.ManagedClusterAADProfile{ + Managed: pointer.Bool(true), + AdminGroupObjectIDs: &[]string{"000000-000000-000000-000000"}, + }, + }, + }, nil) + s.SetControlPlaneEndpoint(clusterv1.APIEndpoint{ + Host: "my-managedcluster-fqdn", + Port: 443, + }) + s.IsAadEnabled().Return(true) + s.IsLocalAcountsDisabled().Return(false) m.GetCredentials(gomockinternal.AContext(), "my-rg", "my-managedcluster").Return([]byte("credentials"), nil) - s.SetKubeConfigData([]byte("credentials")) + m.GetUserCredentials(gomockinternal.AContext(), "my-rg", "my-managedcluster").Return([]byte("credentials-user"), nil) + s.SetAdminKubeConfigData([]byte("credentials")) + s.SetUserKubeConfigData([]byte("credentials-user")) + s.IsManagedVersionUpgrade().Return(false) s.UpdatePutStatus(infrav1.ManagedClusterRunningCondition, serviceName, nil) }, }, { name: "fail to get managed cluster credentials", - expectedError: "failed to get credentials for managed cluster: internal server error", + expectedError: "error while reconciling adminKubeConfigData: failed to get credentials for managed cluster: internal server error", expect: func(m *mock_managedclusters.MockCredentialGetterMockRecorder, s *mock_managedclusters.MockManagedClusterScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { s.ManagedClusterSpec(gomockinternal.AContext()).Return(fakeManagedClusterSpec) r.CreateResource(gomockinternal.AContext(), fakeManagedClusterSpec, serviceName).Return(containerservice.ManagedCluster{ @@ -91,6 +160,8 @@ func TestReconcile(t *testing.T) { Host: "my-managedcluster-fqdn", Port: 443, }) + s.IsAadEnabled().Return(false) + s.IsLocalAcountsDisabled().Return(false) m.GetCredentials(gomockinternal.AContext(), "my-rg", "my-managedcluster").Return([]byte(""), errors.New("internal server error")) }, }, diff --git a/azure/services/managedclusters/mock_managedclusters/client_mock.go b/azure/services/managedclusters/mock_managedclusters/client_mock.go index 735939603c5..77216af037c 100644 --- a/azure/services/managedclusters/mock_managedclusters/client_mock.go +++ b/azure/services/managedclusters/mock_managedclusters/client_mock.go @@ -64,3 +64,18 @@ func (mr *MockCredentialGetterMockRecorder) GetCredentials(arg0, arg1, arg2 inte mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCredentials", reflect.TypeOf((*MockCredentialGetter)(nil).GetCredentials), arg0, arg1, arg2) } + +// GetUserCredentials mocks base method. +func (m *MockCredentialGetter) GetUserCredentials(arg0 context.Context, arg1, arg2 string) ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetUserCredentials", arg0, arg1, arg2) + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetUserCredentials indicates an expected call of GetUserCredentials. +func (mr *MockCredentialGetterMockRecorder) GetUserCredentials(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUserCredentials", reflect.TypeOf((*MockCredentialGetter)(nil).GetUserCredentials), arg0, arg1, arg2) +} diff --git a/azure/services/managedclusters/mock_managedclusters/managedclusters_mock.go b/azure/services/managedclusters/mock_managedclusters/managedclusters_mock.go index abf3c16817b..1badc3806c1 100644 --- a/azure/services/managedclusters/mock_managedclusters/managedclusters_mock.go +++ b/azure/services/managedclusters/mock_managedclusters/managedclusters_mock.go @@ -137,18 +137,18 @@ func (mr *MockManagedClusterScopeMockRecorder) DeleteLongRunningOperationState(a return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLongRunningOperationState", reflect.TypeOf((*MockManagedClusterScope)(nil).DeleteLongRunningOperationState), arg0, arg1) } -// GetKubeConfigData mocks base method. -func (m *MockManagedClusterScope) GetKubeConfigData() []byte { +// GetAdminKubeConfigData mocks base method. +func (m *MockManagedClusterScope) GetAdminKubeConfigData() []byte { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetKubeConfigData") + ret := m.ctrl.Call(m, "GetAdminKubeConfigData") ret0, _ := ret[0].([]byte) return ret0 } -// GetKubeConfigData indicates an expected call of GetKubeConfigData. -func (mr *MockManagedClusterScopeMockRecorder) GetKubeConfigData() *gomock.Call { +// GetAdminKubeConfigData indicates an expected call of GetAdminKubeConfigData. +func (mr *MockManagedClusterScopeMockRecorder) GetAdminKubeConfigData() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).GetKubeConfigData)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAdminKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).GetAdminKubeConfigData)) } // GetLongRunningOperationState mocks base method. @@ -165,6 +165,20 @@ func (mr *MockManagedClusterScopeMockRecorder) GetLongRunningOperationState(arg0 return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLongRunningOperationState", reflect.TypeOf((*MockManagedClusterScope)(nil).GetLongRunningOperationState), arg0, arg1) } +// GetUserKubeConfigData mocks base method. +func (m *MockManagedClusterScope) GetUserKubeConfigData() []byte { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetUserKubeConfigData") + ret0, _ := ret[0].([]byte) + return ret0 +} + +// GetUserKubeConfigData indicates an expected call of GetUserKubeConfigData. +func (mr *MockManagedClusterScopeMockRecorder) GetUserKubeConfigData() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUserKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).GetUserKubeConfigData)) +} + // HashKey mocks base method. func (m *MockManagedClusterScope) HashKey() string { m.ctrl.T.Helper() @@ -179,6 +193,48 @@ func (mr *MockManagedClusterScopeMockRecorder) HashKey() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HashKey", reflect.TypeOf((*MockManagedClusterScope)(nil).HashKey)) } +// IsAadEnabled mocks base method. +func (m *MockManagedClusterScope) IsAadEnabled() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsAadEnabled") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsAadEnabled indicates an expected call of IsAadEnabled. +func (mr *MockManagedClusterScopeMockRecorder) IsAadEnabled() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsAadEnabled", reflect.TypeOf((*MockManagedClusterScope)(nil).IsAadEnabled)) +} + +// IsLocalAcountsDisabled mocks base method. +func (m *MockManagedClusterScope) IsLocalAcountsDisabled() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsLocalAcountsDisabled") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsLocalAcountsDisabled indicates an expected call of IsLocalAcountsDisabled. +func (mr *MockManagedClusterScopeMockRecorder) IsLocalAcountsDisabled() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsLocalAcountsDisabled", reflect.TypeOf((*MockManagedClusterScope)(nil).IsLocalAcountsDisabled)) +} + +// IsManagedVersionUpgrade mocks base method. +func (m *MockManagedClusterScope) IsManagedVersionUpgrade() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsManagedVersionUpgrade") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsManagedVersionUpgrade indicates an expected call of IsManagedVersionUpgrade. +func (mr *MockManagedClusterScopeMockRecorder) IsManagedVersionUpgrade() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsManagedVersionUpgrade", reflect.TypeOf((*MockManagedClusterScope)(nil).IsManagedVersionUpgrade)) +} + // MakeEmptyKubeConfigSecret mocks base method. func (m *MockManagedClusterScope) MakeEmptyKubeConfigSecret() v1.Secret { m.ctrl.T.Helper() @@ -207,28 +263,40 @@ func (mr *MockManagedClusterScopeMockRecorder) ManagedClusterSpec(arg0 interface return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ManagedClusterSpec", reflect.TypeOf((*MockManagedClusterScope)(nil).ManagedClusterSpec), arg0) } -// SetControlPlaneEndpoint mocks base method. -func (m *MockManagedClusterScope) SetControlPlaneEndpoint(arg0 v1beta10.APIEndpoint) { +// SetAdminKubeConfigData mocks base method. +func (m *MockManagedClusterScope) SetAdminKubeConfigData(arg0 []byte) { m.ctrl.T.Helper() - m.ctrl.Call(m, "SetControlPlaneEndpoint", arg0) + m.ctrl.Call(m, "SetAdminKubeConfigData", arg0) } -// SetControlPlaneEndpoint indicates an expected call of SetControlPlaneEndpoint. -func (mr *MockManagedClusterScopeMockRecorder) SetControlPlaneEndpoint(arg0 interface{}) *gomock.Call { +// SetAdminKubeConfigData indicates an expected call of SetAdminKubeConfigData. +func (mr *MockManagedClusterScopeMockRecorder) SetAdminKubeConfigData(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetControlPlaneEndpoint", reflect.TypeOf((*MockManagedClusterScope)(nil).SetControlPlaneEndpoint), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetAdminKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).SetAdminKubeConfigData), arg0) +} + +// SetAutoUpgradeVersionStatus mocks base method. +func (m *MockManagedClusterScope) SetAutoUpgradeVersionStatus(version string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetAutoUpgradeVersionStatus", version) +} + +// SetAutoUpgradeVersionStatus indicates an expected call of SetAutoUpgradeVersionStatus. +func (mr *MockManagedClusterScopeMockRecorder) SetAutoUpgradeVersionStatus(version interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetAutoUpgradeVersionStatus", reflect.TypeOf((*MockManagedClusterScope)(nil).SetAutoUpgradeVersionStatus), version) } -// SetKubeConfigData mocks base method. -func (m *MockManagedClusterScope) SetKubeConfigData(arg0 []byte) { +// SetControlPlaneEndpoint mocks base method. +func (m *MockManagedClusterScope) SetControlPlaneEndpoint(arg0 v1beta10.APIEndpoint) { m.ctrl.T.Helper() - m.ctrl.Call(m, "SetKubeConfigData", arg0) + m.ctrl.Call(m, "SetControlPlaneEndpoint", arg0) } -// SetKubeConfigData indicates an expected call of SetKubeConfigData. -func (mr *MockManagedClusterScopeMockRecorder) SetKubeConfigData(arg0 interface{}) *gomock.Call { +// SetControlPlaneEndpoint indicates an expected call of SetControlPlaneEndpoint. +func (mr *MockManagedClusterScopeMockRecorder) SetControlPlaneEndpoint(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).SetKubeConfigData), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetControlPlaneEndpoint", reflect.TypeOf((*MockManagedClusterScope)(nil).SetControlPlaneEndpoint), arg0) } // SetLongRunningOperationState mocks base method. @@ -243,6 +311,18 @@ func (mr *MockManagedClusterScopeMockRecorder) SetLongRunningOperationState(arg0 return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLongRunningOperationState", reflect.TypeOf((*MockManagedClusterScope)(nil).SetLongRunningOperationState), arg0) } +// SetUserKubeConfigData mocks base method. +func (m *MockManagedClusterScope) SetUserKubeConfigData(arg0 []byte) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetUserKubeConfigData", arg0) +} + +// SetUserKubeConfigData indicates an expected call of SetUserKubeConfigData. +func (mr *MockManagedClusterScopeMockRecorder) SetUserKubeConfigData(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetUserKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).SetUserKubeConfigData), arg0) +} + // SubscriptionID mocks base method. func (m *MockManagedClusterScope) SubscriptionID() string { m.ctrl.T.Helper() diff --git a/azure/services/managedclusters/spec.go b/azure/services/managedclusters/spec.go index 6214c46387e..1d62a19b765 100644 --- a/azure/services/managedclusters/spec.go +++ b/azure/services/managedclusters/spec.go @@ -26,9 +26,12 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/google/go-cmp/cmp" "github.com/pkg/errors" + "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/converters" + expinfrav1 "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-azure/util/versions" ) // ManagedClusterSpec contains properties to create a managed cluster. @@ -63,6 +66,9 @@ type ManagedClusterSpec struct { // NetworkPolicy used for building Kubernetes network. Possible values include: 'calico', 'azure'. Defaults to azure. NetworkPolicy string + // OutboundType used for building Kubernetes network. Possible values include: 'loadBalancer', 'managedNATGateway', 'userAssignedNATGateway', 'userDefinedRouting'. + OutboundType *expinfrav1.ManagedControlPlaneOutboundType + // SSHPublicKey is a string literal containing an ssh public key. Will autogenerate and discard if not provided. SSHPublicKey string @@ -75,9 +81,18 @@ type ManagedClusterSpec struct { // ServiceCIDR is the CIDR block for IP addresses distributed to services ServiceCIDR string + // DockerBridgeCidr - A CIDR notation IP range assigned to the Docker bridge network. It must not overlap with any Subnet IP ranges or the Kubernetes service address range. + DockerBridgeCidr *string + // DNSServiceIP is an IP address assigned to the Kubernetes DNS service DNSServiceIP *string + // DNSPrefix - DNS prefix specified when creating the managed cluster. + DNSPrefix *string + + // FqdnSubdomain - FQDN subdomain specified when creating private cluster with custom private dns zone. + FqdnSubdomain *string + // AddonProfiles are the profiles of managed cluster add-on. AddonProfiles []AddonProfile @@ -95,6 +110,29 @@ type ManagedClusterSpec struct { // Headers is the list of headers to add to the HTTP requests to update this resource. Headers map[string]string + + // UserAssignedIdentities is a list of standalone Azure identities provided by the user to assign the cluster + UserAssignedIdentities []UserAssignedIdentity + + // DisableLocalAccounts - If set to true, getting static credential will be disabled for this cluster. Expected to only be used for AAD clusters. + DisableLocalAccounts *bool + + // AutoUpgradeProfile - Profile of auto upgrade configuration. + AutoUpgradeProfile *ManagedClusterAutoUpgradeProfile +} + +// ManagedClusterAutoUpgradeProfile auto upgrade profile for a managed cluster. +type ManagedClusterAutoUpgradeProfile struct { + // UpgradeChannel - upgrade channel for auto upgrade. Possible values include: 'UpgradeChannelRapid', 'UpgradeChannelStable', 'UpgradeChannelPatch', 'UpgradeChannelNodeImage', 'UpgradeChannelNone' + UpgradeChannel expinfrav1.UpgradeChannel +} + +// UserAssignedIdentity defines the user-assigned identities provided +// by the user to be assigned to Azure resources. +type UserAssignedIdentity struct { + // ProviderID is the identification ID of the user-assigned Identity, the format of an identity is: + // 'azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}' + ProviderID string } // AADProfile is Azure Active Directory configuration to integrate with AKS, for aad authentication. @@ -179,6 +217,28 @@ func (s *ManagedClusterSpec) CustomHeaders() map[string]string { return s.Headers } +// GetManagedClusterVersion gets the desired managed k8s version. +// If autoupgrade channels is set to patch, stable or rapid, clusters can be upgraded to higher version by AKS. +// If autoupgrade is triggered, existing kubernetes version will be higher than the user desired kubernetes version. +// CAPZ should honour the upgrade and it should not downgrade to the lower desired version. +func (s *ManagedClusterSpec) GetManagedClusterVersion(existing interface{}) (string, error) { + version := s.Version + if existing != nil && version != "" { + existingMC, ok := existing.(containerservice.ManagedCluster) + if !ok { + return version, fmt.Errorf("%T is not a containerservice.ManagedCluster", existing) + } + if v, err := versions.GetHigherK8sVersion( + version, + ptr.Deref(existingMC.KubernetesVersion, version)); err != nil { + return "", err + } else { + version = v + } + } + return version, nil +} + // Parameters returns the parameters for the managed clusters. func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{}, err error) { decodedSSHPublicKey, err := base64.StdEncoding.DecodeString(s.SSHPublicKey) @@ -193,8 +253,7 @@ func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{ ManagedClusterProperties: &containerservice.ManagedClusterProperties{ NodeResourceGroup: &s.NodeResourceGroup, EnableRBAC: to.BoolPtr(true), - DNSPrefix: &s.Name, - KubernetesVersion: &s.Version, + DNSPrefix: s.DNSPrefix, LinuxProfile: &containerservice.LinuxProfile{ AdminUsername: to.StringPtr(azure.DefaultAKSUserName), SSH: &containerservice.SSHConfiguration{ @@ -217,6 +276,16 @@ func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{ }, } + if kubernetesVersion, err := s.GetManagedClusterVersion(existing); err != nil { + return nil, err + } else { + managedCluster.KubernetesVersion = &kubernetesVersion + } + + if s.FqdnSubdomain != nil { + managedCluster.FqdnSubdomain = s.FqdnSubdomain + } + if tags := *to.StringMapPtr(s.Tags); len(tags) != 0 { managedCluster.Tags = tags } @@ -226,8 +295,8 @@ func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{ } if s.ServiceCIDR != "" { + managedCluster.NetworkProfile.ServiceCidr = &s.ServiceCIDR if s.DNSServiceIP == nil { - managedCluster.NetworkProfile.ServiceCidr = &s.ServiceCIDR ip, _, err := net.ParseCIDR(s.ServiceCIDR) if err != nil { return nil, fmt.Errorf("failed to parse service cidr: %w", err) @@ -244,12 +313,19 @@ func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{ } } + if s.DockerBridgeCidr != nil { + managedCluster.NetworkProfile.DockerBridgeCidr = s.DockerBridgeCidr + } + if s.AADProfile != nil { managedCluster.AadProfile = &containerservice.ManagedClusterAADProfile{ Managed: &s.AADProfile.Managed, EnableAzureRBAC: &s.AADProfile.EnableAzureRBAC, AdminGroupObjectIDs: &s.AADProfile.AdminGroupObjectIDs, } + if s.DisableLocalAccounts != nil { + managedCluster.DisableLocalAccounts = s.DisableLocalAccounts + } } for i := range s.AddonProfiles { @@ -303,6 +379,32 @@ func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{ } } + if s.OutboundType != nil { + managedCluster.NetworkProfile.OutboundType = containerservice.OutboundType(*s.OutboundType) + } + + if s.UserAssignedIdentities == nil { + // system assigned assumed if no user assigned input + managedCluster.Identity = &containerservice.ManagedClusterIdentity{ + Type: containerservice.ResourceIdentityType(infrav1.VMIdentitySystemAssigned), + } + } else { + uaIDs := make(map[string]*containerservice.ManagedClusterIdentityUserAssignedIdentitiesValue) + uaIDs[s.UserAssignedIdentities[0].ProviderID] = &containerservice.ManagedClusterIdentityUserAssignedIdentitiesValue{ + // intentionally empty + } + managedCluster.Identity = &containerservice.ManagedClusterIdentity{ + Type: containerservice.ResourceIdentityType(infrav1.VMIdentityUserAssigned), + UserAssignedIdentities: uaIDs, + } + } + + if s.AutoUpgradeProfile != nil { + managedCluster.AutoUpgradeProfile = &containerservice.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: containerservice.UpgradeChannel(s.AutoUpgradeProfile.UpgradeChannel), + } + } + if existing != nil { existingMC, ok := existing.(containerservice.ManagedCluster) if !ok { @@ -313,6 +415,22 @@ func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{ return nil, azure.WithTransientError(errors.Errorf("Unable to update existing managed cluster in non-terminal state. Managed cluster must be in one of the following provisioning states: Canceled, Failed, or Succeeded. Actual state: %s", ps), 20*time.Second) } + if managedCluster.AddonProfiles == nil && existingMC.AddonProfiles != nil { + managedCluster.AddonProfiles = map[string]*containerservice.ManagedClusterAddonProfile{} + } + for key, item := range existingMC.AddonProfiles { + if _, ok := managedCluster.AddonProfiles[key]; !ok { + addonProfile := &containerservice.ManagedClusterAddonProfile{ + Enabled: to.BoolPtr(false), + } + if item.Config != nil { + addonProfile.Config = item.Config + } + managedCluster.AddonProfiles[key] = addonProfile + } + existingMC.AddonProfiles[key].Identity = nil + } + // Normalize the LoadBalancerProfile so the diff below doesn't get thrown off by AKS added properties. if managedCluster.NetworkProfile.LoadBalancerProfile == nil { // If our LoadBalancerProfile generated by the spec is nil, then don't worry about what AKS has added. @@ -400,12 +518,28 @@ func computeDiffOfNormalizedClusters(managedCluster containerservice.ManagedClus propertiesNormalized.APIServerAccessProfile = &containerservice.ManagedClusterAPIServerAccessProfile{ AuthorizedIPRanges: managedCluster.APIServerAccessProfile.AuthorizedIPRanges, } + if managedCluster.APIServerAccessProfile.AuthorizedIPRanges == nil || len(*managedCluster.APIServerAccessProfile.AuthorizedIPRanges) == 0 { + propertiesNormalized.APIServerAccessProfile.AuthorizedIPRanges = nil + } + if managedCluster.APIServerAccessProfile.PrivateDNSZone != nil { + propertiesNormalized.APIServerAccessProfile.PrivateDNSZone = managedCluster.APIServerAccessProfile.PrivateDNSZone + } } if existingMC.APIServerAccessProfile != nil { existingMCPropertiesNormalized.APIServerAccessProfile = &containerservice.ManagedClusterAPIServerAccessProfile{ AuthorizedIPRanges: existingMC.APIServerAccessProfile.AuthorizedIPRanges, } + if existingMC.APIServerAccessProfile.AuthorizedIPRanges == nil || len(*existingMC.APIServerAccessProfile.AuthorizedIPRanges) == 0 { + propertiesNormalized.APIServerAccessProfile.AuthorizedIPRanges = nil + } + if existingMC.APIServerAccessProfile.PrivateDNSZone != nil { + existingMCPropertiesNormalized.APIServerAccessProfile.PrivateDNSZone = existingMC.APIServerAccessProfile.PrivateDNSZone + } + } + + if managedCluster.AddonProfiles != nil { + propertiesNormalized.AddonProfiles = managedCluster.AddonProfiles } clusterNormalized := &containerservice.ManagedCluster{ @@ -417,6 +551,10 @@ func computeDiffOfNormalizedClusters(managedCluster containerservice.ManagedClus Tags: existingMC.Tags, } + if existingMC.AddonProfiles != nil { + existingMCClusterNormalized.AddonProfiles = existingMC.AddonProfiles + } + if managedCluster.Sku != nil { clusterNormalized.Sku = managedCluster.Sku } @@ -424,6 +562,53 @@ func computeDiffOfNormalizedClusters(managedCluster containerservice.ManagedClus existingMCClusterNormalized.Sku = existingMC.Sku } + if managedCluster.Identity != nil { + uaIDs := make(map[string]*containerservice.ManagedClusterIdentityUserAssignedIdentitiesValue) + for key := range managedCluster.Identity.UserAssignedIdentities { + uaIDs[key] = &containerservice.ManagedClusterIdentityUserAssignedIdentitiesValue{ + // intentionally empty + } + } + clusterNormalized.Identity = &containerservice.ManagedClusterIdentity{ + UserAssignedIdentities: uaIDs, + } + } + + if existingMC.Identity != nil { + uaIDs := make(map[string]*containerservice.ManagedClusterIdentityUserAssignedIdentitiesValue) + for key := range existingMC.Identity.UserAssignedIdentities { + uaIDs[key] = &containerservice.ManagedClusterIdentityUserAssignedIdentitiesValue{ + // intentionally empty + } + } + existingMCClusterNormalized.Identity = &containerservice.ManagedClusterIdentity{ + UserAssignedIdentities: uaIDs, + } + } + + if managedCluster.AutoUpgradeProfile != nil { + clusterNormalized.AutoUpgradeProfile = &containerservice.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: managedCluster.AutoUpgradeProfile.UpgradeChannel, + } + } + + if existingMC.AutoUpgradeProfile != nil { + if existingMC.AutoUpgradeProfile.UpgradeChannel == "" { + existingMC.AutoUpgradeProfile = nil + } else { + existingMCClusterNormalized.AutoUpgradeProfile = &containerservice.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: existingMC.AutoUpgradeProfile.UpgradeChannel, + } + } + } + if managedCluster.DisableLocalAccounts != nil { + clusterNormalized.DisableLocalAccounts = managedCluster.DisableLocalAccounts + } + + if existingMC.DisableLocalAccounts != nil { + existingMCClusterNormalized.DisableLocalAccounts = existingMC.DisableLocalAccounts + } + diff := cmp.Diff(clusterNormalized, existingMCClusterNormalized) return diff } diff --git a/azure/services/managedclusters/spec_test.go b/azure/services/managedclusters/spec_test.go index e6256e900a0..0347d9f4107 100644 --- a/azure/services/managedclusters/spec_test.go +++ b/azure/services/managedclusters/spec_test.go @@ -110,6 +110,23 @@ func TestParameters(t *testing.T) { g.Expect(result).To(BeNil()) }, }, + { + name: "managedcluster exists, no update needed", + existing: getExistingCluster(), + spec: &ManagedClusterSpec{ + Name: "test-managedcluster", + ResourceGroup: "test-rg", + Location: "test-location", + Tags: map[string]string{ + "test-tag": "test-value", + }, + Version: "v1.21.0", + LoadBalancerSKU: "Standard", + }, + expect: func(g *WithT, result interface{}) { + g.Expect(result).To(BeNil()) + }, + }, { name: "managedcluster exists and an update is needed", existing: getExistingCluster(), @@ -158,7 +175,6 @@ func getSampleManagedCluster() containerservice.ManagedCluster { return containerservice.ManagedCluster{ ManagedClusterProperties: &containerservice.ManagedClusterProperties{ KubernetesVersion: to.StringPtr("v1.22.0"), - DNSPrefix: to.StringPtr("test-managedcluster"), AgentPoolProfiles: &[]containerservice.ManagedClusterAgentPoolProfile{ converters.AgentPoolToManagedClusterAgentPoolProfile(azure.AgentPoolSpec{ Name: "test-agentpool-0", diff --git a/azure/services/scalesets/scalesets.go b/azure/services/scalesets/scalesets.go index dafd7706897..2bddfe7ddb7 100644 --- a/azure/services/scalesets/scalesets.go +++ b/azure/services/scalesets/scalesets.go @@ -29,6 +29,7 @@ import ( "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/converters" "sigs.k8s.io/cluster-api-provider-azure/azure/services/resourceskus" + azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" "sigs.k8s.io/cluster-api-provider-azure/util/generators" "sigs.k8s.io/cluster-api-provider-azure/util/slice" "sigs.k8s.io/cluster-api-provider-azure/util/tele" @@ -104,7 +105,12 @@ func (s *Service) Reconcile(ctx context.Context) (retErr error) { } if fetchedVMSS != nil { - s.Scope.SetProviderID(azure.ProviderIDPrefix + fetchedVMSS.ID) + // Transform the VMSS resource representation to conform to the cloud-provider-azure representation + providerID, err := azureutil.ConvertResourceGroupNameToLower(azure.ProviderIDPrefix + fetchedVMSS.ID) + if err != nil { + log.Error(err, "failed to parse VMSS ID", "ID", fetchedVMSS.ID) + } + s.Scope.SetProviderID(providerID) s.Scope.SetVMSSState(fetchedVMSS) } }() diff --git a/azure/services/scalesets/scalesets_test.go b/azure/services/scalesets/scalesets_test.go index b252c5e77a1..fabe05e48c0 100644 --- a/azure/services/scalesets/scalesets_test.go +++ b/azure/services/scalesets/scalesets_test.go @@ -909,7 +909,7 @@ func newWindowsVMSSSpec() azure.ScaleSetSpec { func newDefaultExistingVMSS(vmSize string) compute.VirtualMachineScaleSet { vmss := newDefaultVMSS(vmSize) - vmss.ID = to.StringPtr("vmss-id") + vmss.ID = to.StringPtr("subscriptions/1234/resourceGroups/my_resource_group/providers/Microsoft.Compute/virtualMachines/my-vm") return vmss } @@ -1153,7 +1153,7 @@ func newDefaultInstances() []compute.VirtualMachineScaleSetVM { } func setupDefaultVMSSInProgressOperationDoneExpectations(s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder, createdVMSS compute.VirtualMachineScaleSet, instances []compute.VirtualMachineScaleSetVM) { - createdVMSS.ID = to.StringPtr("vmss-id") + createdVMSS.ID = to.StringPtr("subscriptions/1234/resourceGroups/my_resource_group/providers/Microsoft.Compute/virtualMachines/my-vm") createdVMSS.ProvisioningState = to.StringPtr(string(infrav1.Succeeded)) setupDefaultVMSSExpectations(s) future := &infrav1.Future{ @@ -1243,7 +1243,7 @@ func setupVMSSExpectationsWithoutVMImage(s *mock_scalesets.MockScaleSetScopeMock func setupDefaultVMSSUpdateExpectations(s *mock_scalesets.MockScaleSetScopeMockRecorder) { setupUpdateVMSSExpectations(s) - s.SetProviderID(azure.ProviderIDPrefix + "vmss-id") + s.SetProviderID(azure.ProviderIDPrefix + "subscriptions/1234/resourceGroups/my_resource_group/providers/Microsoft.Compute/virtualMachines/my-vm") s.GetLongRunningOperationState(defaultVMSSName, serviceName).Return(nil) s.MaxSurge().Return(1, nil) s.SetVMSSState(gomock.Any()) diff --git a/azure/services/subnets/subnets.go b/azure/services/subnets/subnets.go index 8a92bb1b968..a427beb947e 100644 --- a/azure/services/subnets/subnets.go +++ b/azure/services/subnets/subnets.go @@ -111,7 +111,7 @@ func (s *Service) Delete(ctx context.Context) error { defer cancel() if managed, err := s.IsManaged(ctx); err == nil && !managed { - log.V(4).Info("Skipping subnets deletion in custom vnet mode") + log.Info("Skipping subnets deletion in custom vnet mode") return nil } else if err != nil { return errors.Wrap(err, "failed to check if subnets are managed") diff --git a/azure/services/token/client.go b/azure/services/token/client.go new file mode 100644 index 00000000000..23f6e473cab --- /dev/null +++ b/azure/services/token/client.go @@ -0,0 +1,80 @@ +package token + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + az "github.com/Azure/go-autorest/autorest/azure" + "github.com/pkg/errors" + "sigs.k8s.io/cluster-api-provider-azure/azure" + "sigs.k8s.io/cluster-api-provider-azure/util/tele" +) + +const ( + defaultEnvironmentName = "AzurePublicCloud" +) + +type AzureClient struct { + aadToken *azidentity.ClientSecretCredential +} + +// newClient creates a new managed cluster client from an authorizer. +func NewClient(auth azure.Authorizer) (*AzureClient, error) { + aadToken, err := newAzureActiveDirectoryTokenClient(auth.TenantID(), + auth.ClientID(), + auth.ClientSecret(), + auth.CloudEnvironment()) + if err != nil { + return nil, err + } + return &AzureClient{ + aadToken: aadToken, + }, nil +} + +// newAzureActiveDirectoryTokenClient creates a new aad token client from an authorizer. +func newAzureActiveDirectoryTokenClient(tenantId, clientId, clientSecret, envName string) (*azidentity.ClientSecretCredential, error) { + cliOpts, err := getAzureClientOptions(envName) + if err != nil { + return nil, errors.Wrap(err, "error while getting client options") + } + clientOptions := &azidentity.ClientSecretCredentialOptions{ + ClientOptions: cliOpts, + } + cred, err := azidentity.NewClientSecretCredential(tenantId, clientId, clientSecret, clientOptions) + if err != nil { + return nil, errors.Wrap(err, "error while getting az client secret credentials") + } + return cred, nil +} + +func getAzureClientOptions(environment string) (azcore.ClientOptions, error) { + + if environment == "" { + environment = defaultEnvironmentName + } + env, err := az.EnvironmentFromName(environment) + if err != nil { + return azcore.ClientOptions{}, errors.Wrap(err, "error while getting azure env") + } + c := cloud.Configuration{ + ActiveDirectoryAuthorityHost: env.ActiveDirectoryEndpoint, + } + return azcore.ClientOptions{ + Cloud: c, + }, nil +} + +func (ac *AzureClient) GetAzureActiveDirectoryToken(ctx context.Context, resourceId string) (string, error) { + ctx, _, done := tele.StartSpanWithLogger(ctx, "aadToken.GetToken") + defer done() + + spnAccessToken, err := ac.aadToken.GetToken(ctx, policy.TokenRequestOptions{Scopes: []string{resourceId + "/.default"}}) + if err != nil { + return "", errors.Wrap(err, "failed to get token") + } + return spnAccessToken.Token, nil +} diff --git a/azure/services/virtualmachines/virtualmachines.go b/azure/services/virtualmachines/virtualmachines.go index 79b0ba5d5d8..c52cd9738f6 100644 --- a/azure/services/virtualmachines/virtualmachines.go +++ b/azure/services/virtualmachines/virtualmachines.go @@ -31,6 +31,7 @@ import ( "sigs.k8s.io/cluster-api-provider-azure/azure/services/async" "sigs.k8s.io/cluster-api-provider-azure/azure/services/networkinterfaces" "sigs.k8s.io/cluster-api-provider-azure/azure/services/publicips" + azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" "sigs.k8s.io/cluster-api-provider-azure/util/tele" ) @@ -98,7 +99,12 @@ func (s *Service) Reconcile(ctx context.Context) error { if err != nil { return err } - s.Scope.SetProviderID(azure.ProviderIDPrefix + infraVM.ID) + // Transform the VM resource representation to conform to the cloud-provider-azure representation + providerID, err := azureutil.ConvertResourceGroupNameToLower(azure.ProviderIDPrefix + infraVM.ID) + if err != nil { + return errors.Wrapf(err, "failed to parse VM ID %s", infraVM.ID) + } + s.Scope.SetProviderID(providerID) s.Scope.SetAnnotation("cluster-api-provider-azure", "true") // Discover addresses for NICs associated with the VM diff --git a/azure/services/virtualmachines/virtualmachines_test.go b/azure/services/virtualmachines/virtualmachines_test.go index d3bb10a9d74..674cb5678f1 100644 --- a/azure/services/virtualmachines/virtualmachines_test.go +++ b/azure/services/virtualmachines/virtualmachines_test.go @@ -53,7 +53,7 @@ var ( BootstrapData: "fake data", } fakeExistingVM = compute.VirtualMachine{ - ID: to.StringPtr("test-vm-id"), + ID: to.StringPtr("subscriptions/123/resourceGroups/my_resource_group/providers/Microsoft.Compute/virtualMachines/my-vm"), Name: to.StringPtr("test-vm-name"), VirtualMachineProperties: &compute.VirtualMachineProperties{ ProvisioningState: to.StringPtr("Succeeded"), @@ -127,7 +127,7 @@ func TestReconcileVM(t *testing.T) { r.CreateResource(gomockinternal.AContext(), &fakeVMSpec, serviceName).Return(fakeExistingVM, nil) s.UpdatePutStatus(infrav1.VMRunningCondition, serviceName, nil) s.UpdatePutStatus(infrav1.DisksReadyCondition, serviceName, nil) - s.SetProviderID("azure://test-vm-id") + s.SetProviderID("azure://subscriptions/123/resourceGroups/my_resource_group/providers/Microsoft.Compute/virtualMachines/my-vm") s.SetAnnotation("cluster-api-provider-azure", "true") mnic.Get(gomockinternal.AContext(), &fakeNetworkInterfaceGetterSpec).Return(fakeNetworkInterface, nil) mpip.Get(gomockinternal.AContext(), "test-group", "pip-1").Return(fakePublicIPs, nil) @@ -153,7 +153,7 @@ func TestReconcileVM(t *testing.T) { r.CreateResource(gomockinternal.AContext(), &fakeVMSpec, serviceName).Return(fakeExistingVM, nil) s.UpdatePutStatus(infrav1.VMRunningCondition, serviceName, nil) s.UpdatePutStatus(infrav1.DisksReadyCondition, serviceName, nil) - s.SetProviderID("azure://test-vm-id") + s.SetProviderID("azure://subscriptions/123/resourceGroups/my_resource_group/providers/Microsoft.Compute/virtualMachines/my-vm") s.SetAnnotation("cluster-api-provider-azure", "true") mnic.Get(gomockinternal.AContext(), &fakeNetworkInterfaceGetterSpec).Return(network.Interface{}, internalError) }, @@ -166,7 +166,7 @@ func TestReconcileVM(t *testing.T) { r.CreateResource(gomockinternal.AContext(), &fakeVMSpec, serviceName).Return(fakeExistingVM, nil) s.UpdatePutStatus(infrav1.VMRunningCondition, serviceName, nil) s.UpdatePutStatus(infrav1.DisksReadyCondition, serviceName, nil) - s.SetProviderID("azure://test-vm-id") + s.SetProviderID("azure://subscriptions/123/resourceGroups/my_resource_group/providers/Microsoft.Compute/virtualMachines/my-vm") s.SetAnnotation("cluster-api-provider-azure", "true") mnic.Get(gomockinternal.AContext(), &fakeNetworkInterfaceGetterSpec).Return(fakeNetworkInterface, nil) mpip.Get(gomockinternal.AContext(), "test-group", "pip-1").Return(network.PublicIPAddress{}, internalError) diff --git a/azure/types.go b/azure/types.go index 3e0ad04199e..fd681fdfb2f 100644 --- a/azure/types.go +++ b/azure/types.go @@ -105,6 +105,9 @@ type AgentPoolSpec struct { // EnableUltraSSD enables the storage type UltraSSD_LRS for the agent pool. EnableUltraSSD *bool `json:"enableUltraSSD,omitempty"` + + // OSType specifies the operating system for the node pool. Allowed values are 'Linux' and 'Windows' + OSType *string `json:"osType,omitempty"` } // ScaleSetSpec defines the specification for a Scale Set. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusteridentities.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusteridentities.yaml index 22517fe9ac5..ceefbf2f571 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusteridentities.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusteridentities.yaml @@ -56,11 +56,11 @@ spec: either a Service Principal password or certificate secret. properties: name: - description: Name is unique within a namespace to reference a + description: name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which the secret + description: namespace defines the space within which the secret name must be unique. type: string type: object @@ -233,11 +233,11 @@ spec: either a Service Principal password or certificate secret. properties: name: - description: Name is unique within a namespace to reference a + description: name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which the secret + description: namespace defines the space within which the secret name must be unique. type: string type: object @@ -411,11 +411,11 @@ spec: either a Service Principal password or certificate secret. properties: name: - description: Name is unique within a namespace to reference a + description: name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which the secret + description: namespace defines the space within which the secret name must be unique. type: string type: object diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml index c95d2fbcf14..e56393c5d16 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml @@ -1370,6 +1370,7 @@ spec: - node - control-plane - bastion + - all type: string routeTable: description: RouteTable defines the route table that should @@ -1679,6 +1680,8 @@ spec: the TCP idle connection. format: int32 type: integer + ipAllocationMethod: + type: string name: type: string sku: @@ -1733,6 +1736,8 @@ spec: the TCP idle connection. format: int32 type: integer + ipAllocationMethod: + type: string name: type: string sku: @@ -1785,6 +1790,8 @@ spec: the TCP idle connection. format: int32 type: integer + ipAllocationMethod: + type: string name: type: string sku: @@ -1846,6 +1853,7 @@ spec: - node - control-plane - bastion + - all type: string routeTable: description: RouteTable defines the route table that should diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclustertemplates.yaml index 685920a786b..59e6c1da99a 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclustertemplates.yaml @@ -88,6 +88,7 @@ spec: - node - control-plane - bastion + - all type: string securityGroup: description: SecurityGroup defines the NSG (network @@ -334,6 +335,8 @@ spec: for the TCP idle connection. format: int32 type: integer + ipAllocationMethod: + type: string sku: description: SKU defines an Azure load balancer SKU. type: string @@ -353,6 +356,8 @@ spec: for the TCP idle connection. format: int32 type: integer + ipAllocationMethod: + type: string sku: description: SKU defines an Azure load balancer SKU. type: string @@ -370,6 +375,8 @@ spec: for the TCP idle connection. format: int32 type: integer + ipAllocationMethod: + type: string sku: description: SKU defines an Azure load balancer SKU. type: string @@ -409,6 +416,7 @@ spec: - node - control-plane - bastion + - all type: string securityGroup: description: SecurityGroup defines the NSG (network diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml index f161c796070..5ab7c8e8ac0 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml @@ -407,6 +407,10 @@ spec: type: string name: type: string + resourceGroupName: + description: ResourceGroupName is the name of the Azure resource + group for the VNet and Subnet. + type: string subnet: description: ManagedControlPlaneSubnet describes a subnet for an AKS cluster. @@ -571,10 +575,23 @@ spec: privateDNSZone: description: PrivateDNSZone - Private dns zone mode for private cluster. + type: string + type: object + autoUpgradeProfile: + description: AutoUpgradeProfile - Profile of auto upgrade configuration. + properties: + upgradeChannel: + description: 'UpgradeChannel - upgrade channel for auto upgrade. + Possible values include: "node-image","none","patch","rapid","stable"' enum: - - System - - None + - node-image + - none + - patch + - rapid + - stable type: string + required: + - upgradeChannel type: object controlPlaneEndpoint: description: ControlPlaneEndpoint represents the endpoint used to @@ -591,11 +608,29 @@ spec: - host - port type: object + disableLocalAccounts: + description: DisableLocalAccounts - If set to true, getting static + credential will be disabled for this cluster. Expected to only be + used for AAD clusters. + type: boolean + dnsPrefix: + description: DNSPrefix - DNS prefix specified when creating the managed + cluster. + type: string dnsServiceIP: description: DNSServiceIP is an IP address assigned to the Kubernetes DNS service. It must be within the Kubernetes service address range specified in serviceCidr. type: string + dockerBridgeCidr: + description: DockerBridgeCidr - A CIDR notation IP range assigned + to the Docker bridge network. It must not overlap with any Subnet + IP ranges or the Kubernetes service address range. + type: string + fqdnSubdomain: + description: FqdnSubdomain - FQDN subdomain specified when creating + private cluster with custom private dns zone. + type: string identityRef: description: IdentityRef is a reference to a AzureClusterIdentity to be used when reconciling this cluster @@ -696,6 +731,14 @@ spec: containing cluster IaaS resources. Will be populated to default in webhook. type: string + outboundType: + description: Outbound configuration used by Nodes. + enum: + - loadBalancer + - managedNATGateway + - userAssignedNATGateway + - userDefinedRouting + type: string resourceGroupName: description: ResourceGroupName is the name of the Azure resource group for this AKS Cluster. @@ -720,6 +763,21 @@ spec: description: SubscriptionID is the GUID of the Azure subscription to hold this cluster. type: string + userAssignedIdentities: + description: UserAssignedIdentities is a list of standalone Azure + identities provided by the user to assign the cluster + items: + description: UserAssignedIdentity defines the user-assigned identities + provided by the user to be assigned to Azure resources. + properties: + providerID: + description: 'ProviderID is the identification ID of the user-assigned + Identity, the format of an identity is: ''azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}''' + type: string + required: + - providerID + type: object + type: array version: description: Version defines the desired Kubernetes version. minLength: 2 @@ -732,6 +790,10 @@ spec: type: string name: type: string + resourceGroupName: + description: ResourceGroupName is the name of the Azure resource + group for the VNet and Subnet. + type: string subnet: description: ManagedControlPlaneSubnet describes a subnet for an AKS cluster. @@ -758,6 +820,11 @@ spec: description: AzureManagedControlPlaneStatus defines the observed state of AzureManagedControlPlane. properties: + autoUpgradeVersion: + description: AutoUpgradeVersion is the Kubernetes version populated + after autoupgrade based on the upgrade channel. + minLength: 2 + type: string conditions: description: Conditions defines current service state of the AzureManagedControlPlane. items: diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml index 425b15f688a..3b18a66ad2b 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml @@ -133,6 +133,13 @@ spec: according to the vmSize specified. format: int32 type: integer + osType: + description: 'OSType specifies the virtual machine operating system. + Default to Linux. Possible values include: ''Linux'', ''Windows''' + enum: + - Linux + - Windows + type: string providerIDList: description: ProviderIDList is the unique identifier as specified by the cloud provider. @@ -245,6 +252,13 @@ spec: - Ephemeral - Managed type: string + osType: + description: 'OSType specifies the virtual machine operating system. + Default to Linux. Possible values include: ''Linux'', ''Windows''' + enum: + - Linux + - Windows + type: string providerIDList: description: ProviderIDList is the unique identifier as specified by the cloud provider. @@ -265,6 +279,10 @@ spec: sku: description: SKU is the size of the VMs in the node pool. type: string + subnetName: + description: SubnetName selects the Subnet where the MachinePool will + be placed + type: string taints: description: Taints specifies the taints for nodes present in this agent pool. diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index f41ea1f58f7..38f11946501 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -28,9 +28,9 @@ patchesStrategicMerge: - patches/webhook_in_azuremachinetemplates.yaml - patches/webhook_in_azuremachinepools.yaml - patches/webhook_in_azuremachinepoolmachines.yaml - # - patches/webhook_in_azuremanagedmachinepools.yaml - # - patches/webhook_in_azuremanagedclusters.yaml - # - patches/webhook_in_azuremanagedcontrolplanes.yaml + - patches/webhook_in_azuremanagedmachinepools.yaml + - patches/webhook_in_azuremanagedclusters.yaml + - patches/webhook_in_azuremanagedcontrolplanes.yaml # +kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. @@ -42,9 +42,9 @@ patchesStrategicMerge: - patches/cainjection_in_azuremachinetemplates.yaml - patches/cainjection_in_azuremachinepools.yaml - patches/cainjection_in_azuremachinepoolmachines.yaml - # - patches/cainjection_in_azuremanagedmachinepools.yaml - # - patches/cainjection_in_azuremanagedclusters.yaml - # - patches/cainjection_in_azuremanagedcontrolplanes.yaml + - patches/cainjection_in_azuremanagedmachinepools.yaml + - patches/cainjection_in_azuremanagedclusters.yaml + - patches/cainjection_in_azuremanagedcontrolplanes.yaml # +kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml index 0876a1db40d..4242ef31345 100644 --- a/config/default/manager_image_patch.yaml +++ b/config/default/manager_image_patch.yaml @@ -8,5 +8,5 @@ spec: spec: containers: # Change the value of image field below to your controller image URL - - image: gcr.io/k8s-staging-cluster-api-azure/cluster-api-azure-controller:latest + - image: gcr.io/spectro-dev-public/ubuntu/release/cluster-api-azure-controller:v1.3.2-spectro-4.0.0-dev name: manager diff --git a/controllers/azureidentity_controller.go b/controllers/azureidentity_controller.go index a8b399e6c94..9688d36df69 100644 --- a/controllers/azureidentity_controller.go +++ b/controllers/azureidentity_controller.go @@ -185,7 +185,7 @@ func (r *AzureIdentityReconciler) Reconcile(ctx context.Context, req ctrl.Reques } } - // delete bindings and identites no longer used by a cluster + // delete bindings and identities no longer used by a cluster for _, bindingToDelete := range bindingsToDelete { binding := bindingToDelete identityName := binding.Spec.AzureIdentity diff --git a/controllers/azurejson_machine_controller.go b/controllers/azurejson_machine_controller.go index 37348396a94..9839565a061 100644 --- a/controllers/azurejson_machine_controller.go +++ b/controllers/azurejson_machine_controller.go @@ -18,7 +18,6 @@ package controllers import ( "context" - "fmt" "time" "github.com/go-logr/logr" @@ -195,8 +194,8 @@ func (r *AzureJSONMachineReconciler) Reconcile(ctx context.Context, req ctrl.Req } if azureMachine.Spec.Identity == infrav1.VMIdentityNone { - log.Info(fmt.Sprintf("WARNING, %s", spIdentityWarning)) - r.Recorder.Eventf(azureMachine, corev1.EventTypeWarning, "VMIdentityNone", spIdentityWarning) + //log.Info(fmt.Sprintf("WARNING, %s", spIdentityWarning)) + //r.Recorder.Eventf(azureMachine, corev1.EventTypeWarning, "VMIdentityNone", spIdentityWarning) } newSecret, err := GetCloudProviderSecret( diff --git a/controllers/azurejson_machinepool_controller.go b/controllers/azurejson_machinepool_controller.go index 0e1d1a7db2a..921ef544347 100644 --- a/controllers/azurejson_machinepool_controller.go +++ b/controllers/azurejson_machinepool_controller.go @@ -18,7 +18,6 @@ package controllers import ( "context" - "fmt" "time" "github.com/pkg/errors" @@ -163,8 +162,8 @@ func (r *AzureJSONMachinePoolReconciler) Reconcile(ctx context.Context, req ctrl } if azureMachinePool.Spec.Identity == infrav1.VMIdentityNone { - log.Info(fmt.Sprintf("WARNING, %s", spIdentityWarning)) - r.Recorder.Eventf(azureMachinePool, corev1.EventTypeWarning, "VMIdentityNone", spIdentityWarning) + //log.Info(fmt.Sprintf("WARNING, %s", spIdentityWarning)) + //r.Recorder.Eventf(azureMachinePool, corev1.EventTypeWarning, "VMIdentityNone", spIdentityWarning) } newSecret, err := GetCloudProviderSecret( diff --git a/controllers/azurejson_machinetemplate_controller.go b/controllers/azurejson_machinetemplate_controller.go index 6b63756e129..291c2fb91d4 100644 --- a/controllers/azurejson_machinetemplate_controller.go +++ b/controllers/azurejson_machinetemplate_controller.go @@ -18,7 +18,6 @@ package controllers import ( "context" - "fmt" "time" "github.com/pkg/errors" @@ -154,8 +153,8 @@ func (r *AzureJSONTemplateReconciler) Reconcile(ctx context.Context, req ctrl.Re } if azureMachineTemplate.Spec.Template.Spec.Identity == infrav1.VMIdentityNone { - log.Info(fmt.Sprintf("WARNING, %s", spIdentityWarning)) - r.Recorder.Eventf(azureMachineTemplate, corev1.EventTypeWarning, "VMIdentityNone", spIdentityWarning) + //log.Info(fmt.Sprintf("WARNING, %s", spIdentityWarning)) + //r.Recorder.Eventf(azureMachineTemplate, corev1.EventTypeWarning, "VMIdentityNone", spIdentityWarning) } newSecret, err := GetCloudProviderSecret( diff --git a/controllers/helpers.go b/controllers/helpers.go index d04b63f37cd..ed7478d6006 100644 --- a/controllers/helpers.go +++ b/controllers/helpers.go @@ -283,7 +283,7 @@ func newCloudProviderConfig(d azure.ClusterScoper) (controlPlaneConfig *CloudPro // getOneNodeSubnet returns one of the subnets for the node role. func getOneNodeSubnet(d azure.ClusterScoper) infrav1.SubnetSpec { for _, subnet := range d.Subnets() { - if subnet.Role == infrav1.SubnetNode { + if subnet.Role == infrav1.SubnetNode || subnet.Role == infrav1.SubnetAll { return subnet } } diff --git a/exp/api/v1alpha3/azuremanagedcontrolplane_conversion.go b/exp/api/v1alpha3/azuremanagedcontrolplane_conversion.go index b0e1f4fd729..75d2e332808 100644 --- a/exp/api/v1alpha3/azuremanagedcontrolplane_conversion.go +++ b/exp/api/v1alpha3/azuremanagedcontrolplane_conversion.go @@ -81,3 +81,12 @@ func (dst *AzureManagedControlPlaneList) ConvertFrom(srcRaw conversion.Hub) erro src := srcRaw.(*expv1beta1.AzureManagedControlPlaneList) return Convert_v1beta1_AzureManagedControlPlaneList_To_v1alpha3_AzureManagedControlPlaneList(src, dst, nil) } + +func Convert_v1beta1_ManagedControlPlaneVirtualNetwork_To_v1alpha3_ManagedControlPlaneVirtualNetwork(in *expv1beta1.ManagedControlPlaneVirtualNetwork, out *ManagedControlPlaneVirtualNetwork, s apiconversion.Scope) error { + out.Name = in.Name + out.Subnet.Name = in.Subnet.Name + out.Subnet.CIDRBlock = in.Subnet.CIDRBlock + out.CIDRBlock = in.CIDRBlock + + return nil +} diff --git a/exp/api/v1alpha3/azuremanagedmachinepool_conversion.go b/exp/api/v1alpha3/azuremanagedmachinepool_conversion.go index 503fa8b4bf6..ed1a40c73e0 100644 --- a/exp/api/v1alpha3/azuremanagedmachinepool_conversion.go +++ b/exp/api/v1alpha3/azuremanagedmachinepool_conversion.go @@ -44,6 +44,7 @@ func (src *AzureManagedMachinePool) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.OsDiskType = restored.Spec.OsDiskType dst.Spec.NodeLabels = restored.Spec.NodeLabels dst.Spec.EnableUltraSSD = restored.Spec.EnableUltraSSD + dst.Spec.SubnetName = restored.Spec.SubnetName dst.Status.LongRunningOperationStates = restored.Status.LongRunningOperationStates dst.Status.Conditions = restored.Status.Conditions diff --git a/exp/api/v1alpha3/zz_generated.conversion.go b/exp/api/v1alpha3/zz_generated.conversion.go index af9e278987c..f2af706016e 100644 --- a/exp/api/v1alpha3/zz_generated.conversion.go +++ b/exp/api/v1alpha3/zz_generated.conversion.go @@ -206,11 +206,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta1.ManagedControlPlaneVirtualNetwork)(nil), (*ManagedControlPlaneVirtualNetwork)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_ManagedControlPlaneVirtualNetwork_To_v1alpha3_ManagedControlPlaneVirtualNetwork(a.(*v1beta1.ManagedControlPlaneVirtualNetwork), b.(*ManagedControlPlaneVirtualNetwork), scope) - }); err != nil { - return err - } if err := s.AddConversionFunc((*apiv1alpha3.APIEndpoint)(nil), (*apiv1beta1.APIEndpoint)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_APIEndpoint_To_v1beta1_APIEndpoint(a.(*apiv1alpha3.APIEndpoint), b.(*apiv1beta1.APIEndpoint), scope) }); err != nil { @@ -276,6 +271,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1beta1.ManagedControlPlaneVirtualNetwork)(nil), (*ManagedControlPlaneVirtualNetwork)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ManagedControlPlaneVirtualNetwork_To_v1alpha3_ManagedControlPlaneVirtualNetwork(a.(*v1beta1.ManagedControlPlaneVirtualNetwork), b.(*ManagedControlPlaneVirtualNetwork), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*clusterapiproviderazureapiv1beta1.OSDisk)(nil), (*clusterapiproviderazureapiv1alpha3.OSDisk)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_OSDisk_To_v1alpha3_OSDisk(a.(*clusterapiproviderazureapiv1beta1.OSDisk), b.(*clusterapiproviderazureapiv1alpha3.OSDisk), scope) }); err != nil { @@ -758,6 +758,10 @@ func autoConvert_v1beta1_AzureManagedControlPlaneSpec_To_v1alpha3_AzureManagedCo out.AdditionalTags = *(*clusterapiproviderazureapiv1alpha3.Tags)(unsafe.Pointer(&in.AdditionalTags)) out.NetworkPlugin = (*string)(unsafe.Pointer(in.NetworkPlugin)) out.NetworkPolicy = (*string)(unsafe.Pointer(in.NetworkPolicy)) + // WARNING: in.OutboundType requires manual conversion: does not exist in peer-type + // WARNING: in.DockerBridgeCidr requires manual conversion: does not exist in peer-type + // WARNING: in.DNSPrefix requires manual conversion: does not exist in peer-type + // WARNING: in.FqdnSubdomain requires manual conversion: does not exist in peer-type out.SSHPublicKey = in.SSHPublicKey out.DNSServiceIP = (*string)(unsafe.Pointer(in.DNSServiceIP)) out.LoadBalancerSKU = (*string)(unsafe.Pointer(in.LoadBalancerSKU)) @@ -767,6 +771,9 @@ func autoConvert_v1beta1_AzureManagedControlPlaneSpec_To_v1alpha3_AzureManagedCo // WARNING: in.SKU requires manual conversion: does not exist in peer-type // WARNING: in.LoadBalancerProfile requires manual conversion: does not exist in peer-type // WARNING: in.APIServerAccessProfile requires manual conversion: does not exist in peer-type + // WARNING: in.UserAssignedIdentities requires manual conversion: does not exist in peer-type + // WARNING: in.AutoUpgradeProfile requires manual conversion: does not exist in peer-type + // WARNING: in.DisableLocalAccounts requires manual conversion: does not exist in peer-type return nil } @@ -782,6 +789,7 @@ func Convert_v1alpha3_AzureManagedControlPlaneStatus_To_v1beta1_AzureManagedCont } func autoConvert_v1beta1_AzureManagedControlPlaneStatus_To_v1alpha3_AzureManagedControlPlaneStatus(in *v1beta1.AzureManagedControlPlaneStatus, out *AzureManagedControlPlaneStatus, s conversion.Scope) error { + // WARNING: in.AutoUpgradeVersion requires manual conversion: does not exist in peer-type out.Ready = in.Ready out.Initialized = in.Initialized // WARNING: in.Conditions requires manual conversion: does not exist in peer-type @@ -886,9 +894,11 @@ func autoConvert_v1beta1_AzureManagedMachinePoolSpec_To_v1alpha3_AzureManagedMac // WARNING: in.Taints requires manual conversion: does not exist in peer-type out.ProviderIDList = *(*[]string)(unsafe.Pointer(&in.ProviderIDList)) // WARNING: in.Scaling requires manual conversion: does not exist in peer-type + // WARNING: in.OSType requires manual conversion: does not exist in peer-type // WARNING: in.MaxPods requires manual conversion: does not exist in peer-type // WARNING: in.OsDiskType requires manual conversion: does not exist in peer-type // WARNING: in.EnableUltraSSD requires manual conversion: does not exist in peer-type + // WARNING: in.SubnetName requires manual conversion: does not exist in peer-type return nil } @@ -957,10 +967,6 @@ func autoConvert_v1beta1_ManagedControlPlaneVirtualNetwork_To_v1alpha3_ManagedCo if err := Convert_v1beta1_ManagedControlPlaneSubnet_To_v1alpha3_ManagedControlPlaneSubnet(&in.Subnet, &out.Subnet, s); err != nil { return err } + // WARNING: in.ResourceGroupName requires manual conversion: does not exist in peer-type return nil } - -// Convert_v1beta1_ManagedControlPlaneVirtualNetwork_To_v1alpha3_ManagedControlPlaneVirtualNetwork is an autogenerated conversion function. -func Convert_v1beta1_ManagedControlPlaneVirtualNetwork_To_v1alpha3_ManagedControlPlaneVirtualNetwork(in *v1beta1.ManagedControlPlaneVirtualNetwork, out *ManagedControlPlaneVirtualNetwork, s conversion.Scope) error { - return autoConvert_v1beta1_ManagedControlPlaneVirtualNetwork_To_v1alpha3_ManagedControlPlaneVirtualNetwork(in, out, s) -} diff --git a/exp/api/v1alpha4/azuremanagedcontrolplane_conversion.go b/exp/api/v1alpha4/azuremanagedcontrolplane_conversion.go index 2f891523b3b..9413ce78138 100644 --- a/exp/api/v1alpha4/azuremanagedcontrolplane_conversion.go +++ b/exp/api/v1alpha4/azuremanagedcontrolplane_conversion.go @@ -74,3 +74,13 @@ func (dst *AzureManagedControlPlaneList) ConvertFrom(srcRaw conversion.Hub) erro src := srcRaw.(*expv1beta1.AzureManagedControlPlaneList) return Convert_v1beta1_AzureManagedControlPlaneList_To_v1alpha4_AzureManagedControlPlaneList(src, dst, nil) } + +// +//func Convert_v1beta1_ManagedControlPlaneVirtualNetwork_To_v1alpha4_ManagedControlPlaneVirtualNetwork(in *expv1beta1.ManagedControlPlaneVirtualNetwork, out *ManagedControlPlaneVirtualNetwork, s apiconversion.Scope) error { +// out.Name = in.Name +// out.Subnet.Name = in.Subnet.Name +// out.Subnet.CIDRBlock = in.Subnet.CIDRBlock +// out.CIDRBlock = in.CIDRBlock +// +// return nil +//} diff --git a/exp/api/v1alpha4/azuremanagedcontrolplane_types.go b/exp/api/v1alpha4/azuremanagedcontrolplane_types.go index bbc929817ac..cbb167c50fd 100644 --- a/exp/api/v1alpha4/azuremanagedcontrolplane_types.go +++ b/exp/api/v1alpha4/azuremanagedcontrolplane_types.go @@ -176,6 +176,8 @@ type ManagedControlPlaneVirtualNetwork struct { Name string `json:"name"` CIDRBlock string `json:"cidrBlock"` Subnet ManagedControlPlaneSubnet `json:"subnet,omitempty"` + // ResourceGroupName is the name of the Azure resource group for the VNet and Subnet. + ResourceGroupName string `json:"resourceGroupName,omitempty"` } // ManagedControlPlaneSubnet describes a subnet for an AKS cluster. diff --git a/exp/api/v1alpha4/azuremanagedmachinepool_conversion.go b/exp/api/v1alpha4/azuremanagedmachinepool_conversion.go index 8d329b9df95..af9f1ca93ef 100644 --- a/exp/api/v1alpha4/azuremanagedmachinepool_conversion.go +++ b/exp/api/v1alpha4/azuremanagedmachinepool_conversion.go @@ -44,6 +44,7 @@ func (src *AzureManagedMachinePool) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.OsDiskType = restored.Spec.OsDiskType dst.Spec.NodeLabels = restored.Spec.NodeLabels dst.Spec.EnableUltraSSD = restored.Spec.EnableUltraSSD + dst.Spec.SubnetName = restored.Spec.SubnetName dst.Status.LongRunningOperationStates = restored.Status.LongRunningOperationStates dst.Status.Conditions = restored.Status.Conditions diff --git a/exp/api/v1alpha4/azuremanagedmachinepool_types.go b/exp/api/v1alpha4/azuremanagedmachinepool_types.go index 646c8a3b859..ee34c72706d 100644 --- a/exp/api/v1alpha4/azuremanagedmachinepool_types.go +++ b/exp/api/v1alpha4/azuremanagedmachinepool_types.go @@ -56,6 +56,11 @@ type AzureManagedMachinePoolSpec struct { // ProviderIDList is the unique identifier as specified by the cloud provider. // +optional ProviderIDList []string `json:"providerIDList,omitempty"` + + // OSType specifies the virtual machine operating system. Default to Linux. Possible values include: 'Linux', 'Windows' + // +kubebuilder:validation:Enum=Linux;Windows + // +optional + OSType *string `json:"osType,omitempty"` } // AzureManagedMachinePoolStatus defines the observed state of AzureManagedMachinePool. diff --git a/exp/api/v1alpha4/zz_generated.conversion.go b/exp/api/v1alpha4/zz_generated.conversion.go index 0a9bd141e71..3ed27aae260 100644 --- a/exp/api/v1alpha4/zz_generated.conversion.go +++ b/exp/api/v1alpha4/zz_generated.conversion.go @@ -1061,6 +1061,10 @@ func autoConvert_v1beta1_AzureManagedControlPlaneSpec_To_v1alpha4_AzureManagedCo out.AdditionalTags = *(*clusterapiproviderazureapiv1alpha4.Tags)(unsafe.Pointer(&in.AdditionalTags)) out.NetworkPlugin = (*string)(unsafe.Pointer(in.NetworkPlugin)) out.NetworkPolicy = (*string)(unsafe.Pointer(in.NetworkPolicy)) + // WARNING: in.OutboundType requires manual conversion: does not exist in peer-type + // WARNING: in.DockerBridgeCidr requires manual conversion: does not exist in peer-type + // WARNING: in.DNSPrefix requires manual conversion: does not exist in peer-type + // WARNING: in.FqdnSubdomain requires manual conversion: does not exist in peer-type out.SSHPublicKey = in.SSHPublicKey out.DNSServiceIP = (*string)(unsafe.Pointer(in.DNSServiceIP)) out.LoadBalancerSKU = (*string)(unsafe.Pointer(in.LoadBalancerSKU)) @@ -1070,6 +1074,9 @@ func autoConvert_v1beta1_AzureManagedControlPlaneSpec_To_v1alpha4_AzureManagedCo out.SKU = (*SKU)(unsafe.Pointer(in.SKU)) out.LoadBalancerProfile = (*LoadBalancerProfile)(unsafe.Pointer(in.LoadBalancerProfile)) out.APIServerAccessProfile = (*APIServerAccessProfile)(unsafe.Pointer(in.APIServerAccessProfile)) + // WARNING: in.UserAssignedIdentities requires manual conversion: does not exist in peer-type + // WARNING: in.AutoUpgradeProfile requires manual conversion: does not exist in peer-type + // WARNING: in.DisableLocalAccounts requires manual conversion: does not exist in peer-type return nil } @@ -1086,6 +1093,7 @@ func Convert_v1alpha4_AzureManagedControlPlaneStatus_To_v1beta1_AzureManagedCont } func autoConvert_v1beta1_AzureManagedControlPlaneStatus_To_v1alpha4_AzureManagedControlPlaneStatus(in *v1beta1.AzureManagedControlPlaneStatus, out *AzureManagedControlPlaneStatus, s conversion.Scope) error { + // WARNING: in.AutoUpgradeVersion requires manual conversion: does not exist in peer-type out.Ready = in.Ready out.Initialized = in.Initialized // WARNING: in.Conditions requires manual conversion: does not exist in peer-type @@ -1173,6 +1181,7 @@ func autoConvert_v1alpha4_AzureManagedMachinePoolSpec_To_v1beta1_AzureManagedMac out.SKU = in.SKU out.OSDiskSizeGB = (*int32)(unsafe.Pointer(in.OSDiskSizeGB)) out.ProviderIDList = *(*[]string)(unsafe.Pointer(&in.ProviderIDList)) + out.OSType = (*string)(unsafe.Pointer(in.OSType)) return nil } @@ -1191,9 +1200,11 @@ func autoConvert_v1beta1_AzureManagedMachinePoolSpec_To_v1alpha4_AzureManagedMac // WARNING: in.Taints requires manual conversion: does not exist in peer-type out.ProviderIDList = *(*[]string)(unsafe.Pointer(&in.ProviderIDList)) // WARNING: in.Scaling requires manual conversion: does not exist in peer-type + out.OSType = (*string)(unsafe.Pointer(in.OSType)) // WARNING: in.MaxPods requires manual conversion: does not exist in peer-type // WARNING: in.OsDiskType requires manual conversion: does not exist in peer-type // WARNING: in.EnableUltraSSD requires manual conversion: does not exist in peer-type + // WARNING: in.SubnetName requires manual conversion: does not exist in peer-type return nil } @@ -1300,6 +1311,7 @@ func autoConvert_v1alpha4_ManagedControlPlaneVirtualNetwork_To_v1beta1_ManagedCo if err := Convert_v1alpha4_ManagedControlPlaneSubnet_To_v1beta1_ManagedControlPlaneSubnet(&in.Subnet, &out.Subnet, s); err != nil { return err } + out.ResourceGroupName = in.ResourceGroupName return nil } @@ -1314,6 +1326,7 @@ func autoConvert_v1beta1_ManagedControlPlaneVirtualNetwork_To_v1alpha4_ManagedCo if err := Convert_v1beta1_ManagedControlPlaneSubnet_To_v1alpha4_ManagedControlPlaneSubnet(&in.Subnet, &out.Subnet, s); err != nil { return err } + out.ResourceGroupName = in.ResourceGroupName return nil } diff --git a/exp/api/v1alpha4/zz_generated.deepcopy.go b/exp/api/v1alpha4/zz_generated.deepcopy.go index 56fba3899c4..cbf301576c0 100644 --- a/exp/api/v1alpha4/zz_generated.deepcopy.go +++ b/exp/api/v1alpha4/zz_generated.deepcopy.go @@ -766,6 +766,11 @@ func (in *AzureManagedMachinePoolSpec) DeepCopyInto(out *AzureManagedMachinePool *out = make([]string, len(*in)) copy(*out, *in) } + if in.OSType != nil { + in, out := &in.OSType, &out.OSType + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedMachinePoolSpec. diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_types.go b/exp/api/v1beta1/azuremanagedcontrolplane_types.go index ce698de16b6..4f8afccfb33 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_types.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_types.go @@ -31,6 +31,36 @@ const ( PrivateDNSZoneModeNone string = "None" ) +// ManagedControlPlaneOutboundType enumerates the values for the managed control plane OutboundType. +type ManagedControlPlaneOutboundType string + +const ( + // ManagedControlPlaneOutboundTypeLoadBalancer ... + ManagedControlPlaneOutboundTypeLoadBalancer ManagedControlPlaneOutboundType = "loadBalancer" + // ManagedControlPlaneOutboundTypeManagedNATGateway ... + ManagedControlPlaneOutboundTypeManagedNATGateway ManagedControlPlaneOutboundType = "managedNATGateway" + // ManagedControlPlaneOutboundTypeUserAssignedNATGateway ... + ManagedControlPlaneOutboundTypeUserAssignedNATGateway ManagedControlPlaneOutboundType = "userAssignedNATGateway" + // ManagedControlPlaneOutboundTypeUserDefinedRouting ... + ManagedControlPlaneOutboundTypeUserDefinedRouting ManagedControlPlaneOutboundType = "userDefinedRouting" +) + +// UpgradeChannel enumerates the values for upgrade channel. +type UpgradeChannel string + +const ( + // UpgradeChannelNodeImage ... + UpgradeChannelNodeImage UpgradeChannel = "node-image" + // UpgradeChannelNone ... + UpgradeChannelNone UpgradeChannel = "none" + // UpgradeChannelPatch ... + UpgradeChannelPatch UpgradeChannel = "patch" + // UpgradeChannelRapid ... + UpgradeChannelRapid UpgradeChannel = "rapid" + // UpgradeChannelStable ... + UpgradeChannelStable UpgradeChannel = "stable" +) + // AzureManagedControlPlaneSpec defines the desired state of AzureManagedControlPlane. type AzureManagedControlPlaneSpec struct { // Version defines the desired Kubernetes version. @@ -76,6 +106,23 @@ type AzureManagedControlPlaneSpec struct { // +optional NetworkPolicy *string `json:"networkPolicy,omitempty"` + // Outbound configuration used by Nodes. + // +kubebuilder:validation:Enum=loadBalancer;managedNATGateway;userAssignedNATGateway;userDefinedRouting + // +optional + OutboundType *ManagedControlPlaneOutboundType `json:"outboundType,omitempty"` + + // DockerBridgeCidr - A CIDR notation IP range assigned to the Docker bridge network. It must not overlap with any Subnet IP ranges or the Kubernetes service address range. + // +optional + DockerBridgeCidr *string `json:"dockerBridgeCidr,omitempty"` + + // DNSPrefix - DNS prefix specified when creating the managed cluster. + // +optional + DNSPrefix *string `json:"dnsPrefix,omitempty"` + + // FqdnSubdomain - FQDN subdomain specified when creating private cluster with custom private dns zone. + // +optional + FqdnSubdomain *string `json:"fqdnSubdomain,omitempty"` + // SSHPublicKey is a string literal containing an ssh public key base64 encoded. SSHPublicKey string `json:"sshPublicKey"` @@ -112,6 +159,26 @@ type AzureManagedControlPlaneSpec struct { // APIServerAccessProfile is the access profile for AKS API server. // +optional APIServerAccessProfile *APIServerAccessProfile `json:"apiServerAccessProfile,omitempty"` + + // UserAssignedIdentities is a list of standalone Azure identities provided by the user to assign the cluster + // +optional + UserAssignedIdentities []infrav1.UserAssignedIdentity `json:"userAssignedIdentities,omitempty"` + + // AutoUpgradeProfile - Profile of auto upgrade configuration. + // +optional + AutoUpgradeProfile *ManagedClusterAutoUpgradeProfile `json:"autoUpgradeProfile,omitempty"` + + // DisableLocalAccounts - If set to true, getting static credential will be disabled for this cluster. Expected to only be used for AAD clusters. + // +optional + DisableLocalAccounts *bool `json:"disableLocalAccounts,omitempty"` +} + +// ManagedClusterAutoUpgradeProfile auto upgrade profile for a managed cluster. +type ManagedClusterAutoUpgradeProfile struct { + // UpgradeChannel - upgrade channel for auto upgrade. Possible values include: "node-image","none","patch","rapid","stable" + // +kubebuilder:validation:Enum=node-image;none;patch;rapid;stable + // +kubebuilder:validation:Required + UpgradeChannel UpgradeChannel `json:"upgradeChannel"` } // AADProfile - AAD integration managed by AKS. @@ -191,7 +258,6 @@ type APIServerAccessProfile struct { // +optional EnablePrivateCluster *bool `json:"enablePrivateCluster,omitempty"` // PrivateDNSZone - Private dns zone mode for private cluster. - // +kubebuilder:validation:Enum=System;None // +optional PrivateDNSZone *string `json:"privateDNSZone,omitempty"` // EnablePrivateClusterPublicFQDN - Whether to create additional public FQDN for private cluster or not. @@ -205,6 +271,8 @@ type ManagedControlPlaneVirtualNetwork struct { CIDRBlock string `json:"cidrBlock"` // +optional Subnet ManagedControlPlaneSubnet `json:"subnet,omitempty"` + // ResourceGroupName is the name of the Azure resource group for the VNet and Subnet. + ResourceGroupName string `json:"resourceGroupName,omitempty"` } // ManagedControlPlaneSubnet describes a subnet for an AKS cluster. @@ -215,6 +283,12 @@ type ManagedControlPlaneSubnet struct { // AzureManagedControlPlaneStatus defines the observed state of AzureManagedControlPlane. type AzureManagedControlPlaneStatus struct { + + // AutoUpgradeVersion is the Kubernetes version populated after autoupgrade based on the upgrade channel. + // +kubebuilder:validation:MinLength:=2 + // +optional + AutoUpgradeVersion string `json:"autoUpgradeVersion,omitempty"` + // Ready is true when the provider resource is ready. // +optional Ready bool `json:"ready,omitempty"` diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go b/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go index c0afa5a8ca2..6d397a37821 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go @@ -18,18 +18,21 @@ package v1beta1 import ( "context" - "errors" "fmt" "net" "reflect" "regexp" "strings" + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-azure/util/versions" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -88,6 +91,14 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client var allErrs field.ErrorList old := oldRaw.(*AzureManagedControlPlane) + if m.Name != old.Name { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Name"), + m.Name, + "field is immutable")) + } + if m.Spec.SubscriptionID != old.Spec.SubscriptionID { allErrs = append(allErrs, field.Invalid( @@ -150,6 +161,16 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client } } + if old.Spec.AutoUpgradeProfile != nil && m.Spec.AutoUpgradeProfile == nil { + // Prevent AutoUpgradeProfile to be set to nil. + // unsetting the field is not allowed + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "AutoUpgradeProfile"), + m.Spec.AutoUpgradeProfile, + "field is cannot be set to nil, to disable auto upgrades set the channel to none.")) + } + if old.Spec.NetworkPlugin != nil { // Prevent NetworkPlugin modification if it was already set to some value if m.Spec.NetworkPlugin == nil { @@ -232,6 +253,91 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client } } + if m.Spec.DisableLocalAccounts != nil && + m.Spec.AADProfile == nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "DisableLocalAccounts"), + m.Spec.DisableLocalAccounts, + "DisableLocalAccounts can be set only for AAD enabled clusters")) + } + + if old.Spec.DisableLocalAccounts != nil { + // Prevent DisableLocalAccounts modification if it was already set to some value + if m.Spec.DisableLocalAccounts == nil { + // unsetting the field is not allowed + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "DisableLocalAccounts"), + m.Spec.DisableLocalAccounts, + "field is immutable, unsetting is not allowed")) + } else if *m.Spec.DisableLocalAccounts != *old.Spec.DisableLocalAccounts { + // changing the field is not allowed + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "DisableLocalAccounts"), + *m.Spec.DisableLocalAccounts, + "field is immutable")) + } + } + + if old.Spec.OutboundType != nil { + // Prevent OutboundType modification if it was already set to some value + if m.Spec.OutboundType == nil { + // unsetting the field is not allowed + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "OutboundType"), + m.Spec.OutboundType, + "field is immutable, unsetting is not allowed")) + } else if *m.Spec.OutboundType != *old.Spec.OutboundType { + // changing the field is not allowed + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "OutboundType"), + *m.Spec.OutboundType, + "field is immutable")) + } + } + + if ptr.Deref[string](m.Spec.DNSPrefix, "") != ptr.Deref[string](old.Spec.DNSPrefix, "") { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec.DNSPrefix"), + m.Spec.DNSPrefix, + "field is immutable")) + } + + if ptr.Deref[string](m.Spec.DockerBridgeCidr, "") != ptr.Deref[string](old.Spec.DockerBridgeCidr, "") { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec.DockerBridgeCidr"), + m.Spec.DockerBridgeCidr, + "field is immutable")) + } + + if ptr.Deref[string](m.Spec.FqdnSubdomain, "") != ptr.Deref[string](old.Spec.FqdnSubdomain, "") { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec.FqdnSubdomain"), + m.Spec.FqdnSubdomain, + "field is immutable")) + } + + if hv, _ := versions.GetHigherK8sVersion(m.Spec.Version, old.Spec.Version); hv != m.Spec.Version { + allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "Version"), + m.Spec.Version, "fields version cannot be downgraded"), + ) + } + + if old.Status.AutoUpgradeVersion != "" && m.Spec.Version != old.Spec.Version { + if hv, _ := versions.GetHigherK8sVersion(m.Spec.Version, old.Status.AutoUpgradeVersion); hv != m.Spec.Version { + allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "Version"), + m.Spec.Version, "fields version cannot be downgraded"), + ) + } + } + if errs := m.validateAPIServerAccessProfileUpdate(old); len(errs) > 0 { allErrs = append(allErrs, errs...) } @@ -251,12 +357,15 @@ func (m *AzureManagedControlPlane) ValidateDelete(_ client.Client) error { // Validate the Azure Machine Pool and return an aggregate error. func (m *AzureManagedControlPlane) Validate(cli client.Client) error { validators := []func(client client.Client) error{ + m.validateName, m.validateVersion, m.validateDNSServiceIP, m.validateSSHKey, m.validateLoadBalancerProfile, m.validateAPIServerAccessProfile, - m.validateManagedClusterNetwork, + m.validateDNSPrefix, + m.validateDisableLocalAccounts, + //m.validateManagedClusterNetwork, } var errs []error @@ -269,6 +378,31 @@ func (m *AzureManagedControlPlane) Validate(cli client.Client) error { return kerrors.NewAggregate(errs) } +func (m *AzureManagedControlPlane) validateDisableLocalAccounts(_ client.Client) error { + + if m.Spec.DisableLocalAccounts != nil && m.Spec.AADProfile == nil { + return errors.New("DisableLocalAccounts should be set only for AAD enabled clusters") + } + + return nil +} + +func (m *AzureManagedControlPlane) validateDNSPrefix(_ client.Client) error { + if m.Spec.DNSPrefix == nil { + return nil + } + + // Regex pattern for DNS prefix validation + // 1. Between 1 and 54 characters long: {1,54} + // 2. Alphanumerics and hyphens: [a-zA-Z0-9-] + // 3. Start and end with alphanumeric: ^[a-zA-Z0-9].*[a-zA-Z0-9]$ + regex := regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9-]{0,52}[a-zA-Z0-9]$`) + if regex.MatchString(ptr.Deref[string](m.Spec.DNSPrefix, "")) { + return nil + } + return errors.New("DNSPrefix is invalid") +} + // validateDNSServiceIP validates the DNSServiceIP. func (m *AzureManagedControlPlane) validateDNSServiceIP(_ client.Client) error { if m.Spec.DNSServiceIP != nil { @@ -280,12 +414,23 @@ func (m *AzureManagedControlPlane) validateDNSServiceIP(_ client.Client) error { return nil } +// validateName validates the AzureManagedControlPlane name +func (m *AzureManagedControlPlane) validateName(_ client.Client) error { + lName := strings.ToLower(m.Name) + if strings.Contains(lName, "microsoft") || + strings.Contains(lName, "windows") { + return field.Invalid(field.NewPath("Name"), m.Name, + "cluster name is invalid because 'MICROSOFT' and 'WINDOWS' can't be used as either a whole word or a substring in the name") + } + + return nil +} + // validateVersion validates the Kubernetes version. func (m *AzureManagedControlPlane) validateVersion(_ client.Client) error { if !kubeSemver.MatchString(m.Spec.Version) { return errors.New("must be a valid semantic version") } - return nil } @@ -440,14 +585,12 @@ func (m *AzureManagedControlPlane) validateAPIServerAccessProfileUpdate(old *Azu if m.Spec.APIServerAccessProfile != nil { newAPIServerAccessProfileNormalized = &APIServerAccessProfile{ EnablePrivateCluster: m.Spec.APIServerAccessProfile.EnablePrivateCluster, - PrivateDNSZone: m.Spec.APIServerAccessProfile.PrivateDNSZone, EnablePrivateClusterPublicFQDN: m.Spec.APIServerAccessProfile.EnablePrivateClusterPublicFQDN, } } if old.Spec.APIServerAccessProfile != nil { oldAPIServerAccessProfileNormalized = &APIServerAccessProfile{ EnablePrivateCluster: old.Spec.APIServerAccessProfile.EnablePrivateCluster, - PrivateDNSZone: old.Spec.APIServerAccessProfile.PrivateDNSZone, EnablePrivateClusterPublicFQDN: old.Spec.APIServerAccessProfile.EnablePrivateClusterPublicFQDN, } } @@ -455,7 +598,46 @@ func (m *AzureManagedControlPlane) validateAPIServerAccessProfileUpdate(old *Azu if !reflect.DeepEqual(newAPIServerAccessProfileNormalized, oldAPIServerAccessProfileNormalized) { allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "APIServerAccessProfile"), - m.Spec.APIServerAccessProfile, "fields (except for AuthorizedIPRanges) are immutable"), + m.Spec.APIServerAccessProfile, "fields EnablePrivateCluster and EnablePrivateClusterPublicFQDN are immutable"), + ) + } + + if errs := m.validateAPIServerAccessProfileDNSZoneUpdate(old); len(errs) > 0 { + allErrs = append(allErrs, errs...) + } + + return allErrs +} + +func (m *AzureManagedControlPlane) validateAPIServerAccessProfileDNSZoneUpdate(old *AzureManagedControlPlane) field.ErrorList { + var allErrs field.ErrorList + + // You can only update from byo or system to none. No other combination of update values is supported. + if m.Spec.APIServerAccessProfile != nil && old.Spec.APIServerAccessProfile != nil && + ptr.Deref(m.Spec.APIServerAccessProfile.EnablePrivateCluster, false) && + ptr.Deref(old.Spec.APIServerAccessProfile.EnablePrivateCluster, false) && + m.Spec.APIServerAccessProfile.PrivateDNSZone != old.Spec.APIServerAccessProfile.PrivateDNSZone && + ptr.Deref[string](m.Spec.APIServerAccessProfile.PrivateDNSZone, "") == "None" { + return nil + } + + newAPIServerAccessProfileNormalized := &APIServerAccessProfile{} + oldAPIServerAccessProfileNormalized := &APIServerAccessProfile{} + if m.Spec.APIServerAccessProfile != nil { + newAPIServerAccessProfileNormalized = &APIServerAccessProfile{ + PrivateDNSZone: m.Spec.APIServerAccessProfile.PrivateDNSZone, + } + } + if old.Spec.APIServerAccessProfile != nil { + oldAPIServerAccessProfileNormalized = &APIServerAccessProfile{ + PrivateDNSZone: old.Spec.APIServerAccessProfile.PrivateDNSZone, + } + } + + if !reflect.DeepEqual(newAPIServerAccessProfileNormalized, oldAPIServerAccessProfileNormalized) { + allErrs = append(allErrs, + field.Invalid(field.NewPath("Spec", "APIServerAccessProfile"), + m.Spec.APIServerAccessProfile, "invalid update operation performed on PrivateDNSZone. You can only update from byo or system to none. No other combination of update values is supported."), ) } diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go b/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go index 2e49570b3b5..c5481550ddf 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go @@ -22,7 +22,7 @@ import ( "github.com/Azure/go-autorest/autorest/to" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ) func TestDefaultingWebhook(t *testing.T) { @@ -82,11 +82,81 @@ func TestValidatingWebhook(t *testing.T) { amcp AzureManagedControlPlane expectErr bool }{ + { + name: "Testing inValid DNSPrefix for starting with invalid characters", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSPrefix: ptr.To("-thisi$"), + Version: "v1.17.8", + }, + }, + expectErr: true, + }, + { + name: "Testing inValid DNSPrefix with more then 54 characters", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSPrefix: ptr.To("thisisaverylong$^clusternameconsistingofmorethan54characterswhichshouldbeinvalid"), + Version: "v1.17.8", + }, + }, + expectErr: true, + }, + { + name: "Testing inValid DNSPrefix with underscore", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSPrefix: ptr.To("no_underscore"), + Version: "v1.17.8", + }, + }, + expectErr: true, + }, + { + name: "Testing inValid DNSPrefix with special characters", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSPrefix: ptr.To("no-dollar$@%"), + Version: "v1.17.8", + }, + }, + expectErr: true, + }, + { + name: "Testing Valid DNSPrefix with hyphen characters", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSPrefix: ptr.To("hyphen-allowed"), + Version: "v1.17.8", + }, + }, + expectErr: false, + }, + { + name: "Testing Valid DNSPrefix with hyphen characters", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSPrefix: ptr.To("palette-test07"), + Version: "v1.17.8", + }, + }, + expectErr: false, + }, + { + name: "Testing valid DNSPrefix ", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSPrefix: ptr.To("thisisavlerylongclu7l0sternam3leconsistingofmorethan54"), + Version: "v1.17.8", + }, + }, + expectErr: false, + }, { name: "Testing valid DNSServiceIP", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: pointer.StringPtr("192.168.0.0"), + DNSServiceIP: ptr.To("192.168.0.0"), Version: "v1.17.8", }, }, @@ -96,7 +166,7 @@ func TestValidatingWebhook(t *testing.T) { name: "Testing invalid DNSServiceIP", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: pointer.StringPtr("192.168.0.0.3"), + DNSServiceIP: ptr.To("192.168.0.0.3"), Version: "v1.17.8", }, }, @@ -106,7 +176,7 @@ func TestValidatingWebhook(t *testing.T) { name: "Invalid Version", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: pointer.StringPtr("192.168.0.0"), + DNSServiceIP: ptr.To("192.168.0.0"), Version: "honk", }, }, @@ -116,7 +186,7 @@ func TestValidatingWebhook(t *testing.T) { name: "not following the Kubernetes Version pattern", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: pointer.StringPtr("192.168.0.0"), + DNSServiceIP: ptr.To("192.168.0.0"), Version: "1.19.0", }, }, @@ -126,7 +196,7 @@ func TestValidatingWebhook(t *testing.T) { name: "Version not set", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: pointer.StringPtr("192.168.0.0"), + DNSServiceIP: ptr.To("192.168.0.0"), Version: "", }, }, @@ -136,7 +206,7 @@ func TestValidatingWebhook(t *testing.T) { name: "Valid Version", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: pointer.StringPtr("192.168.0.0"), + DNSServiceIP: ptr.To("192.168.0.0"), Version: "v1.17.8", }, }, @@ -234,6 +304,30 @@ func TestValidatingWebhook(t *testing.T) { }, expectErr: true, }, + { + name: "DisableLocalAccounts cannot be set for non AAD clusters", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.21.2", + DisableLocalAccounts: ptr.To(true), + }, + }, + expectErr: true, + }, + { + name: "DisableLocalAccounts can be set for AAD clusters", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.21.2", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: ptr.To(true), + }, + }, + expectErr: false, + }, } for _, tt := range tests { @@ -300,6 +394,36 @@ func TestAzureManagedControlPlane_ValidateCreate(t *testing.T) { wantErr: true, errorLen: 1, }, + { + name: "invalid name with microsoft", + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "microsoft-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + SSHPublicKey: generateSSHPublicKey(true), + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.23.5", + }, + }, + wantErr: true, + errorLen: 1, + }, + { + name: "invalid name with windows", + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "a-windows-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + SSHPublicKey: generateSSHPublicKey(true), + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.23.5", + }, + }, + wantErr: true, + errorLen: 1, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -347,6 +471,117 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { amcp: createAzureManagedControlPlane("192.168.0.0", "1.999.9", generateSSHPublicKey(true)), wantErr: true, }, + { + name: "AzureManagedControlPlane invalid version downgrade change", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.17.0", + }, + }, + wantErr: true, + }, + { + name: "AzureManagedControlPlane invalid version downgrade change", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + Status: AzureManagedControlPlaneStatus{ + AutoUpgradeVersion: "v1.18.3", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.1", + }, + }, + wantErr: true, + }, + { + name: "AzureManagedControlPlane invalid version downgrade change", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + Status: AzureManagedControlPlaneStatus{ + AutoUpgradeVersion: "1.19.3", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.6", + }, + }, + wantErr: true, + }, + { + name: "AzureManagedControlPlane no version change", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + Status: AzureManagedControlPlaneStatus{ + AutoUpgradeVersion: "1.19.3", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + }, + wantErr: false, + }, + { + name: "AzureManagedControlPlane valid version upgrade change", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + Status: AzureManagedControlPlaneStatus{ + AutoUpgradeVersion: "1.19.3", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.19.5", + }, + }, + wantErr: false, + }, + { + name: "AzureManagedControlPlane valid version change", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + Status: AzureManagedControlPlaneStatus{ + AutoUpgradeVersion: "1.19.3", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.19.3", + }, + }, + wantErr: false, + }, { name: "AzureManagedControlPlane SubscriptionID is immutable", oldAMCP: &AzureManagedControlPlane{ @@ -703,6 +938,202 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { }, wantErr: true, }, + { + name: "AzureManagedControlPlane DockerBridgeCidr is immutable", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DockerBridgeCidr: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + }, + wantErr: true, + }, + { + name: "AzureManagedControlPlane DockerBridgeCidr is immutable", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DockerBridgeCidr: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DockerBridgeCidr: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + }, + wantErr: false, + }, + { + name: "AzureManagedControlPlane DockerBridgeCidr is immutable", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DockerBridgeCidr: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + }, + wantErr: true, + }, + { + name: "AzureManagedControlPlane DNSPrefix is immutable", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSPrefix: ptr.To("capz-aks-1"), + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSPrefix: ptr.To("capz-aks"), + Version: "v1.18.0", + }, + }, + wantErr: true, + }, + { + name: "AzureManagedControlPlane DNSPrefix is immutable", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSPrefix: ptr.To("capz-aks"), + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSPrefix: ptr.To("capz-aks"), + Version: "v1.18.0", + }, + }, + wantErr: false, + }, + { + name: "AzureManagedControlPlane FqdnSubdomain is immutable", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + FqdnSubdomain: ptr.To("capzaks.api"), + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + FqdnSubdomain: ptr.To("capzaks.com"), + }, + }, + wantErr: true, + }, + { + name: "AzureManagedControlPlane FqdnSubdomain is immutable", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + FqdnSubdomain: ptr.To("capzaks.api"), + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + FqdnSubdomain: ptr.To("capzaks.api"), + }, + }, + wantErr: false, + }, + { + name: "Valid AzureManagedControlPlane PrivateDNSZone update", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + APIServerAccessProfile: &APIServerAccessProfile{ + EnablePrivateCluster: ptr.To(true), + }, + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + APIServerAccessProfile: &APIServerAccessProfile{ + EnablePrivateCluster: ptr.To(true), + PrivateDNSZone: ptr.To("None"), + }, + Version: "v1.18.0", + }, + }, + wantErr: false, + }, + { + name: "Valid AzureManagedControlPlane PrivateDNSZone update", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + APIServerAccessProfile: &APIServerAccessProfile{ + EnablePrivateCluster: ptr.To(true), + PrivateDNSZone: ptr.To("example-resource-id"), + }, + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + APIServerAccessProfile: &APIServerAccessProfile{ + EnablePrivateCluster: ptr.To(true), + PrivateDNSZone: ptr.To("None"), + }, + }, + }, + wantErr: false, + }, + { + name: "Invalid AzureManagedControlPlane PrivateDNSZone update", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + APIServerAccessProfile: &APIServerAccessProfile{ + EnablePrivateCluster: ptr.To(true), + PrivateDNSZone: ptr.To("example-resource-id"), + }, + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + APIServerAccessProfile: &APIServerAccessProfile{ + EnablePrivateCluster: ptr.To(true), + }, + }, + }, + wantErr: true, + }, + { + name: "Invalid AzureManagedControlPlane PrivateDNSZone update", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + APIServerAccessProfile: &APIServerAccessProfile{ + EnablePrivateCluster: ptr.To(true), + PrivateDNSZone: ptr.To("example-resource-id"), + }, + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + APIServerAccessProfile: &APIServerAccessProfile{ + EnablePrivateCluster: ptr.To(true), + PrivateDNSZone: ptr.To("example-resource-id-1"), + }, + }, + }, + wantErr: true, + }, { name: "AzureManagedControlPlane AuthorizedIPRanges is mutable", oldAMCP: &AzureManagedControlPlane{ @@ -722,6 +1153,235 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { }, wantErr: false, }, + { + name: "AzureManagedControlPlane Name is mutable", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "new-test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + APIServerAccessProfile: &APIServerAccessProfile{ + AuthorizedIPRanges: []string{"192.168.0.1/32"}, + }, + }, + }, + wantErr: true, + }, + { + name: "DisableLocalAccounts can be set only for AAD enabled clusters", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + DisableLocalAccounts: ptr.To(true), + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + }, + }, + wantErr: false, + }, + { + name: "DisableLocalAccounts cannot be set only for non AAD clusters", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + DisableLocalAccounts: ptr.To(true), + }, + }, + wantErr: true, + }, + { + name: "DisableLocalAccounts cannot be disabled AAD clusters", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: ptr.To(true), + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + }, + }, + wantErr: true, + }, + { + name: "DisableLocalAccounts cannot change the value AAD clusters", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: ptr.To(true), + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: ptr.To(false), + }, + }, + wantErr: true, + }, + { + name: "Auto upgrade profile cannot be disabled", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AutoUpgradeProfile: &ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: UpgradeChannelNone, + }, + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + }, + }, + wantErr: true, + }, + { + name: "Auto upgrade profile channel can be updated", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AutoUpgradeProfile: &ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: UpgradeChannelPatch, + }, + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AutoUpgradeProfile: &ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: UpgradeChannelStable, + }, + }, + }, + wantErr: false, + }, + { + name: "Auto upgrade profile channel can remain same", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AutoUpgradeProfile: &ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: UpgradeChannelPatch, + }, + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AutoUpgradeProfile: &ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: UpgradeChannelPatch, + }, + }, + }, + wantErr: false, + }, + { + name: "OutboundType update", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + OutboundType: (*ManagedControlPlaneOutboundType)(to.StringPtr(string(ManagedControlPlaneOutboundTypeUserDefinedRouting))), + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + OutboundType: (*ManagedControlPlaneOutboundType)(to.StringPtr(string(ManagedControlPlaneOutboundTypeLoadBalancer))), + }, + }, + wantErr: true, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { diff --git a/exp/api/v1beta1/azuremanagedmachinepool_types.go b/exp/api/v1beta1/azuremanagedmachinepool_types.go index cb07ea973ca..46efb3dfda2 100644 --- a/exp/api/v1beta1/azuremanagedmachinepool_types.go +++ b/exp/api/v1beta1/azuremanagedmachinepool_types.go @@ -76,6 +76,11 @@ type AzureManagedMachinePoolSpec struct { // +optional Scaling *ManagedMachinePoolScaling `json:"scaling,omitempty"` + // OSType specifies the virtual machine operating system. Default to Linux. Possible values include: 'Linux', 'Windows' + // +kubebuilder:validation:Enum=Linux;Windows + // +optional + OSType *string `json:"osType,omitempty"` + // MaxPods specifies the kubelet --max-pods configuration for the node pool. // +optional MaxPods *int32 `json:"maxPods,omitempty"` @@ -89,6 +94,10 @@ type AzureManagedMachinePoolSpec struct { // EnableUltraSSD enables the storage type UltraSSD_LRS for the agent pool. // +optional EnableUltraSSD *bool `json:"enableUltraSSD,omitempty"` + + // SubnetName selects the Subnet where the MachinePool will be placed + // +optional + SubnetName string `json:"subnetName,omitempty"` } // ManagedMachinePoolScaling specifies scaling options. diff --git a/exp/api/v1beta1/azuremanagedmachinepool_webhook.go b/exp/api/v1beta1/azuremanagedmachinepool_webhook.go index 17259f2ff8d..f916cf40bf2 100644 --- a/exp/api/v1beta1/azuremanagedmachinepool_webhook.go +++ b/exp/api/v1beta1/azuremanagedmachinepool_webhook.go @@ -17,19 +17,16 @@ limitations under the License. package v1beta1 import ( - "context" "fmt" "reflect" "github.com/Azure/go-autorest/autorest/to" - "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/validation/field" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/util/maps" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -53,6 +50,8 @@ func (m *AzureManagedMachinePool) Default(client client.Client) { func (m *AzureManagedMachinePool) ValidateCreate(client client.Client) error { validators := []func() error{ m.validateMaxPods, + m.validateOSType, + m.validateName, } var errs []error @@ -78,6 +77,25 @@ func (m *AzureManagedMachinePool) ValidateUpdate(oldRaw runtime.Object, client c "field is immutable")) } + if old.Spec.OSType != nil { + // Prevent OSType modification if it was already set to some value + if m.Spec.OSType == nil { + // unsetting the field is not allowed + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "OSType"), + m.Spec.OSType, + "field is immutable, unsetting is not allowed")) + } else if *m.Spec.OSType != *old.Spec.OSType { + // changing the field is not allowed + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "OSType"), + *m.Spec.OSType, + "field is immutable")) + } + } + if old.Spec.OSDiskSizeGB != nil { // Prevent OSDiskSizeGB modification if it was already set to some value if m.Spec.OSDiskSizeGB == nil { @@ -124,16 +142,6 @@ func (m *AzureManagedMachinePool) ValidateUpdate(oldRaw runtime.Object, client c "field is immutable")) } - if m.Spec.Mode != string(NodePoolModeSystem) && old.Spec.Mode == string(NodePoolModeSystem) { - // validate for last system node pool - if err := m.validateLastSystemNodePool(client); err != nil { - allErrs = append(allErrs, field.Invalid( - field.NewPath("Spec", "Mode"), - m.Spec.Mode, - "Last system node pool cannot be mutated to user node pool")) - } - } - if old.Spec.MaxPods != nil { // Prevent MaxPods modification if it was already set to some value if m.Spec.MaxPods == nil { @@ -207,62 +215,41 @@ func (m *AzureManagedMachinePool) ValidateUpdate(oldRaw runtime.Object, client c // ValidateDelete implements webhook.Validator so a webhook will be registered for the type. func (m *AzureManagedMachinePool) ValidateDelete(client client.Client) error { - if m.Spec.Mode != string(NodePoolModeSystem) { - return nil - } - - return errors.Wrapf(m.validateLastSystemNodePool(client), "if the delete is triggered via owner MachinePool please refer to trouble shooting section in https://capz.sigs.k8s.io/topics/managedcluster.html") + return nil } -// validateLastSystemNodePool is used to check if the existing system node pool is the last system node pool. -// If it is a last system node pool it cannot be deleted or mutated to user node pool as AKS expects min 1 system node pool. -func (m *AzureManagedMachinePool) validateLastSystemNodePool(cli client.Client) error { - ctx := context.Background() - - // Fetch the Cluster. - clusterName, ok := m.Labels[clusterv1.ClusterLabelName] - if !ok { - return nil - } - - ownerCluster := &clusterv1.Cluster{} - key := client.ObjectKey{ - Namespace: m.Namespace, - Name: clusterName, - } - - if err := cli.Get(ctx, key, ownerCluster); err != nil { - return err - } - - if !ownerCluster.DeletionTimestamp.IsZero() { - return nil +func (m *AzureManagedMachinePool) validateMaxPods() error { + if m.Spec.MaxPods != nil { + if to.Int32(m.Spec.MaxPods) < 10 || to.Int32(m.Spec.MaxPods) > 250 { + return field.Invalid( + field.NewPath("Spec", "MaxPods"), + m.Spec.MaxPods, + "MaxPods must be between 10 and 250") + } } - opt1 := client.InNamespace(m.Namespace) - opt2 := client.MatchingLabels(map[string]string{ - clusterv1.ClusterLabelName: clusterName, - LabelAgentPoolMode: string(NodePoolModeSystem), - }) + return nil +} - ammpList := &AzureManagedMachinePoolList{} - if err := cli.List(ctx, ammpList, opt1, opt2); err != nil { - return err +func (r *AzureManagedMachinePool) validateOSType() error { + if r.Spec.Mode == string(NodePoolModeSystem) { + if r.Spec.OSType != nil && *r.Spec.OSType != azure.LinuxOS { + return field.Forbidden( + field.NewPath("Spec", "OSType"), + "System node pooll must have OSType 'Linux'") + } } - if len(ammpList.Items) <= 1 { - return errors.New("AKS Cluster must have at least one system pool") - } return nil } -func (m *AzureManagedMachinePool) validateMaxPods() error { - if m.Spec.MaxPods != nil { - if to.Int32(m.Spec.MaxPods) < 10 || to.Int32(m.Spec.MaxPods) > 250 { +func (r *AzureManagedMachinePool) validateName() error { + if r.Spec.OSType != nil && *r.Spec.OSType == azure.WindowsOS { + if len(r.Name) > 6 { return field.Invalid( - field.NewPath("Spec", "MaxPods"), - m.Spec.MaxPods, - "MaxPods must be between 10 and 250") + field.NewPath("Name"), + r.Name, + "Windows agent pool name can not be longer than 6 characters.") } } diff --git a/exp/api/v1beta1/azuremanagedmachinepool_webhook_test.go b/exp/api/v1beta1/azuremanagedmachinepool_webhook_test.go index 006f03926ed..5c17db43360 100644 --- a/exp/api/v1beta1/azuremanagedmachinepool_webhook_test.go +++ b/exp/api/v1beta1/azuremanagedmachinepool_webhook_test.go @@ -19,6 +19,8 @@ package v1beta1 import ( "testing" + "sigs.k8s.io/cluster-api-provider-azure/azure" + "github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2021-05-01/containerservice" "github.com/Azure/go-autorest/autorest/to" . "github.com/onsi/gomega" @@ -441,6 +443,26 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { }, wantErr: true, }, + { + name: "Cannot change OSType of the agentpool", + new: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + OSType: to.StringPtr("Linux"), + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + }, + }, + old: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + OSType: to.StringPtr("Windows"), + Mode: "System", + SKU: "StandardD2S_V4", + OSDiskSizeGB: to.Int32Ptr(512), + }, + }, + wantErr: true, + }, } var client client.Client for _, tc := range tests { @@ -511,6 +533,54 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { wantErr: true, errorLen: 1, }, + { + name: "ostype Windows with System mode not allowed", + ammp: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + OSType: to.StringPtr(azure.WindowsOS), + }, + }, + wantErr: true, + errorLen: 1, + }, + { + name: "ostype windows with User mode", + ammp: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "User", + OSType: to.StringPtr(azure.WindowsOS), + }, + }, + wantErr: false, + }, + { + name: "Windows clusters with 6char or less name", + ammp: &AzureManagedMachinePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pool0", + }, + Spec: AzureManagedMachinePoolSpec{ + Mode: "User", + OSType: to.StringPtr(azure.WindowsOS), + }, + }, + wantErr: false, + }, + { + name: "Windows clusters with more than 6char names are not allowed", + ammp: &AzureManagedMachinePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pool0-name-too-long", + }, + Spec: AzureManagedMachinePoolSpec{ + Mode: "User", + OSType: to.StringPtr(azure.WindowsOS), + }, + }, + wantErr: true, + errorLen: 1, + }, } var client client.Client for _, tc := range tests { diff --git a/exp/api/v1beta1/zz_generated.deepcopy.go b/exp/api/v1beta1/zz_generated.deepcopy.go index 9e35f3cf62f..84656a0a7f2 100644 --- a/exp/api/v1beta1/zz_generated.deepcopy.go +++ b/exp/api/v1beta1/zz_generated.deepcopy.go @@ -644,6 +644,26 @@ func (in *AzureManagedControlPlaneSpec) DeepCopyInto(out *AzureManagedControlPla *out = new(string) **out = **in } + if in.OutboundType != nil { + in, out := &in.OutboundType, &out.OutboundType + *out = new(ManagedControlPlaneOutboundType) + **out = **in + } + if in.DockerBridgeCidr != nil { + in, out := &in.DockerBridgeCidr, &out.DockerBridgeCidr + *out = new(string) + **out = **in + } + if in.DNSPrefix != nil { + in, out := &in.DNSPrefix, &out.DNSPrefix + *out = new(string) + **out = **in + } + if in.FqdnSubdomain != nil { + in, out := &in.FqdnSubdomain, &out.FqdnSubdomain + *out = new(string) + **out = **in + } if in.DNSServiceIP != nil { in, out := &in.DNSServiceIP, &out.DNSServiceIP *out = new(string) @@ -686,6 +706,21 @@ func (in *AzureManagedControlPlaneSpec) DeepCopyInto(out *AzureManagedControlPla *out = new(APIServerAccessProfile) (*in).DeepCopyInto(*out) } + if in.UserAssignedIdentities != nil { + in, out := &in.UserAssignedIdentities, &out.UserAssignedIdentities + *out = make([]apiv1beta1.UserAssignedIdentity, len(*in)) + copy(*out, *in) + } + if in.AutoUpgradeProfile != nil { + in, out := &in.AutoUpgradeProfile, &out.AutoUpgradeProfile + *out = new(ManagedClusterAutoUpgradeProfile) + **out = **in + } + if in.DisableLocalAccounts != nil { + in, out := &in.DisableLocalAccounts, &out.DisableLocalAccounts + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneSpec. @@ -824,6 +859,11 @@ func (in *AzureManagedMachinePoolSpec) DeepCopyInto(out *AzureManagedMachinePool *out = new(ManagedMachinePoolScaling) (*in).DeepCopyInto(*out) } + if in.OSType != nil { + in, out := &in.OSType, &out.OSType + *out = new(string) + **out = **in + } if in.MaxPods != nil { in, out := &in.MaxPods, &out.MaxPods *out = new(int32) @@ -953,6 +993,21 @@ func (in *MachineRollingUpdateDeployment) DeepCopy() *MachineRollingUpdateDeploy return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ManagedClusterAutoUpgradeProfile) DeepCopyInto(out *ManagedClusterAutoUpgradeProfile) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ManagedClusterAutoUpgradeProfile. +func (in *ManagedClusterAutoUpgradeProfile) DeepCopy() *ManagedClusterAutoUpgradeProfile { + if in == nil { + return nil + } + out := new(ManagedClusterAutoUpgradeProfile) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ManagedControlPlaneSubnet) DeepCopyInto(out *ManagedControlPlaneSubnet) { *out = *in diff --git a/exp/controllers/azuremanagedcontrolplane_controller.go b/exp/controllers/azuremanagedcontrolplane_controller.go index 86c0819b9d8..76b270cddad 100644 --- a/exp/controllers/azuremanagedcontrolplane_controller.go +++ b/exp/controllers/azuremanagedcontrolplane_controller.go @@ -260,7 +260,7 @@ func (amcpr *AzureManagedControlPlaneReconciler) reconcileNormal(ctx context.Con // No errors, so mark us ready so the Cluster API Cluster Controller can pull it scope.ControlPlane.Status.Ready = true scope.ControlPlane.Status.Initialized = true - amcpr.Recorder.Event(scope.ControlPlane, corev1.EventTypeNormal, "AzureManagedControlPlane available", "successfully reconciled") + //amcpr.Recorder.Event(scope.ControlPlane, corev1.EventTypeNormal, "AzureManagedControlPlane available", "successfully reconciled") return reconcile.Result{}, nil } diff --git a/exp/controllers/azuremanagedcontrolplane_reconciler.go b/exp/controllers/azuremanagedcontrolplane_reconciler.go index afc0b301d38..26fbba56688 100644 --- a/exp/controllers/azuremanagedcontrolplane_reconciler.go +++ b/exp/controllers/azuremanagedcontrolplane_reconciler.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "fmt" "github.com/pkg/errors" "sigs.k8s.io/cluster-api-provider-azure/azure" @@ -92,20 +93,25 @@ func (r *azureManagedControlPlaneService) reconcileKubeconfig(ctx context.Contex ctx, _, done := tele.StartSpanWithLogger(ctx, "controllers.azureManagedControlPlaneService.reconcileKubeconfig") defer done() - kubeConfigData := r.scope.GetKubeConfigData() - if kubeConfigData == nil { - return nil - } - kubeConfigSecret := r.scope.MakeEmptyKubeConfigSecret() + kubeConfigs := [][]byte{r.scope.GetAdminKubeConfigData(), r.scope.GetUserKubeConfigData()} - // Always update credentials in case of rotation - if _, err := controllerutil.CreateOrUpdate(ctx, r.kubeclient, &kubeConfigSecret, func() error { - kubeConfigSecret.Data = map[string][]byte{ - secret.KubeconfigDataName: kubeConfigData, + for i, kubeConfigData := range kubeConfigs { + if len(kubeConfigData) == 0 { + continue + } + kubeConfigSecret := r.scope.MakeEmptyKubeConfigSecret() + if i == 1 { + // 2nd kubeconfig is the user kubeconfig + kubeConfigSecret.Name = fmt.Sprintf("%s-user", kubeConfigSecret.Name) + } + if _, err := controllerutil.CreateOrUpdate(ctx, r.kubeclient, &kubeConfigSecret, func() error { + kubeConfigSecret.Data = map[string][]byte{ + secret.KubeconfigDataName: kubeConfigData, + } + return nil + }); err != nil { + return errors.Wrap(err, "failed to kubeconfig secret for cluster") } - return nil - }); err != nil { - return errors.Wrap(err, "failed to kubeconfig secret for cluster") } return nil diff --git a/exp/controllers/azuremanagedmachinepool_controller.go b/exp/controllers/azuremanagedmachinepool_controller.go index 9b5f2ad51a0..d309352bc27 100644 --- a/exp/controllers/azuremanagedmachinepool_controller.go +++ b/exp/controllers/azuremanagedmachinepool_controller.go @@ -21,7 +21,6 @@ import ( "time" "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/tools/record" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" @@ -225,6 +224,9 @@ func (ammpr *AzureManagedMachinePoolReconciler) Reconcile(ctx context.Context, r if err := mcpScope.PatchObject(ctx); err != nil && reterr == nil { reterr = err } + if err := mcpScope.PatchCAPIMachinePoolObject(ctx); err != nil && reterr == nil { + reterr = err + } }() // Handle deleted clusters @@ -277,7 +279,7 @@ func (ammpr *AzureManagedMachinePoolReconciler) reconcileNormal(ctx context.Cont // No errors, so mark us ready so the Cluster API Cluster Controller can pull it scope.InfraMachinePool.Status.Ready = true - ammpr.Recorder.Eventf(scope.InfraMachinePool, corev1.EventTypeNormal, "AzureManagedMachinePool available", "agent pool successfully reconciled") + //ammpr.Recorder.Eventf(scope.InfraMachinePool, corev1.EventTypeNormal, "AzureManagedMachinePool available", "agent pool successfully reconciled") return reconcile.Result{}, nil } diff --git a/exp/controllers/azuremanagedmachinepool_reconciler.go b/exp/controllers/azuremanagedmachinepool_reconciler.go index 927a7ec4744..67fd56bbc3e 100644 --- a/exp/controllers/azuremanagedmachinepool_reconciler.go +++ b/exp/controllers/azuremanagedmachinepool_reconciler.go @@ -19,7 +19,6 @@ package controllers import ( "context" "fmt" - "strings" "time" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" @@ -28,6 +27,7 @@ import ( "sigs.k8s.io/cluster-api-provider-azure/azure/scope" "sigs.k8s.io/cluster-api-provider-azure/azure/services/agentpools" "sigs.k8s.io/cluster-api-provider-azure/azure/services/scalesets" + azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" "sigs.k8s.io/cluster-api-provider-azure/util/tele" ) @@ -133,7 +133,12 @@ func (s *azureManagedMachinePoolService) Reconcile(ctx context.Context) error { var providerIDs = make([]string, len(instances)) for i := 0; i < len(instances); i++ { - providerIDs[i] = strings.ToLower(azure.ProviderIDPrefix + *instances[i].ID) + // Transform the VMSS instance resource representation to conform to the cloud-provider-azure representation + providerID, err := azureutil.ConvertResourceGroupNameToLower(azure.ProviderIDPrefix + *instances[i].ID) + if err != nil { + return errors.Wrapf(err, "failed to parse instance ID %s", *instances[i].ID) + } + providerIDs[i] = providerID } s.scope.SetAgentPoolProviderIDList(providerIDs) diff --git a/go.mod b/go.mod index 3596b2c6797..86dda86eed5 100644 --- a/go.mod +++ b/go.mod @@ -1,28 +1,30 @@ module sigs.k8s.io/cluster-api-provider-azure -go 1.17 +go 1.19 require ( - github.com/Azure/aad-pod-identity v1.8.6 - github.com/Azure/azure-sdk-for-go v63.4.0+incompatible - github.com/Azure/go-autorest/autorest v0.11.23 - github.com/Azure/go-autorest/autorest/adal v0.9.18 + github.com/Azure/aad-pod-identity v1.8.13 + github.com/Azure/azure-sdk-for-go v68.0.0+incompatible + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 + github.com/Azure/go-autorest/autorest v0.11.27 + github.com/Azure/go-autorest/autorest/adal v0.9.20 github.com/Azure/go-autorest/autorest/azure/auth v0.5.10 github.com/Azure/go-autorest/autorest/to v0.4.0 github.com/Azure/go-autorest/tracing v0.6.0 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d github.com/blang/semver v3.5.1+incompatible - github.com/go-logr/logr v1.2.2 + github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 - github.com/google/go-cmp v0.5.7 + github.com/google/go-cmp v0.5.8 github.com/google/gofuzz v1.2.0 github.com/google/uuid v1.3.0 github.com/hashicorp/go-retryablehttp v0.7.0 github.com/hashicorp/golang-lru v0.5.4 github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.17.0 + github.com/onsi/gomega v1.19.0 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.12.1 + github.com/prometheus/client_golang v1.12.2 github.com/spf13/pflag v1.0.5 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.27.0 go.opentelemetry.io/otel v1.4.0 @@ -32,29 +34,31 @@ require ( go.opentelemetry.io/otel/sdk v1.4.0 go.opentelemetry.io/otel/sdk/metric v0.27.0 go.opentelemetry.io/otel/trace v1.4.0 - golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 - golang.org/x/mod v0.5.1 + golang.org/x/crypto v0.12.0 + golang.org/x/mod v0.8.0 helm.sh/helm/v3 v3.8.1 - k8s.io/api v0.23.5 - k8s.io/apimachinery v0.23.5 - k8s.io/client-go v0.23.5 - k8s.io/component-base v0.23.5 - k8s.io/klog/v2 v2.30.0 - k8s.io/kubectl v0.23.4 - k8s.io/utils v0.0.0-20211116205334-6203023598ed + k8s.io/api v0.25.0 + k8s.io/apimachinery v0.25.0 + k8s.io/client-go v0.25.0 + k8s.io/component-base v0.25.0 + k8s.io/klog/v2 v2.80.1 + k8s.io/kubectl v0.25.0 + k8s.io/utils v0.0.0-20230726121419-3b25d923346b sigs.k8s.io/cluster-api v1.1.1 sigs.k8s.io/cluster-api/test v1.1.4 - sigs.k8s.io/controller-runtime v0.11.2 + sigs.k8s.io/controller-runtime v0.13.1 sigs.k8s.io/kind v0.14.0 ) require ( + github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect github.com/BurntSushi/toml v1.0.0 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect @@ -65,11 +69,12 @@ require ( github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/alessio/shellescape v1.4.1 // indirect - github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e // indirect + github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.1.2 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect + github.com/chai2010/gettext-go v1.0.2 // indirect github.com/containerd/containerd v1.5.9 // indirect github.com/coredns/caddy v1.1.0 // indirect github.com/coredns/corefile-migration v1.0.14 // indirect @@ -77,19 +82,20 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/docker/cli v20.10.11+incompatible // indirect - github.com/docker/distribution v2.7.1+incompatible // indirect + github.com/docker/distribution v2.8.1+incompatible // indirect github.com/docker/docker v20.10.16+incompatible // indirect github.com/docker/docker-credential-helpers v0.6.4 // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 // indirect + github.com/emicklei/go-restful/v3 v3.8.0 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect github.com/fatih/color v1.13.0 // indirect github.com/felixge/httpsnoop v1.0.2 // indirect - github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/go-errors/errors v1.0.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect @@ -99,15 +105,16 @@ require ( github.com/gobwas/glob v0.2.3 // indirect github.com/gofrs/uuid v4.2.0+incompatible // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.0.0 // indirect + github.com/golang-jwt/jwt/v4 v4.4.3 // indirect + github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/btree v1.0.1 // indirect - github.com/google/cel-go v0.9.0 // indirect + github.com/google/cel-go v0.12.4 // indirect + github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/go-github/v33 v33.0.0 // indirect github.com/google/go-querystring v1.0.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/googleapis/gnostic v0.5.5 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gosuri/uitable v0.0.4 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect @@ -121,6 +128,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.13.6 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/lib/pq v1.10.4 // indirect @@ -138,17 +146,18 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/spdystream v0.2.0 // indirect - github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect + github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/morikuni/aec v1.0.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nxadm/tail v1.4.8 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect github.com/pelletier/go-toml v1.9.4 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect @@ -163,47 +172,45 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/viper v1.10.0 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect - github.com/stretchr/testify v1.7.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect github.com/valyala/fastjson v1.6.3 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect - github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect + github.com/xlab/treeprint v1.1.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.4.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.4.0 // indirect go.opentelemetry.io/otel/internal/metric v0.27.0 // indirect go.opentelemetry.io/proto/otlp v0.12.0 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect - golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d // indirect + golang.org/x/net v0.14.0 // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect - golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect - golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect - golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect - golang.org/x/text v0.3.7 // indirect - golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/term v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect + golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect - google.golang.org/grpc v1.44.0 // indirect - google.golang.org/protobuf v1.27.1 // indirect + google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect + google.golang.org/grpc v1.47.0 // indirect + google.golang.org/protobuf v1.28.0 // indirect gopkg.in/gorp.v1 v1.7.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.66.2 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect - k8s.io/apiextensions-apiserver v0.23.5 // indirect - k8s.io/apiserver v0.23.5 // indirect - k8s.io/cli-runtime v0.23.4 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.25.0 // indirect + k8s.io/apiserver v0.25.0 // indirect + k8s.io/cli-runtime v0.25.0 // indirect k8s.io/cluster-bootstrap v0.23.0 // indirect - k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect + k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect oras.land/oras-go v1.1.0 // indirect - sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect - sigs.k8s.io/kustomize/api v0.10.1 // indirect - sigs.k8s.io/kustomize/kyaml v0.13.0 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect + sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect + sigs.k8s.io/kustomize/api v0.12.1 // indirect + sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index 1c34a4a3d8b..37d6030d3a1 100644 --- a/go.sum +++ b/go.sum @@ -37,7 +37,6 @@ cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM7 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/firestore v1.6.0/go.mod h1:afJwI0vaXwAG54kI7A//lP/lSPDkQORQuMkv56TxEPU= cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= @@ -48,15 +47,19 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/aad-pod-identity v1.8.6 h1:aa5ybqA5d2e18MPEQyXvIDDGmXMa6bN7ZxK5PKpWQoE= -github.com/Azure/aad-pod-identity v1.8.6/go.mod h1:A+7rb0WOEhBmVaFSl/MtdVCiugoTilY7GpwCnrgzm2w= +github.com/Azure/aad-pod-identity v1.8.13 h1:/gUmacA0z7+lsOlGYAYzkGvAB/KOkUe5Pb6qSeiHD0k= +github.com/Azure/aad-pod-identity v1.8.13/go.mod h1:uxM/lsPo/abzqdk0rwEm4SqO9pavMz0fCmKpYAj4HL8= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v56.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v57.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v63.4.0+incompatible h1:fle3M5Q7vr8auaiPffKyUQmLbvYeqpw30bKU6PrWJFo= -github.com/Azure/azure-sdk-for-go v63.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 h1:/iHxaJhsFr0+xVFfbMr5vxz848jyiWuIEDhYq3y5odY= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 h1:LNHhpdK7hzUcx/k1LIcuh5k7k1LGIWLQfCjaneSj7Fc= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1/go.mod h1:uE9zaUfEQT/nbQjVi2IblCG9iaLtZsuYZ8ne+PuQ02M= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= @@ -68,15 +71,15 @@ github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKn github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest v0.11.19/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest v0.11.20/go.mod h1:o3tqFY+QR40VOlk+pV4d77mORO64jOXSgEnPQgLK6JY= -github.com/Azure/go-autorest/autorest v0.11.23 h1:bRQWsW25/YkoxnIqXMPF94JW33qWDcrPMZ3bINaAruU= -github.com/Azure/go-autorest/autorest v0.11.23/go.mod h1:BAWYUWGPEtKPzjVkp0Q6an0MJcJDsoh5Z1BFAEFs4Xs= +github.com/Azure/go-autorest/autorest v0.11.27 h1:F3R3q42aWytozkV8ihzcgMO4OA4cuqr3bNlsEuF6//A= +github.com/Azure/go-autorest/autorest v0.11.27/go.mod h1:7l8ybrIdUmGqZMTD0sRtAr8NvbHjfofbf8RSP2q7w7U= github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/adal v0.9.14/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.15/go.mod h1:tGMin8I49Yij6AQ+rvV+Xa/zwxYQB5hmsd6DkfAx2+A= -github.com/Azure/go-autorest/autorest/adal v0.9.18 h1:kLnPsRjzZZUF3K5REu/Kc+qMQrvuza2bwSnNdhmzLfQ= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/adal v0.9.20 h1:gJ3E98kMpFB1MFqQCvA1yFab8vthOeD4VlFRQULxahg= +github.com/Azure/go-autorest/autorest/adal v0.9.20/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= github.com/Azure/go-autorest/autorest/azure/auth v0.5.10 h1:F9A3Z++TtAoFysBsNOIJILoHuYBaYvhVGsMGEqPtIS8= github.com/Azure/go-autorest/autorest/azure/auth v0.5.10/go.mod h1:zQXYYNX9kXzRMrJNVXWUfNy38oPMF5/2TeZ4Wylc9fE= github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 h1:dMOmEJfkLKW/7JsokJqkyoYSgmR08hi9KrhjZb+JALY= @@ -84,8 +87,9 @@ github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpz github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= +github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= @@ -95,6 +99,8 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw72xHJc34BNNykqSOeEJDAWkhf0u12/Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU= @@ -109,11 +115,9 @@ github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6 github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60= github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= @@ -128,7 +132,6 @@ github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugX github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= @@ -155,31 +158,29 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= -github.com/ajeddeloh/go-json v0.0.0-20160803184958-73d058cf8437/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= -github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e h1:GCzyKMDDjSGnlpl3clrdAK7I1AaVoaiKDOYkUzChZzg= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= +github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed h1:ue9pVfIcP+QMEjfgo/Ez4ZjNZfonGgR6NgjMaJMu1Cg= +github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/aws/aws-sdk-go v1.8.39/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.34.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= @@ -199,6 +200,8 @@ github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqO github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70= @@ -217,13 +220,13 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 h1:7aWHqerlJ41y6FOsEUvknqgXnGmJyJSbjhAWq5pO4F8= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= +github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= +github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -347,18 +350,14 @@ github.com/coredns/corefile-migration v1.0.14/go.mod h1:XnhgULOEouimnzgn0t4WPuFD github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-iptables v0.3.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.1.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -394,13 +393,15 @@ github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/ github.com/distribution/distribution/v3 v3.0.0-20211118083504-a29a3c99a684 h1:DBZ2sN7CK6dgvHVpQsQj4sRMCbWTmd17l+5SUCjnQSY= github.com/distribution/distribution/v3 v3.0.0-20211118083504-a29a3c99a684/go.mod h1:UfCu3YXJJCI+IdnqGgYP82dk2+Joxmv+mUTVBES6wac= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v20.10.11+incompatible h1:tXU1ezXcruZQRrMP8RN2z9N91h+6egZTS1gsPsKantc= github.com/docker/cli v20.10.11+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= +github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.11+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.12+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= @@ -431,6 +432,8 @@ github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7fo github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful/v3 v3.8.0 h1:eCZ8ulSerjdAiaNpF7GxXIE7ZCMo1moN1qX+S609eVw= +github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -440,6 +443,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= @@ -459,16 +463,15 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o= github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/flatcar-linux/container-linux-config-transpiler v0.9.2/go.mod h1:AGVTulMzeIKwurV9ExYH3UiokET1Ur65g+EIeRDMwzM= -github.com/flatcar-linux/ignition v0.36.1/go.mod h1:0jS5n4AopgOdwgi7QDo5MFgkMx/fQUDYjuxlGJC1Txg= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= @@ -491,12 +494,13 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.2 h1:ahHml/yUpnlb96Rp8HCvtYVPY8ZYpxq3g7UYchIYwbs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= +github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -530,7 +534,6 @@ github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= -github.com/godbus/dbus v0.0.0-20181025153459-66d97aec3384/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -548,8 +551,12 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.0.0 h1:RAqyYixv1p7uEnocuy8P1nru5wprCh/MH2BIlW5z5/o= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= +github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= +github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= @@ -596,9 +603,12 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/cel-go v0.9.0 h1:u1hg7lcZ/XWw2d3aV1jFS30ijQQ6q0/h1C2ZBeBD1gY= github.com/google/cel-go v0.9.0/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w= +github.com/google/cel-go v0.12.4 h1:YINKfuHZ8n72tPOqSPZBwGiDpew2CJS48mdM5W8LZQU= +github.com/google/cel-go v0.12.4/go.mod h1:Av7CU6r6X3YmcHR9GXqVDaEJYfEtSxl6wvIjUQTriCw= github.com/google/cel-spec v0.6.0/go.mod h1:Nwjgxy5CbjlPrtCWjeDjUyKMl8w41YBYGjsyDdqk0xA= +github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -611,8 +621,9 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= github.com/google/go-github/v33 v33.0.0 h1:qAf9yP0qc54ufQxzwv+u9H0tiVOnPJxo0lI/JXqw3ZM= github.com/google/go-github/v33 v33.0.0/go.mod h1:GMdDnVZY/2TsWgp/lkYnpSAh6TrzhANBBwm6k6TTEXg= @@ -655,7 +666,6 @@ github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pf github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= -github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= @@ -681,7 +691,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= @@ -792,6 +801,8 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= @@ -873,7 +884,6 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= @@ -890,8 +900,9 @@ github.com/moby/sys/mountinfo v0.5.0 h1:2Ks8/r6lopsxWi9m58nlwjaeSzUX9iiL1vj5qB/9 github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= -github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 h1:yH0SvLzcbZxcJXho2yh7CqdENGMQe73Cw3woZBpPli0= github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -905,6 +916,7 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -928,9 +940,9 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -939,8 +951,8 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= -github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -970,7 +982,6 @@ github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xA github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -980,7 +991,8 @@ github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+v github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= -github.com/pin/tftp v2.1.0+incompatible/go.mod h1:xVpZOMCXTy+A5QMjEVN0Glwa1sUvaJhFXbr/aAxuxGY= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -994,15 +1006,15 @@ github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSg github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34= +github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -1011,7 +1023,6 @@ github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= @@ -1023,7 +1034,6 @@ github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuI github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -1048,10 +1058,8 @@ github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNue github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= -github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYIR88KRMEuODE= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= @@ -1062,8 +1070,6 @@ github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNX github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sigma/bdoor v0.0.0-20160202064022-babf2a4017b0/go.mod h1:WBu7REWbxC/s/J06jsk//d+9DOz9BbsmcIrimuGRFbs= -github.com/sigma/vmw-guestinfo v0.0.0-20160204083807-95dd4126d6e8/go.mod h1:JrRFFC0veyh0cibh0DAhriSY7/gV3kDdNaVUOmfx01U= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -1074,10 +1080,8 @@ github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -1111,7 +1115,6 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= -github.com/spf13/viper v1.9.0/go.mod h1:+i6ajR7OX2XaiBkrcZJFK21htRk7eDeLg7+O6bhUPP4= github.com/spf13/viper v1.10.0 h1:mXH0UwHS4D2HwWZa75im4xIQynLfblmWV7qcWpfv0yk= github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= @@ -1147,15 +1150,12 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc= github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= -github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= -github.com/vmware/vmw-ovflib v0.0.0-20170608004843-1f217b9dc714/go.mod h1:jiPk45kn7klhByRvUq5i2vo1RtHKBHj+iWGFpxbXuuI= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= @@ -1166,8 +1166,9 @@ github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk= +github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1195,7 +1196,6 @@ go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3 go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= -go.etcd.io/etcd/client/v3 v3.5.1/go.mod h1:OnjH4M8OnAotwaB2l9bVgZzRFKru7/ZMoS46OtKyd3Q= go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= @@ -1208,7 +1208,6 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9p0= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= @@ -1218,7 +1217,6 @@ go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzox go.opentelemetry.io/otel v1.2.0/go.mod h1:aT17Fk0Z1Nor9e0uisf98LrntPGMnk4frBO9+dkf69I= go.opentelemetry.io/otel v1.4.0 h1:7ESuKPq6zpjRaY5nvVDGiuwK7VAJ8MwkKnmNJ9whNZ4= go.opentelemetry.io/otel v1.4.0/go.mod h1:jeAqMFKy2uLIxCtKxoFj0FAL5zAPKQagc3+GtBWakzk= -go.opentelemetry.io/otel/exporters/otlp v0.20.0 h1:PTNgq9MRmQqqJY0REVbZFvwkYOA85vbdQU/nVfxDyqg= go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.4.0 h1:j7AwzDdAQBJjcqayAaYbvpYeZzII7cEe5qJTu+De6UY= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.4.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4= @@ -1257,7 +1255,6 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -1266,10 +1263,7 @@ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9i go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= -go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= -go4.org v0.0.0-20160314031811-03efcb870d84/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= -go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= +go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1292,8 +1286,10 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 h1:/pEO3GD/ABYAjuakUS6xSEmmlyVS4kxBNkeA9tLJiTI= golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1330,8 +1326,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38= -golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1343,7 +1339,6 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -1385,7 +1380,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -1393,8 +1387,9 @@ golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d h1:62NvYBuaanGXR2ZOfwDFkhhl6X1DUgf8qg3GuQvxZsE= golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1424,8 +1419,9 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1523,6 +1519,7 @@ golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1534,27 +1531,29 @@ golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20190321115727-fe223c5a2583/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1562,8 +1561,9 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220609170525-579cf78fd858 h1:Dpdu/EMxGMFgq0CeYMh4fazTD2vtlZRYE7wyynxJb9U= +golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1636,7 +1636,6 @@ golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpd golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= @@ -1753,8 +1752,9 @@ google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 h1:Et6SkiuvnBn+SgrSYXs/BrUpGB4mbdwt4R3vaPIlicA= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 h1:hrbNEivu7Zn1pxvHk6MBrq9iE22woVILTHqexqBxe6I= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1787,8 +1787,10 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8= +google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1802,8 +1804,9 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1823,7 +1826,6 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= @@ -1845,8 +1847,9 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= @@ -1866,50 +1869,43 @@ k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= k8s.io/api v0.23.0/go.mod h1:8wmDdLBHBNxtOIytwLstXt5E9PddnZb0GaMcqsvDBpg= k8s.io/api v0.23.4/go.mod h1:i77F4JfyNNrhOjZF7OwwNJS5Y1S9dpwvb9iYRYRczfI= -k8s.io/api v0.23.5 h1:zno3LUiMubxD/V1Zw3ijyKO3wxrhbUF1Ck+VjBvfaoA= -k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8= -k8s.io/apiextensions-apiserver v0.23.0/go.mod h1:xIFAEEDlAZgpVBl/1VSjGDmLoXAWRG40+GsWhKhAxY4= +k8s.io/api v0.25.0 h1:H+Q4ma2U/ww0iGB78ijZx6DRByPz6/733jIuFpX70e0= +k8s.io/api v0.25.0/go.mod h1:ttceV1GyV1i1rnmvzT3BST08N6nGt+dudGrquzVQWPk= k8s.io/apiextensions-apiserver v0.23.4/go.mod h1:TWYAKymJx7nLMxWCgWm2RYGXHrGlVZnxIlGnvtfYu+g= -k8s.io/apiextensions-apiserver v0.23.5 h1:5SKzdXyvIJKu+zbfPc3kCbWpbxi+O+zdmAJBm26UJqI= -k8s.io/apiextensions-apiserver v0.23.5/go.mod h1:ntcPWNXS8ZPKN+zTXuzYMeg731CP0heCTl6gYBxLcuQ= +k8s.io/apiextensions-apiserver v0.25.0 h1:CJ9zlyXAbq0FIW8CD7HHyozCMBpDSiH7EdrSTCZcZFY= +k8s.io/apiextensions-apiserver v0.25.0/go.mod h1:3pAjZiN4zw7R8aZC5gR0y3/vCkGlAjCazcg1me8iB/E= k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= k8s.io/apimachinery v0.23.0/go.mod h1:fFCTTBKvKcwTPFzjlcxp91uPFZr+JA0FubU4fLzzFYc= k8s.io/apimachinery v0.23.4/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= -k8s.io/apimachinery v0.23.5 h1:Va7dwhp8wgkUPWsEXk6XglXWU4IKYLKNlv8VkX7SDM0= -k8s.io/apimachinery v0.23.5/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= +k8s.io/apimachinery v0.25.0 h1:MlP0r6+3XbkUG2itd6vp3oxbtdQLQI94fD5gCS+gnoU= +k8s.io/apimachinery v0.25.0/go.mod h1:qMx9eAk0sZQGsXGu86fab8tZdffHbwUfsvzqKn4mfB0= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= -k8s.io/apiserver v0.23.0/go.mod h1:Cec35u/9zAepDPPFyT+UMrgqOCjgJ5qtfVJDxjZYmt4= k8s.io/apiserver v0.23.4/go.mod h1:A6l/ZcNtxGfPSqbFDoxxOjEjSKBaQmE+UTveOmMkpNc= -k8s.io/apiserver v0.23.5 h1:2Ly8oUjz5cnZRn1YwYr+aFgDZzUmEVL9RscXbnIeDSE= -k8s.io/apiserver v0.23.5/go.mod h1:7wvMtGJ42VRxzgVI7jkbKvMbuCbVbgsWFT7RyXiRNTw= -k8s.io/cli-runtime v0.23.0/go.mod h1:B5N3YH0KP1iKr6gEuJ/RRmGjO0mJQ/f/JrsmEiPQAlU= -k8s.io/cli-runtime v0.23.4 h1:C3AFQmo4TK4dlVPLOI62gtHEHu0OfA2Cp4UVRZ1JXns= +k8s.io/apiserver v0.25.0 h1:8kl2ifbNffD440MyvHtPaIz1mw4mGKVgWqM0nL+oyu4= +k8s.io/apiserver v0.25.0/go.mod h1:BKwsE+PTC+aZK+6OJQDPr0v6uS91/HWxX7evElAH6xo= k8s.io/cli-runtime v0.23.4/go.mod h1:7KywUNTUibmHPqmpDFuRO1kc9RhsufHv2lkjCm2YZyM= +k8s.io/cli-runtime v0.25.0 h1:XBnTc2Fi+w818jcJGzhiJKQuXl8479sZ4FhtV5hVJ1Q= +k8s.io/cli-runtime v0.25.0/go.mod h1:bHOI5ZZInRHhbq12OdUiYZQN8ml8aKZLwQgt9QlLINw= k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= -k8s.io/client-go v0.23.0/go.mod h1:hrDnpnK1mSr65lHHcUuIZIXDgEbzc7/683c6hyG4jTA= k8s.io/client-go v0.23.4/go.mod h1:PKnIL4pqLuvYUK1WU7RLTMYKPiIh7MYShLshtRY9cj0= -k8s.io/client-go v0.23.5 h1:zUXHmEuqx0RY4+CsnkOn5l0GU+skkRXKGJrhmE2SLd8= -k8s.io/client-go v0.23.5/go.mod h1:flkeinTO1CirYgzMPRWxUCnV0G4Fbu2vLhYCObnt/r4= +k8s.io/client-go v0.25.0 h1:CVWIaCETLMBNiTUta3d5nzRbXvY5Hy9Dpl+VvREpu5E= +k8s.io/client-go v0.25.0/go.mod h1:lxykvypVfKilxhTklov0wz1FoaUZ8X4EwbhS6rpRfN8= k8s.io/cluster-bootstrap v0.23.0 h1:8pZuuAWPoygewSNB4IddX3HBwXcQkPDXL/ca7GtGf4o= k8s.io/cluster-bootstrap v0.23.0/go.mod h1:VltEnKWfrRTiKgOXp3ts3vh7yqNlH6KFKFflo9GtCBg= k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= -k8s.io/code-generator v0.23.0/go.mod h1:vQvOhDXhuzqiVfM/YHp+dmg10WDZCchJVObc9MvowsE= k8s.io/code-generator v0.23.4/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/code-generator v0.23.5/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= -k8s.io/component-base v0.23.0/go.mod h1:DHH5uiFvLC1edCpvcTDV++NKULdYYU6pR9Tt3HIKMKI= k8s.io/component-base v0.23.4/go.mod h1:8o3Gg8i2vnUXGPOwciiYlkSaZT+p+7gA9Scoz8y4W4E= -k8s.io/component-base v0.23.5 h1:8qgP5R6jG1BBSXmRYW+dsmitIrpk8F/fPEvgDenMCCE= -k8s.io/component-base v0.23.5/go.mod h1:c5Nq44KZyt1aLl0IpHX82fhsn84Sb0jjzwjpcA42bY0= -k8s.io/component-helpers v0.23.0/go.mod h1:liXMh6FZS4qamKtMJQ7uLHnFe3tlC86RX5mJEk/aerg= +k8s.io/component-base v0.25.0 h1:haVKlLkPCFZhkcqB6WCvpVxftrg6+FK5x1ZuaIDaQ5Y= +k8s.io/component-base v0.25.0/go.mod h1:F2Sumv9CnbBlqrpdf7rKZTmmd2meJq0HizeyY/yAFxk= k8s.io/component-helpers v0.23.4/go.mod h1:1Pl7L4zukZ054ElzRbvmZ1FJIU8roBXFOeRFu8zipa4= k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= @@ -1922,24 +1918,26 @@ k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAE k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.30.0 h1:bUO6drIvCIsvZ/XFgfxoGFQU/a4Qkh0iAlvUR7vlHJw= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= +k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= -k8s.io/kubectl v0.23.0/go.mod h1:TfcGEs3u4dkmoC2eku1GYymdGaMtPMcaLLFrX/RB2kI= -k8s.io/kubectl v0.23.4 h1:mAa+zEOlyZieecEy+xSrhjkpMcukYyHWzcNdX28dzMY= +k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 h1:MQ8BAZPZlWk3S9K4a9NCkIFQtZShWqoha7snGixVgEA= +k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU= k8s.io/kubectl v0.23.4/go.mod h1:Dgb0Rvx/8JKS/C2EuvsNiQc6RZnX0SbHJVG3XUzH6ok= +k8s.io/kubectl v0.25.0 h1:/Wn1cFqo8ik3iee1EvpxYre3bkWsGLXzLQI6uCCAkQc= +k8s.io/kubectl v0.25.0/go.mod h1:n16ULWsOl2jmQpzt2o7Dud1t4o0+Y186ICb4O+GwKAU= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= -k8s.io/metrics v0.23.0/go.mod h1:NDiZTwppEtAuKJ1Rxt3S4dhyRzdp6yUcJf0vo023dPo= k8s.io/metrics v0.23.4/go.mod h1:cl6sY9BdVT3DubbpqnkPIKi6mn/F2ltkU4yH1tEJ3Bo= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20211116205334-6203023598ed h1:ck1fRPWPJWsMd8ZRFsWc6mh/zHp5fZ/shhbrgPUxDAE= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= oras.land/oras-go v1.1.0 h1:tfWM1RT7PzUwWphqHU6ptPU3ZhwVnSw/9nEGf519rYg= oras.land/oras-go v1.1.0/go.mod h1:1A7vR/0KknT2UkJVWh+xMi95I/AhK8ZrxrnUSmXN0bQ= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= @@ -1947,33 +1945,33 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.25/go.mod h1:Mlj9PNLmG9bZ6BHFwFKDo5afkpWyUISkb9Me0GnK66I= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.27/go.mod h1:tq2nT0Kx7W+/f2JVE+zxYtUhdjuELJkVpNz+x/QN5R4= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= sigs.k8s.io/cluster-api v1.1.1 h1:wU1J+Yyu9xDC5M2uACH4Ii8vrIk37oZiCKeMEfyL0pk= sigs.k8s.io/cluster-api v1.1.1/go.mod h1:aSxmmNwDjmg9rDaL0RhRELpGX9uSTTwLF02AMtk/XGY= sigs.k8s.io/cluster-api/test v1.1.4 h1:QJSVvAeGtBSf4ImRiyPe1cTZJAGC7QGHZckEfCjLGco= sigs.k8s.io/cluster-api/test v1.1.4/go.mod h1:10gpXFNXUuYHoFfmGR98KfsJHuTQIWjRkiU8ggmXbrc= -sigs.k8s.io/controller-runtime v0.11.0/go.mod h1:KKwLiTooNGu+JmLZGn9Sl3Gjmfj66eMbCQznLP5zcqA= -sigs.k8s.io/controller-runtime v0.11.2 h1:H5GTxQl0Mc9UjRJhORusqfJCIjBO8UtUxGggCwL1rLA= -sigs.k8s.io/controller-runtime v0.11.2/go.mod h1:P6QCzrEjLaZGqHsfd+os7JQ+WFZhvB8MRFsn4dWF7O4= -sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s= +sigs.k8s.io/controller-runtime v0.13.1 h1:tUsRCSJVM1QQOOeViGeX3GMT3dQF1eePPw6sEE3xSlg= +sigs.k8s.io/controller-runtime v0.13.1/go.mod h1:Zbz+el8Yg31jubvAEyglRZGdLAjplZl+PgtYNI6WNTI= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kind v0.14.0 h1:cNmI3jGBvp7UegEGbC5we8plDtCUmaNRL+bod7JoSCE= sigs.k8s.io/kind v0.14.0/go.mod h1:UrFRPHG+2a5j0Q7qiR4gtJ4rEyn8TuMQwuOPf+m4oHg= -sigs.k8s.io/kustomize/api v0.10.1 h1:KgU7hfYoscuqag84kxtzKdEC3mKMb99DPI3a0eaV1d0= sigs.k8s.io/kustomize/api v0.10.1/go.mod h1:2FigT1QN6xKdcnGS2Ppp1uIWrtWN28Ms8A3OZUZhwr8= +sigs.k8s.io/kustomize/api v0.12.1 h1:7YM7gW3kYBwtKvoY216ZzY+8hM+lV53LUayghNRJ0vM= +sigs.k8s.io/kustomize/api v0.12.1/go.mod h1:y3JUhimkZkR6sbLNwfJHxvo1TCLwuwm14sCYnkH6S1s= sigs.k8s.io/kustomize/cmd/config v0.10.2/go.mod h1:K2aW7nXJ0AaT+VA/eO0/dzFLxmpFcTzudmAgDwPY1HQ= sigs.k8s.io/kustomize/kustomize/v4 v4.4.1/go.mod h1:qOKJMMz2mBP+vcS7vK+mNz4HBLjaQSWRY22EF6Tb7Io= -sigs.k8s.io/kustomize/kyaml v0.13.0 h1:9c+ETyNfSrVhxvphs+K2dzT3dh5oVPPEqPOE/cUpScY= sigs.k8s.io/kustomize/kyaml v0.13.0/go.mod h1:FTJxEZ86ScK184NpGSAQcfEqee0nul8oLCK30D47m4E= +sigs.k8s.io/kustomize/kyaml v0.13.9 h1:Qz53EAaFFANyNgyOEJbT/yoIHygK40/ZcvU3rgry2Tk= +sigs.k8s.io/kustomize/kyaml v0.13.9/go.mod h1:QsRbD0/KcU+wdk0/L0fIp2KLnohkVzs6fQ85/nOXac4= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/structured-merge-diff/v4 v4.2.0/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= diff --git a/hack/observability/prometheus/resources/bundle.yaml b/hack/observability/prometheus/resources/bundle.yaml index 2fce683d225..e65100bf1f3 100644 --- a/hack/observability/prometheus/resources/bundle.yaml +++ b/hack/observability/prometheus/resources/bundle.yaml @@ -4486,7 +4486,7 @@ spec: If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'. items: - description: PersistentVolumeClaimCondition contails + description: PersistentVolumeClaimCondition contains details about state of pvc properties: lastProbeTime: @@ -11335,7 +11335,7 @@ spec: If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'. items: - description: PersistentVolumeClaimCondition contails + description: PersistentVolumeClaimCondition contains details about state of pvc properties: lastProbeTime: @@ -17146,7 +17146,7 @@ spec: If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'. items: - description: PersistentVolumeClaimCondition contails + description: PersistentVolumeClaimCondition contains details about state of pvc properties: lastProbeTime: diff --git a/main.go b/main.go index e6be45517ce..0a0143bb52a 100644 --- a/main.go +++ b/main.go @@ -24,6 +24,7 @@ import ( _ "net/http/pprof" "os" "time" + "crypto/tls" // +kubebuilder:scaffold:imports aadpodv1 "github.com/Azure/aad-pod-identity/pkg/apis/aadpodidentity/v1" @@ -59,11 +60,18 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/manager" + cliflag "k8s.io/component-base/cli/flag" ) +type TLSOptions struct { + TLSMinVersion string + TLSCipherSuites []string +} + var ( scheme = runtime.NewScheme() setupLog = ctrl.Log.WithName("setup") + tlsOptions = TLSOptions{} ) func init() { @@ -224,7 +232,7 @@ func InitFlags(fs *pflag.FlagSet) { fs.IntVar(&webhookPort, "webhook-port", - 9443, + 0, "Webhook Server port, disabled by default. When enabled, the manager will only work as webhook server, no reconcilers are installed.", ) @@ -268,6 +276,12 @@ func main() { BurstSize: 100, }) + tlsOptionOverrides, err := GetTLSOptionOverrideFuncs(tlsOptions) + if err != nil { + setupLog.Error(err, "unable to add TLS settings to the webhook server") + os.Exit(1) + } + restConfig := ctrl.GetConfigOrDie() restConfig.UserAgent = "cluster-api-provider-azure-manager" mgr, err := ctrl.NewManager(restConfig, ctrl.Options{ @@ -285,6 +299,7 @@ func main() { HealthProbeBindAddress: healthAddr, Port: webhookPort, EventBroadcaster: broadcaster, + TLSOpts: tlsOptionOverrides, }) if err != nil { setupLog.Error(err, "unable to start manager") @@ -309,9 +324,11 @@ func main() { os.Exit(1) } - registerControllers(ctx, mgr) - - registerWebhooks(mgr) + if webhookPort == 0 { + registerControllers(ctx, mgr) + } else { + registerWebhooks(mgr) + } // +kubebuilder:scaffold:builder setupLog.Info("starting manager", "version", version.Get().String()) @@ -321,11 +338,38 @@ func main() { } } +// GetTLSOptionOverrideFuncs returns a list of TLS configuration overrides to be used +// by the webhook server. +func GetTLSOptionOverrideFuncs(options TLSOptions) ([]func(*tls.Config), error) { + var tlsOptions []func(config *tls.Config) + tlsVersion, err := cliflag.TLSVersion(options.TLSMinVersion) + if err != nil { + return nil, err + } + tlsOptions = append(tlsOptions, func(cfg *tls.Config) { + cfg.MinVersion = tlsVersion + cfg.CipherSuites = GetDefaultTLSCipherSuits() + cfg.MaxVersion = tlsVersion + }) + + return tlsOptions, nil +} + +func GetDefaultTLSCipherSuits() []uint16 { + return []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + } +} + func registerControllers(ctx context.Context, mgr manager.Manager) { machineCache, err := coalescing.NewRequestCache(debouncingTimer) if err != nil { setupLog.Error(err, "failed to build machineCache ReconcileCache") } + setupLog.V(0).Info("registerControllers") if err := controllers.NewAzureMachineReconciler(mgr.GetClient(), mgr.GetEventRecorderFor("azuremachine-reconciler"), reconcileTimeout, @@ -472,6 +516,7 @@ func registerControllers(ctx context.Context, mgr manager.Manager) { } func registerWebhooks(mgr manager.Manager) { + setupLog.V(0).Info("registerWebhooks") if err := (&infrav1beta1.AzureCluster{}).SetupWebhookWithManager(mgr); err != nil { setupLog.Error(err, "unable to create webhook", "webhook", "AzureCluster") os.Exit(1) diff --git a/spectro/base/kustomization.yaml b/spectro/base/kustomization.yaml new file mode 100644 index 00000000000..199e3878578 --- /dev/null +++ b/spectro/base/kustomization.yaml @@ -0,0 +1,32 @@ +namespace: capz-system + +namePrefix: capz- + +# Labels to add to all resources and selectors. +commonLabels: + cluster.x-k8s.io/provider: "infrastructure-azure" + +resources: + - ../../config/manager + + +patchesStrategicMerge: + - ../../config/default/manager_image_patch.yaml + - ../../config/default/manager_pull_policy.yaml + - ../../config/default/manager_credentials_patch.yaml + +patchesJson6902: + - target: + group: apps + kind: Deployment + name: controller-manager + namespace: system + version: v1 + path: patch_service_account.yaml + - target: + group: apps + kind: Deployment + name: controller-manager + namespace: system + version: v1 + path: patch_healthcheck.yaml \ No newline at end of file diff --git a/spectro/base/patch_healthcheck.yaml b/spectro/base/patch_healthcheck.yaml new file mode 100644 index 00000000000..1279812a6d2 --- /dev/null +++ b/spectro/base/patch_healthcheck.yaml @@ -0,0 +1,6 @@ +- op: remove + path: "/spec/template/spec/containers/0/ports" +- op: remove + path: "/spec/template/spec/containers/0/livenessProbe" +- op: remove + path: "/spec/template/spec/containers/0/readinessProbe" \ No newline at end of file diff --git a/spectro/base/patch_service_account.yaml b/spectro/base/patch_service_account.yaml new file mode 100644 index 00000000000..99d7e697b56 --- /dev/null +++ b/spectro/base/patch_service_account.yaml @@ -0,0 +1,2 @@ +- op: remove + path: "/spec/template/spec/serviceAccountName" \ No newline at end of file diff --git a/spectro/generated/core-base.yaml b/spectro/generated/core-base.yaml new file mode 100644 index 00000000000..1ee34adc537 --- /dev/null +++ b/spectro/generated/core-base.yaml @@ -0,0 +1,67 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + aadpodidbinding: capz-controller-aadpodidentity-selector + cluster.x-k8s.io/provider: infrastructure-azure + control-plane: capz-controller-manager + name: capz-controller-manager + namespace: capz-system +spec: + replicas: 1 + selector: + matchLabels: + cluster.x-k8s.io/provider: infrastructure-azure + control-plane: capz-controller-manager + template: + metadata: + annotations: + kubectl.kubernetes.io/default-logs-container: manager + labels: + aadpodidbinding: capz-controller-aadpodidentity-selector + cluster.x-k8s.io/provider: infrastructure-azure + control-plane: capz-controller-manager + spec: + containers: + - args: + - --leader-elect + - --metrics-bind-addr=localhost:8080 + - --feature-gates=MachinePool=${EXP_MACHINE_POOL:=false},AKS=${EXP_AKS:=false} + - --v=0 + env: + - name: AZURE_SUBSCRIPTION_ID + valueFrom: + secretKeyRef: + key: subscription-id + name: manager-bootstrap-credentials + - name: AZURE_TENANT_ID + valueFrom: + secretKeyRef: + key: tenant-id + name: manager-bootstrap-credentials + - name: AZURE_CLIENT_ID + valueFrom: + secretKeyRef: + key: client-id + name: manager-bootstrap-credentials + - name: AZURE_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client-secret + name: manager-bootstrap-credentials + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/spectro-dev-public/ubuntu/release/cluster-api-azure-controller:v1.3.2-spectro-4.0.0-dev + imagePullPolicy: Always + name: manager + terminationGracePeriodSeconds: 10 diff --git a/spectro/generated/core-global.yaml b/spectro/generated/core-global.yaml new file mode 100644 index 00000000000..18148548248 --- /dev/null +++ b/spectro/generated/core-global.yaml @@ -0,0 +1,11253 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-azure + name: capi-webhook-system +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-azure + cluster.x-k8s.io/v1beta1: v1beta1 + name: azureclusteridentities.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureClusterIdentity + listKind: AzureClusterIdentityList + plural: azureclusteridentities + singular: azureclusteridentity + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: AzureClusterIdentity is the Schema for the azureclustersidentities + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureClusterIdentitySpec defines the parameters that are + used to create an AzureIdentity. + properties: + allowedNamespaces: + description: "AllowedNamespaces is an array of namespaces that AzureClusters + can use this Identity from. \n An empty list (default) indicates + that AzureClusters can use this Identity from any namespace. This + field is intentionally not a pointer because the nil behavior (no + namespaces) is undesirable here." + items: + type: string + type: array + clientID: + description: Both User Assigned MSI and SP can use this field. + type: string + clientSecret: + description: ClientSecret is a secret reference which should contain + either a Service Principal password or certificate secret. + properties: + name: + description: name is unique within a namespace to reference a + secret resource. + type: string + namespace: + description: namespace defines the space within which the secret + name must be unique. + type: string + type: object + resourceID: + description: User assigned MSI resource id. + type: string + tenantID: + description: Service principal primary tenant id. + type: string + type: + description: UserAssignedMSI or Service Principal + enum: + - ServicePrincipal + - UserAssignedMSI + type: string + required: + - clientID + - tenantID + - type + type: object + status: + description: AzureClusterIdentityStatus defines the observed state of + AzureClusterIdentity. + properties: + conditions: + description: Conditions defines current service state of the AzureClusterIdentity. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + type: object + type: object + served: true + storage: false + subresources: + status: {} + - name: v1alpha4 + schema: + openAPIV3Schema: + description: AzureClusterIdentity is the Schema for the azureclustersidentities + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureClusterIdentitySpec defines the parameters that are + used to create an AzureIdentity. + properties: + allowedNamespaces: + description: AllowedNamespaces is used to identify the namespaces + the clusters are allowed to use the identity from. Namespaces can + be selected either using an array of namespaces or with label selector. + An empty allowedNamespaces object indicates that AzureClusters can + use this identity from any namespace. If this object is nil, no + namespaces will be allowed (default behaviour, if this field is + not provided) A namespace should be either in the NamespaceList + or match with Selector to use the identity. + nullable: true + properties: + list: + description: A nil or empty list indicates that AzureCluster cannot + use the identity from any namespace. + items: + type: string + nullable: true + type: array + selector: + description: "Selector is a selector of namespaces that AzureCluster + can use this Identity from. This is a standard Kubernetes LabelSelector, + a label query over a set of resources. The result of matchLabels + and matchExpressions are ANDed. \n A nil or empty selector indicates + that AzureCluster cannot use this AzureClusterIdentity from + any namespace." + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If + the operator is In or NotIn, the values array must + be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A + single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is "key", + the operator is "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + type: object + clientID: + description: Both User Assigned MSI and SP can use this field. + type: string + clientSecret: + description: ClientSecret is a secret reference which should contain + either a Service Principal password or certificate secret. + properties: + name: + description: name is unique within a namespace to reference a + secret resource. + type: string + namespace: + description: namespace defines the space within which the secret + name must be unique. + type: string + type: object + resourceID: + description: User assigned MSI resource id. + type: string + tenantID: + description: Service principal primary tenant id. + type: string + type: + description: UserAssignedMSI or Service Principal + enum: + - ServicePrincipal + - UserAssignedMSI + type: string + required: + - clientID + - tenantID + - type + type: object + status: + description: AzureClusterIdentityStatus defines the observed state of + AzureClusterIdentity. + properties: + conditions: + description: Conditions defines current service state of the AzureClusterIdentity. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + type: object + type: object + served: true + storage: false + subresources: + status: {} + - name: v1beta1 + schema: + openAPIV3Schema: + description: AzureClusterIdentity is the Schema for the azureclustersidentities + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureClusterIdentitySpec defines the parameters that are + used to create an AzureIdentity. + properties: + allowedNamespaces: + description: AllowedNamespaces is used to identify the namespaces + the clusters are allowed to use the identity from. Namespaces can + be selected either using an array of namespaces or with label selector. + An empty allowedNamespaces object indicates that AzureClusters can + use this identity from any namespace. If this object is nil, no + namespaces will be allowed (default behaviour, if this field is + not provided) A namespace should be either in the NamespaceList + or match with Selector to use the identity. + nullable: true + properties: + list: + description: A nil or empty list indicates that AzureCluster cannot + use the identity from any namespace. + items: + type: string + nullable: true + type: array + selector: + description: "Selector is a selector of namespaces that AzureCluster + can use this Identity from. This is a standard Kubernetes LabelSelector, + a label query over a set of resources. The result of matchLabels + and matchExpressions are ANDed. \n A nil or empty selector indicates + that AzureCluster cannot use this AzureClusterIdentity from + any namespace." + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If + the operator is In or NotIn, the values array must + be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A + single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is "key", + the operator is "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + type: object + clientID: + description: ClientID is the service principal client ID. Both User + Assigned MSI and SP can use this field. + type: string + clientSecret: + description: ClientSecret is a secret reference which should contain + either a Service Principal password or certificate secret. + properties: + name: + description: name is unique within a namespace to reference a + secret resource. + type: string + namespace: + description: namespace defines the space within which the secret + name must be unique. + type: string + type: object + resourceID: + description: ResourceID is the Azure resource ID for the User Assigned + MSI resource. Not currently supported. + type: string + tenantID: + description: TenantID is the service principal primary tenant id. + type: string + type: + description: Type is the type of Azure Identity used. ServicePrincipal, + ServicePrincipalCertificate, or ManualServicePrincipal. + enum: + - ServicePrincipal + - ManualServicePrincipal + - ServicePrincipalCertificate + type: string + required: + - clientID + - tenantID + - type + type: object + status: + description: AzureClusterIdentityStatus defines the observed state of + AzureClusterIdentity. + properties: + conditions: + description: Conditions defines current service state of the AzureClusterIdentity. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-azure + cluster.x-k8s.io/v1beta1: v1beta1 + name: azureclusters.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureCluster + listKind: AzureClusterList + plural: azureclusters + singular: azurecluster + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Cluster to which this AzureCluster belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + type: string + - jsonPath: .status.ready + name: Ready + type: boolean + - jsonPath: .spec.resourceGroup + name: Resource Group + priority: 1 + type: string + - jsonPath: .spec.subscriptionID + name: SubscriptionID + priority: 1 + type: string + - jsonPath: .spec.location + name: Location + priority: 1 + type: string + - description: Control Plane Endpoint + jsonPath: .spec.controlPlaneEndpoint.host + name: Endpoint + priority: 1 + type: string + name: v1alpha3 + schema: + openAPIV3Schema: + description: AzureCluster is the Schema for the azureclusters API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureClusterSpec defines the desired state of AzureCluster. + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to Azure + resources managed by the Azure provider, in addition to the ones + added by default. + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + identityRef: + description: IdentityRef is a reference to a AzureIdentity to be used + when reconciling this cluster + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + location: + type: string + networkSpec: + description: NetworkSpec encapsulates all things related to Azure + network. + properties: + apiServerLB: + description: APIServerLB is the configuration for the control-plane + load balancer. + properties: + frontendIPs: + items: + description: FrontendIP defines a load balancer frontend + IP configuration. + properties: + name: + minLength: 1 + type: string + privateIP: + type: string + publicIP: + description: PublicIPSpec defines the inputs to create + an Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + required: + - name + type: object + type: array + id: + type: string + name: + type: string + sku: + description: SKU defines an Azure load balancer SKU. + type: string + type: + description: LBType defines an Azure load balancer Type. + type: string + type: object + subnets: + description: Subnets is the configuration for the control-plane + subnet and the node subnet. + items: + description: SubnetSpec configures an Azure subnet. + properties: + cidrBlock: + description: 'CidrBlock is the CIDR block to be used when + the provider creates a managed Vnet. Deprecated: Use CIDRBlocks + instead' + type: string + cidrBlocks: + description: CIDRBlocks defines the subnet's address space, + specified as one or more address prefixes in CIDR notation. + items: + type: string + type: array + id: + description: ID defines a unique identifier to reference + this resource. + type: string + internalLBIPAddress: + description: 'InternalLBIPAddress is the IP address that + will be used as the internal LB private IP. For the control + plane subnet only. Deprecated: Use LoadBalancer private + IP instead' + type: string + name: + description: Name defines a name for the subnet resource. + type: string + role: + description: Role defines the subnet role (eg. Node, ControlPlane) + type: string + routeTable: + description: RouteTable defines the route table that should + be attached to this subnet. + properties: + id: + type: string + name: + type: string + type: object + securityGroup: + description: SecurityGroup defines the NSG (network security + group) that should be attached to this subnet. + properties: + id: + type: string + ingressRule: + description: IngressRules is a slice of Azure ingress + rules for security groups. + items: + description: IngressRule defines an Azure ingress + rule for security groups. + properties: + description: + type: string + destination: + description: Destination - The destination address + prefix. CIDR or destination IP range. Asterix + '*' can also be used to match all source IPs. + Default tags such as 'VirtualNetwork', 'AzureLoadBalancer' + and 'Internet' can also be used. + type: string + destinationPorts: + description: DestinationPorts - The destination + port or range. Integer or range between 0 and + 65535. Asterix '*' can also be used to match + all ports. + type: string + name: + type: string + priority: + description: Priority - A number between 100 and + 4096. Each rule should have a unique value for + priority. Rules are processed in priority order, + with lower numbers processed before higher numbers. + Once traffic matches a rule, processing stops. + format: int32 + type: integer + protocol: + description: SecurityGroupProtocol defines the + protocol type for a security group rule. + type: string + source: + description: Source - The CIDR or source IP range. + Asterix '*' can also be used to match all source + IPs. Default tags such as 'VirtualNetwork', + 'AzureLoadBalancer' and 'Internet' can also + be used. If this is an ingress rule, specifies + where network traffic originates from. + type: string + sourcePorts: + description: SourcePorts - The source port or + range. Integer or range between 0 and 65535. + Asterix '*' can also be used to match all ports. + type: string + required: + - description + - name + - protocol + type: object + type: array + name: + type: string + tags: + additionalProperties: + type: string + description: Tags defines a map of tags. + type: object + type: object + required: + - name + type: object + type: array + vnet: + description: Vnet is the configuration for the Azure virtual network. + properties: + cidrBlock: + description: 'CidrBlock is the CIDR block to be used when + the provider creates a managed virtual network. Deprecated: + Use CIDRBlocks instead' + type: string + cidrBlocks: + description: CIDRBlocks defines the virtual network's address + space, specified as one or more address prefixes in CIDR + notation. + items: + type: string + type: array + id: + description: ID is the identifier of the virtual network this + provider should use to create resources. + type: string + name: + description: Name defines a name for the virtual network resource. + type: string + resourceGroup: + description: ResourceGroup is the name of the resource group + of the existing virtual network or the resource group where + a managed virtual network should be created. + type: string + tags: + additionalProperties: + type: string + description: Tags is a collection of tags describing the resource. + type: object + required: + - name + type: object + type: object + resourceGroup: + type: string + subscriptionID: + type: string + required: + - location + type: object + status: + description: AzureClusterStatus defines the observed state of AzureCluster. + properties: + conditions: + description: Conditions defines current service state of the AzureCluster. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + failureDomains: + additionalProperties: + description: FailureDomainSpec is the Schema for Cluster API failure + domains. It allows controllers to understand how many failure + domains a cluster can optionally span across. + properties: + attributes: + additionalProperties: + type: string + description: Attributes is a free form map of attributes an + infrastructure provider might use or require. + type: object + controlPlane: + description: ControlPlane determines if this failure domain + is suitable for use by control plane machines. + type: boolean + type: object + description: 'FailureDomains specifies the list of unique failure + domains for the location/region of the cluster. A FailureDomain + maps to Availability Zone with an Azure Region (if the region support + them). An Availability Zone is a separate data center within a region + and they can be used to ensure the cluster is more resilient to + failure. See: https://docs.microsoft.com/en-us/azure/availability-zones/az-overview + This list will be used by Cluster API to try and spread the machines + across the failure domains.' + type: object + ready: + description: Ready is true when the provider resource is ready. + type: boolean + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Cluster to which this AzureCluster belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].reason + name: Reason + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].message + name: Message + priority: 1 + type: string + - jsonPath: .spec.resourceGroup + name: Resource Group + priority: 1 + type: string + - jsonPath: .spec.subscriptionID + name: SubscriptionID + priority: 1 + type: string + - jsonPath: .spec.location + name: Location + priority: 1 + type: string + - description: Control Plane Endpoint + jsonPath: .spec.controlPlaneEndpoint.host + name: Endpoint + priority: 1 + type: string + name: v1alpha4 + schema: + openAPIV3Schema: + description: AzureCluster is the Schema for the azureclusters API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureClusterSpec defines the desired state of AzureCluster. + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to Azure + resources managed by the Azure provider, in addition to the ones + added by default. + type: object + azureEnvironment: + description: 'AzureEnvironment is the name of the AzureCloud to be + used. The default value that would be used by most users is "AzurePublicCloud", + other values are: - ChinaCloud: "AzureChinaCloud" - GermanCloud: + "AzureGermanCloud" - PublicCloud: "AzurePublicCloud" - USGovernmentCloud: + "AzureUSGovernmentCloud"' + type: string + bastionSpec: + description: BastionSpec encapsulates all things related to the Bastions + in the cluster. + properties: + azureBastion: + description: AzureBastion specifies how the Azure Bastion cloud + component should be configured. + properties: + name: + type: string + publicIP: + description: PublicIPSpec defines the inputs to create an + Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + subnet: + description: SubnetSpec configures an Azure subnet. + properties: + cidrBlocks: + description: CIDRBlocks defines the subnet's address space, + specified as one or more address prefixes in CIDR notation. + items: + type: string + type: array + id: + description: ID defines a unique identifier to reference + this resource. + type: string + name: + description: Name defines a name for the subnet resource. + type: string + natGateway: + description: NatGateway associated with this subnet. + properties: + id: + type: string + ip: + description: PublicIPSpec defines the inputs to create + an Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + name: + type: string + type: object + role: + description: Role defines the subnet role (eg. Node, ControlPlane) + type: string + routeTable: + description: RouteTable defines the route table that should + be attached to this subnet. + properties: + id: + type: string + name: + type: string + type: object + securityGroup: + description: SecurityGroup defines the NSG (network security + group) that should be attached to this subnet. + properties: + id: + type: string + name: + type: string + securityRules: + description: SecurityRules is a slice of Azure security + rules for security groups. + items: + description: SecurityRule defines an Azure security + rule for security groups. + properties: + description: + description: A description for this rule. Restricted + to 140 chars. + type: string + destination: + description: Destination is the destination + address prefix. CIDR or destination IP range. + Asterix '*' can also be used to match all + source IPs. Default tags such as 'VirtualNetwork', + 'AzureLoadBalancer' and 'Internet' can also + be used. + type: string + destinationPorts: + description: DestinationPorts specifies the + destination port or range. Integer or range + between 0 and 65535. Asterix '*' can also + be used to match all ports. + type: string + direction: + description: Direction indicates whether the + rule applies to inbound, or outbound traffic. + "Inbound" or "Outbound". + enum: + - Inbound + - Outbound + type: string + name: + description: Name is a unique name within the + network security group. + type: string + priority: + description: Priority is a number between 100 + and 4096. Each rule should have a unique value + for priority. Rules are processed in priority + order, with lower numbers processed before + higher numbers. Once traffic matches a rule, + processing stops. + format: int32 + type: integer + protocol: + description: Protocol specifies the protocol + type. "Tcp", "Udp", "Icmp", or "*". + enum: + - Tcp + - Udp + - Icmp + - '*' + type: string + source: + description: Source specifies the CIDR or source + IP range. Asterix '*' can also be used to + match all source IPs. Default tags such as + 'VirtualNetwork', 'AzureLoadBalancer' and + 'Internet' can also be used. If this is an + ingress rule, specifies where network traffic + originates from. + type: string + sourcePorts: + description: SourcePorts specifies source port + or range. Integer or range between 0 and 65535. + Asterix '*' can also be used to match all + ports. + type: string + required: + - description + - direction + - name + - protocol + type: object + type: array + tags: + additionalProperties: + type: string + description: Tags defines a map of tags. + type: object + type: object + required: + - name + type: object + type: object + type: object + cloudProviderConfigOverrides: + description: 'CloudProviderConfigOverrides is an optional set of configuration + values that can be overridden in azure cloud provider config. This + is only a subset of options that are available in azure cloud provider + config. Some values for the cloud provider config are inferred from + other parts of cluster api provider azure spec, and may not be available + for overrides. See: https://kubernetes-sigs.github.io/cloud-provider-azure/install/configs + Note: All cloud provider config values can be customized by creating + the secret beforehand. CloudProviderConfigOverrides is only used + when the secret is managed by the Azure Provider.' + properties: + backOffs: + description: BackOffConfig indicates the back-off config options. + properties: + cloudProviderBackoff: + type: boolean + cloudProviderBackoffDuration: + type: integer + cloudProviderBackoffExponent: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cloudProviderBackoffJitter: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cloudProviderBackoffRetries: + type: integer + type: object + rateLimits: + items: + description: 'RateLimitSpec represents the rate limit configuration + for a particular kind of resource. Eg. loadBalancerRateLimit + is used to configure rate limits for load balancers. This + eventually gets converted to CloudProviderRateLimitConfig + that cloud-provider-azure expects. See: https://github.com/kubernetes-sigs/cloud-provider-azure/blob/d585c2031925b39c925624302f22f8856e29e352/pkg/provider/azure_ratelimit.go#L25 + We cannot use CloudProviderRateLimitConfig directly because + floating point values are not supported in controller-tools. + See: https://github.com/kubernetes-sigs/controller-tools/issues/245' + properties: + config: + description: RateLimitConfig indicates the rate limit config + options. + properties: + cloudProviderRateLimit: + type: boolean + cloudProviderRateLimitBucket: + type: integer + cloudProviderRateLimitBucketWrite: + type: integer + cloudProviderRateLimitQPS: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cloudProviderRateLimitQPSWrite: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + name: + description: Name is the name of the rate limit spec. + enum: + - defaultRateLimit + - routeRateLimit + - subnetsRateLimit + - interfaceRateLimit + - routeTableRateLimit + - loadBalancerRateLimit + - publicIPAddressRateLimit + - securityGroupRateLimit + - virtualMachineRateLimit + - storageAccountRateLimit + - diskRateLimit + - snapshotRateLimit + - virtualMachineScaleSetRateLimit + - virtualMachineSizesRateLimit + - availabilitySetRateLimit + type: string + type: object + type: array + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + identityRef: + description: IdentityRef is a reference to an AzureIdentity to be + used when reconciling this cluster + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + location: + type: string + networkSpec: + description: NetworkSpec encapsulates all things related to Azure + network. + properties: + apiServerLB: + description: APIServerLB is the configuration for the control-plane + load balancer. + properties: + frontendIPs: + items: + description: FrontendIP defines a load balancer frontend + IP configuration. + properties: + name: + minLength: 1 + type: string + privateIP: + type: string + publicIP: + description: PublicIPSpec defines the inputs to create + an Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + required: + - name + type: object + type: array + frontendIPsCount: + description: FrontendIPsCount specifies the number of frontend + IP addresses for the load balancer. + format: int32 + type: integer + id: + type: string + idleTimeoutInMinutes: + description: IdleTimeoutInMinutes specifies the timeout for + the TCP idle connection. + format: int32 + type: integer + name: + type: string + sku: + description: SKU defines an Azure load balancer SKU. + type: string + type: + description: LBType defines an Azure load balancer Type. + type: string + type: object + controlPlaneOutboundLB: + description: ControlPlaneOutboundLB is the configuration for the + control-plane outbound load balancer. This is different from + APIServerLB, and is used only in private clusters (optionally) + for enabling outbound traffic. + properties: + frontendIPs: + items: + description: FrontendIP defines a load balancer frontend + IP configuration. + properties: + name: + minLength: 1 + type: string + privateIP: + type: string + publicIP: + description: PublicIPSpec defines the inputs to create + an Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + required: + - name + type: object + type: array + frontendIPsCount: + description: FrontendIPsCount specifies the number of frontend + IP addresses for the load balancer. + format: int32 + type: integer + id: + type: string + idleTimeoutInMinutes: + description: IdleTimeoutInMinutes specifies the timeout for + the TCP idle connection. + format: int32 + type: integer + name: + type: string + sku: + description: SKU defines an Azure load balancer SKU. + type: string + type: + description: LBType defines an Azure load balancer Type. + type: string + type: object + nodeOutboundLB: + description: NodeOutboundLB is the configuration for the node + outbound load balancer. + properties: + frontendIPs: + items: + description: FrontendIP defines a load balancer frontend + IP configuration. + properties: + name: + minLength: 1 + type: string + privateIP: + type: string + publicIP: + description: PublicIPSpec defines the inputs to create + an Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + required: + - name + type: object + type: array + frontendIPsCount: + description: FrontendIPsCount specifies the number of frontend + IP addresses for the load balancer. + format: int32 + type: integer + id: + type: string + idleTimeoutInMinutes: + description: IdleTimeoutInMinutes specifies the timeout for + the TCP idle connection. + format: int32 + type: integer + name: + type: string + sku: + description: SKU defines an Azure load balancer SKU. + type: string + type: + description: LBType defines an Azure load balancer Type. + type: string + type: object + privateDNSZoneName: + description: PrivateDNSZoneName defines the zone name for the + Azure Private DNS. + type: string + subnets: + description: Subnets is the configuration for the control-plane + subnet and the node subnet. + items: + description: SubnetSpec configures an Azure subnet. + properties: + cidrBlocks: + description: CIDRBlocks defines the subnet's address space, + specified as one or more address prefixes in CIDR notation. + items: + type: string + type: array + id: + description: ID defines a unique identifier to reference + this resource. + type: string + name: + description: Name defines a name for the subnet resource. + type: string + natGateway: + description: NatGateway associated with this subnet. + properties: + id: + type: string + ip: + description: PublicIPSpec defines the inputs to create + an Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + name: + type: string + type: object + role: + description: Role defines the subnet role (eg. Node, ControlPlane) + type: string + routeTable: + description: RouteTable defines the route table that should + be attached to this subnet. + properties: + id: + type: string + name: + type: string + type: object + securityGroup: + description: SecurityGroup defines the NSG (network security + group) that should be attached to this subnet. + properties: + id: + type: string + name: + type: string + securityRules: + description: SecurityRules is a slice of Azure security + rules for security groups. + items: + description: SecurityRule defines an Azure security + rule for security groups. + properties: + description: + description: A description for this rule. Restricted + to 140 chars. + type: string + destination: + description: Destination is the destination address + prefix. CIDR or destination IP range. Asterix + '*' can also be used to match all source IPs. + Default tags such as 'VirtualNetwork', 'AzureLoadBalancer' + and 'Internet' can also be used. + type: string + destinationPorts: + description: DestinationPorts specifies the destination + port or range. Integer or range between 0 and + 65535. Asterix '*' can also be used to match + all ports. + type: string + direction: + description: Direction indicates whether the rule + applies to inbound, or outbound traffic. "Inbound" + or "Outbound". + enum: + - Inbound + - Outbound + type: string + name: + description: Name is a unique name within the + network security group. + type: string + priority: + description: Priority is a number between 100 + and 4096. Each rule should have a unique value + for priority. Rules are processed in priority + order, with lower numbers processed before higher + numbers. Once traffic matches a rule, processing + stops. + format: int32 + type: integer + protocol: + description: Protocol specifies the protocol type. + "Tcp", "Udp", "Icmp", or "*". + enum: + - Tcp + - Udp + - Icmp + - '*' + type: string + source: + description: Source specifies the CIDR or source + IP range. Asterix '*' can also be used to match + all source IPs. Default tags such as 'VirtualNetwork', + 'AzureLoadBalancer' and 'Internet' can also + be used. If this is an ingress rule, specifies + where network traffic originates from. + type: string + sourcePorts: + description: SourcePorts specifies source port + or range. Integer or range between 0 and 65535. + Asterix '*' can also be used to match all ports. + type: string + required: + - description + - direction + - name + - protocol + type: object + type: array + tags: + additionalProperties: + type: string + description: Tags defines a map of tags. + type: object + type: object + required: + - name + type: object + type: array + vnet: + description: Vnet is the configuration for the Azure virtual network. + properties: + cidrBlocks: + description: CIDRBlocks defines the virtual network's address + space, specified as one or more address prefixes in CIDR + notation. + items: + type: string + type: array + id: + description: ID is the identifier of the virtual network this + provider should use to create resources. + type: string + name: + description: Name defines a name for the virtual network resource. + type: string + resourceGroup: + description: ResourceGroup is the name of the resource group + of the existing virtual network or the resource group where + a managed virtual network should be created. + type: string + tags: + additionalProperties: + type: string + description: Tags is a collection of tags describing the resource. + type: object + required: + - name + type: object + type: object + resourceGroup: + type: string + subscriptionID: + type: string + required: + - location + type: object + status: + description: AzureClusterStatus defines the observed state of AzureCluster. + properties: + conditions: + description: Conditions defines current service state of the AzureCluster. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + failureDomains: + additionalProperties: + description: FailureDomainSpec is the Schema for Cluster API failure + domains. It allows controllers to understand how many failure + domains a cluster can optionally span across. + properties: + attributes: + additionalProperties: + type: string + description: Attributes is a free form map of attributes an + infrastructure provider might use or require. + type: object + controlPlane: + description: ControlPlane determines if this failure domain + is suitable for use by control plane machines. + type: boolean + type: object + description: 'FailureDomains specifies the list of unique failure + domains for the location/region of the cluster. A FailureDomain + maps to Availability Zone with an Azure Region (if the region support + them). An Availability Zone is a separate data center within a region + and they can be used to ensure the cluster is more resilient to + failure. See: https://docs.microsoft.com/en-us/azure/availability-zones/az-overview + This list will be used by Cluster API to try and spread the machines + across the failure domains.' + type: object + longRunningOperationStates: + description: LongRunningOperationStates saves the states for Azure + long-running operations so they can be continued on the next reconciliation + loop. + items: + description: Future contains the data needed for an Azure long-running + operation to continue across reconcile loops. + properties: + data: + description: Data is the base64 url encoded json Azure AutoRest + Future. + type: string + name: + description: Name is the name of the Azure resource. Together + with the service name, this forms the unique identifier for + the future. + type: string + resourceGroup: + description: ResourceGroup is the Azure resource group for the + resource. + type: string + serviceName: + description: ServiceName is the name of the Azure service. Together + with the name of the resource, this forms the unique identifier + for the future. + type: string + type: + description: Type describes the type of future, such as update, + create, delete, etc. + type: string + required: + - name + - serviceName + - type + type: object + type: array + ready: + description: Ready is true when the provider resource is ready. + type: boolean + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Cluster to which this AzureCluster belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].reason + name: Reason + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].message + name: Message + priority: 1 + type: string + - jsonPath: .spec.resourceGroup + name: Resource Group + priority: 1 + type: string + - jsonPath: .spec.subscriptionID + name: SubscriptionID + priority: 1 + type: string + - jsonPath: .spec.location + name: Location + priority: 1 + type: string + - description: Control Plane Endpoint + jsonPath: .spec.controlPlaneEndpoint.host + name: Endpoint + priority: 1 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: AzureCluster is the Schema for the azureclusters API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureClusterSpec defines the desired state of AzureCluster. + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to Azure + resources managed by the Azure provider, in addition to the ones + added by default. + type: object + azureEnvironment: + description: 'AzureEnvironment is the name of the AzureCloud to be + used. The default value that would be used by most users is "AzurePublicCloud", + other values are: - ChinaCloud: "AzureChinaCloud" - GermanCloud: + "AzureGermanCloud" - PublicCloud: "AzurePublicCloud" - USGovernmentCloud: + "AzureUSGovernmentCloud"' + type: string + bastionSpec: + description: BastionSpec encapsulates all things related to the Bastions + in the cluster. + properties: + azureBastion: + description: AzureBastion specifies how the Azure Bastion cloud + component should be configured. + properties: + name: + type: string + publicIP: + description: PublicIPSpec defines the inputs to create an + Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + subnet: + description: SubnetSpec configures an Azure subnet. + properties: + cidrBlocks: + description: CIDRBlocks defines the subnet's address space, + specified as one or more address prefixes in CIDR notation. + items: + type: string + type: array + id: + description: ID is the Azure resource ID of the subnet. + READ-ONLY + type: string + name: + description: Name defines a name for the subnet resource. + type: string + natGateway: + description: NatGateway associated with this subnet. + properties: + id: + description: ID is the Azure resource ID of the NAT + gateway. READ-ONLY + type: string + ip: + description: PublicIPSpec defines the inputs to create + an Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + name: + type: string + required: + - name + type: object + role: + description: Role defines the subnet role (eg. Node, ControlPlane) + enum: + - node + - control-plane + - bastion + - all + type: string + routeTable: + description: RouteTable defines the route table that should + be attached to this subnet. + properties: + id: + description: ID is the Azure resource ID of the route + table. READ-ONLY + type: string + name: + type: string + required: + - name + type: object + securityGroup: + description: SecurityGroup defines the NSG (network security + group) that should be attached to this subnet. + properties: + id: + description: ID is the Azure resource ID of the security + group. READ-ONLY + type: string + name: + type: string + securityRules: + description: SecurityRules is a slice of Azure security + rules for security groups. + items: + description: SecurityRule defines an Azure security + rule for security groups. + properties: + description: + description: A description for this rule. Restricted + to 140 chars. + type: string + destination: + description: Destination is the destination + address prefix. CIDR or destination IP range. + Asterix '*' can also be used to match all + source IPs. Default tags such as 'VirtualNetwork', + 'AzureLoadBalancer' and 'Internet' can also + be used. + type: string + destinationPorts: + description: DestinationPorts specifies the + destination port or range. Integer or range + between 0 and 65535. Asterix '*' can also + be used to match all ports. + type: string + direction: + description: Direction indicates whether the + rule applies to inbound, or outbound traffic. + "Inbound" or "Outbound". + enum: + - Inbound + - Outbound + type: string + name: + description: Name is a unique name within the + network security group. + type: string + priority: + description: Priority is a number between 100 + and 4096. Each rule should have a unique value + for priority. Rules are processed in priority + order, with lower numbers processed before + higher numbers. Once traffic matches a rule, + processing stops. + format: int32 + type: integer + protocol: + description: Protocol specifies the protocol + type. "Tcp", "Udp", "Icmp", or "*". + enum: + - Tcp + - Udp + - Icmp + - '*' + type: string + source: + description: Source specifies the CIDR or source + IP range. Asterix '*' can also be used to + match all source IPs. Default tags such as + 'VirtualNetwork', 'AzureLoadBalancer' and + 'Internet' can also be used. If this is an + ingress rule, specifies where network traffic + originates from. + type: string + sourcePorts: + description: SourcePorts specifies source port + or range. Integer or range between 0 and 65535. + Asterix '*' can also be used to match all + ports. + type: string + required: + - description + - direction + - name + - protocol + type: object + type: array + tags: + additionalProperties: + type: string + description: Tags defines a map of tags. + type: object + required: + - name + type: object + required: + - name + - role + type: object + type: object + type: object + cloudProviderConfigOverrides: + description: 'CloudProviderConfigOverrides is an optional set of configuration + values that can be overridden in azure cloud provider config. This + is only a subset of options that are available in azure cloud provider + config. Some values for the cloud provider config are inferred from + other parts of cluster api provider azure spec, and may not be available + for overrides. See: https://kubernetes-sigs.github.io/cloud-provider-azure/install/configs + Note: All cloud provider config values can be customized by creating + the secret beforehand. CloudProviderConfigOverrides is only used + when the secret is managed by the Azure Provider.' + properties: + backOffs: + description: BackOffConfig indicates the back-off config options. + properties: + cloudProviderBackoff: + type: boolean + cloudProviderBackoffDuration: + type: integer + cloudProviderBackoffExponent: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cloudProviderBackoffJitter: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cloudProviderBackoffRetries: + type: integer + type: object + rateLimits: + items: + description: 'RateLimitSpec represents the rate limit configuration + for a particular kind of resource. Eg. loadBalancerRateLimit + is used to configure rate limits for load balancers. This + eventually gets converted to CloudProviderRateLimitConfig + that cloud-provider-azure expects. See: https://github.com/kubernetes-sigs/cloud-provider-azure/blob/d585c2031925b39c925624302f22f8856e29e352/pkg/provider/azure_ratelimit.go#L25 + We cannot use CloudProviderRateLimitConfig directly because + floating point values are not supported in controller-tools. + See: https://github.com/kubernetes-sigs/controller-tools/issues/245' + properties: + config: + description: RateLimitConfig indicates the rate limit config + options. + properties: + cloudProviderRateLimit: + type: boolean + cloudProviderRateLimitBucket: + type: integer + cloudProviderRateLimitBucketWrite: + type: integer + cloudProviderRateLimitQPS: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cloudProviderRateLimitQPSWrite: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + name: + description: Name is the name of the rate limit spec. + enum: + - defaultRateLimit + - routeRateLimit + - subnetsRateLimit + - interfaceRateLimit + - routeTableRateLimit + - loadBalancerRateLimit + - publicIPAddressRateLimit + - securityGroupRateLimit + - virtualMachineRateLimit + - storageAccountRateLimit + - diskRateLimit + - snapshotRateLimit + - virtualMachineScaleSetRateLimit + - virtualMachineSizesRateLimit + - availabilitySetRateLimit + type: string + required: + - name + type: object + type: array + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to + communicate with the control plane. It is not recommended to set + this when creating an AzureCluster as CAPZ will set this for you. + However, if it is set, CAPZ will not change it. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + identityRef: + description: IdentityRef is a reference to an AzureIdentity to be + used when reconciling this cluster + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + location: + type: string + networkSpec: + description: NetworkSpec encapsulates all things related to Azure + network. + properties: + apiServerLB: + description: APIServerLB is the configuration for the control-plane + load balancer. + properties: + frontendIPs: + items: + description: FrontendIP defines a load balancer frontend + IP configuration. + properties: + name: + minLength: 1 + type: string + privateIP: + type: string + publicIP: + description: PublicIPSpec defines the inputs to create + an Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + required: + - name + type: object + type: array + frontendIPsCount: + description: FrontendIPsCount specifies the number of frontend + IP addresses for the load balancer. + format: int32 + type: integer + id: + description: ID is the Azure resource ID of the load balancer. + READ-ONLY + type: string + idleTimeoutInMinutes: + description: IdleTimeoutInMinutes specifies the timeout for + the TCP idle connection. + format: int32 + type: integer + ipAllocationMethod: + type: string + name: + type: string + sku: + description: SKU defines an Azure load balancer SKU. + type: string + type: + description: LBType defines an Azure load balancer Type. + type: string + type: object + controlPlaneOutboundLB: + description: ControlPlaneOutboundLB is the configuration for the + control-plane outbound load balancer. This is different from + APIServerLB, and is used only in private clusters (optionally) + for enabling outbound traffic. + properties: + frontendIPs: + items: + description: FrontendIP defines a load balancer frontend + IP configuration. + properties: + name: + minLength: 1 + type: string + privateIP: + type: string + publicIP: + description: PublicIPSpec defines the inputs to create + an Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + required: + - name + type: object + type: array + frontendIPsCount: + description: FrontendIPsCount specifies the number of frontend + IP addresses for the load balancer. + format: int32 + type: integer + id: + description: ID is the Azure resource ID of the load balancer. + READ-ONLY + type: string + idleTimeoutInMinutes: + description: IdleTimeoutInMinutes specifies the timeout for + the TCP idle connection. + format: int32 + type: integer + ipAllocationMethod: + type: string + name: + type: string + sku: + description: SKU defines an Azure load balancer SKU. + type: string + type: + description: LBType defines an Azure load balancer Type. + type: string + type: object + nodeOutboundLB: + description: NodeOutboundLB is the configuration for the node + outbound load balancer. + properties: + frontendIPs: + items: + description: FrontendIP defines a load balancer frontend + IP configuration. + properties: + name: + minLength: 1 + type: string + privateIP: + type: string + publicIP: + description: PublicIPSpec defines the inputs to create + an Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + required: + - name + type: object + type: array + frontendIPsCount: + description: FrontendIPsCount specifies the number of frontend + IP addresses for the load balancer. + format: int32 + type: integer + id: + description: ID is the Azure resource ID of the load balancer. + READ-ONLY + type: string + idleTimeoutInMinutes: + description: IdleTimeoutInMinutes specifies the timeout for + the TCP idle connection. + format: int32 + type: integer + ipAllocationMethod: + type: string + name: + type: string + sku: + description: SKU defines an Azure load balancer SKU. + type: string + type: + description: LBType defines an Azure load balancer Type. + type: string + type: object + privateDNSZoneName: + description: PrivateDNSZoneName defines the zone name for the + Azure Private DNS. + type: string + subnets: + description: Subnets is the configuration for the control-plane + subnet and the node subnet. + items: + description: SubnetSpec configures an Azure subnet. + properties: + cidrBlocks: + description: CIDRBlocks defines the subnet's address space, + specified as one or more address prefixes in CIDR notation. + items: + type: string + type: array + id: + description: ID is the Azure resource ID of the subnet. + READ-ONLY + type: string + name: + description: Name defines a name for the subnet resource. + type: string + natGateway: + description: NatGateway associated with this subnet. + properties: + id: + description: ID is the Azure resource ID of the NAT + gateway. READ-ONLY + type: string + ip: + description: PublicIPSpec defines the inputs to create + an Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + name: + type: string + required: + - name + type: object + role: + description: Role defines the subnet role (eg. Node, ControlPlane) + enum: + - node + - control-plane + - bastion + - all + type: string + routeTable: + description: RouteTable defines the route table that should + be attached to this subnet. + properties: + id: + description: ID is the Azure resource ID of the route + table. READ-ONLY + type: string + name: + type: string + required: + - name + type: object + securityGroup: + description: SecurityGroup defines the NSG (network security + group) that should be attached to this subnet. + properties: + id: + description: ID is the Azure resource ID of the security + group. READ-ONLY + type: string + name: + type: string + securityRules: + description: SecurityRules is a slice of Azure security + rules for security groups. + items: + description: SecurityRule defines an Azure security + rule for security groups. + properties: + description: + description: A description for this rule. Restricted + to 140 chars. + type: string + destination: + description: Destination is the destination address + prefix. CIDR or destination IP range. Asterix + '*' can also be used to match all source IPs. + Default tags such as 'VirtualNetwork', 'AzureLoadBalancer' + and 'Internet' can also be used. + type: string + destinationPorts: + description: DestinationPorts specifies the destination + port or range. Integer or range between 0 and + 65535. Asterix '*' can also be used to match + all ports. + type: string + direction: + description: Direction indicates whether the rule + applies to inbound, or outbound traffic. "Inbound" + or "Outbound". + enum: + - Inbound + - Outbound + type: string + name: + description: Name is a unique name within the + network security group. + type: string + priority: + description: Priority is a number between 100 + and 4096. Each rule should have a unique value + for priority. Rules are processed in priority + order, with lower numbers processed before higher + numbers. Once traffic matches a rule, processing + stops. + format: int32 + type: integer + protocol: + description: Protocol specifies the protocol type. + "Tcp", "Udp", "Icmp", or "*". + enum: + - Tcp + - Udp + - Icmp + - '*' + type: string + source: + description: Source specifies the CIDR or source + IP range. Asterix '*' can also be used to match + all source IPs. Default tags such as 'VirtualNetwork', + 'AzureLoadBalancer' and 'Internet' can also + be used. If this is an ingress rule, specifies + where network traffic originates from. + type: string + sourcePorts: + description: SourcePorts specifies source port + or range. Integer or range between 0 and 65535. + Asterix '*' can also be used to match all ports. + type: string + required: + - description + - direction + - name + - protocol + type: object + type: array + tags: + additionalProperties: + type: string + description: Tags defines a map of tags. + type: object + required: + - name + type: object + required: + - name + - role + type: object + type: array + vnet: + description: Vnet is the configuration for the Azure virtual network. + properties: + cidrBlocks: + description: CIDRBlocks defines the virtual network's address + space, specified as one or more address prefixes in CIDR + notation. + items: + type: string + type: array + id: + description: ID is the Azure resource ID of the virtual network. + READ-ONLY + type: string + name: + description: Name defines a name for the virtual network resource. + type: string + peerings: + description: Peerings defines a list of peerings of the newly + created virtual network with existing virtual networks. + items: + description: VnetPeeringSpec specifies an existing remote + virtual network to peer with the AzureCluster's virtual + network. + properties: + remoteVnetName: + description: RemoteVnetName defines name of the remote + virtual network. + type: string + resourceGroup: + description: ResourceGroup is the resource group name + of the remote virtual network. + type: string + required: + - remoteVnetName + type: object + type: array + resourceGroup: + description: ResourceGroup is the name of the resource group + of the existing virtual network or the resource group where + a managed virtual network should be created. + type: string + tags: + additionalProperties: + type: string + description: Tags is a collection of tags describing the resource. + type: object + required: + - name + type: object + type: object + resourceGroup: + type: string + subscriptionID: + type: string + required: + - location + type: object + status: + description: AzureClusterStatus defines the observed state of AzureCluster. + properties: + conditions: + description: Conditions defines current service state of the AzureCluster. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + failureDomains: + additionalProperties: + description: FailureDomainSpec is the Schema for Cluster API failure + domains. It allows controllers to understand how many failure + domains a cluster can optionally span across. + properties: + attributes: + additionalProperties: + type: string + description: Attributes is a free form map of attributes an + infrastructure provider might use or require. + type: object + controlPlane: + description: ControlPlane determines if this failure domain + is suitable for use by control plane machines. + type: boolean + type: object + description: 'FailureDomains specifies the list of unique failure + domains for the location/region of the cluster. A FailureDomain + maps to Availability Zone with an Azure Region (if the region support + them). An Availability Zone is a separate data center within a region + and they can be used to ensure the cluster is more resilient to + failure. See: https://docs.microsoft.com/en-us/azure/availability-zones/az-overview + This list will be used by Cluster API to try and spread the machines + across the failure domains.' + type: object + longRunningOperationStates: + description: LongRunningOperationStates saves the states for Azure + long-running operations so they can be continued on the next reconciliation + loop. + items: + description: Future contains the data needed for an Azure long-running + operation to continue across reconcile loops. + properties: + data: + description: Data is the base64 url encoded json Azure AutoRest + Future. + type: string + name: + description: Name is the name of the Azure resource. Together + with the service name, this forms the unique identifier for + the future. + type: string + resourceGroup: + description: ResourceGroup is the Azure resource group for the + resource. + type: string + serviceName: + description: ServiceName is the name of the Azure service. Together + with the name of the resource, this forms the unique identifier + for the future. + type: string + type: + description: Type describes the type of future, such as update, + create, delete, etc. + type: string + required: + - data + - name + - serviceName + - type + type: object + type: array + ready: + description: Ready is true when the provider resource is ready. + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-azure + cluster.x-k8s.io/v1beta1: v1beta1 + name: azureclustertemplates.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureClusterTemplate + listKind: AzureClusterTemplateList + plural: azureclustertemplates + singular: azureclustertemplate + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: AzureClusterTemplate is the Schema for the azureclustertemplates + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureClusterTemplateSpec defines the desired state of AzureClusterTemplate. + properties: + template: + description: AzureClusterTemplateResource describes the data needed + to create an AzureCluster from a template. + properties: + spec: + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to + add to Azure resources managed by the Azure provider, in + addition to the ones added by default. + type: object + azureEnvironment: + description: 'AzureEnvironment is the name of the AzureCloud + to be used. The default value that would be used by most + users is "AzurePublicCloud", other values are: - ChinaCloud: + "AzureChinaCloud" - GermanCloud: "AzureGermanCloud" - PublicCloud: + "AzurePublicCloud" - USGovernmentCloud: "AzureUSGovernmentCloud"' + type: string + bastionSpec: + description: BastionSpec encapsulates all things related to + the Bastions in the cluster. + properties: + azureBastion: + properties: + subnet: + properties: + cidrBlocks: + description: CIDRBlocks defines the subnet's address + space, specified as one or more address prefixes + in CIDR notation. + items: + type: string + type: array + natGateway: + description: NatGateway associated with this subnet. + properties: + name: + type: string + required: + - name + type: object + role: + description: Role defines the subnet role (eg. + Node, ControlPlane) + enum: + - node + - control-plane + - bastion + - all + type: string + securityGroup: + description: SecurityGroup defines the NSG (network + security group) that should be attached to this + subnet. + properties: + securityRules: + description: SecurityRules is a slice of Azure + security rules for security groups. + items: + description: SecurityRule defines an Azure + security rule for security groups. + properties: + description: + description: A description for this + rule. Restricted to 140 chars. + type: string + destination: + description: Destination is the destination + address prefix. CIDR or destination + IP range. Asterix '*' can also be + used to match all source IPs. Default + tags such as 'VirtualNetwork', 'AzureLoadBalancer' + and 'Internet' can also be used. + type: string + destinationPorts: + description: DestinationPorts specifies + the destination port or range. Integer + or range between 0 and 65535. Asterix + '*' can also be used to match all + ports. + type: string + direction: + description: Direction indicates whether + the rule applies to inbound, or outbound + traffic. "Inbound" or "Outbound". + enum: + - Inbound + - Outbound + type: string + name: + description: Name is a unique name within + the network security group. + type: string + priority: + description: Priority is a number between + 100 and 4096. Each rule should have + a unique value for priority. Rules + are processed in priority order, with + lower numbers processed before higher + numbers. Once traffic matches a rule, + processing stops. + format: int32 + type: integer + protocol: + description: Protocol specifies the + protocol type. "Tcp", "Udp", "Icmp", + or "*". + enum: + - Tcp + - Udp + - Icmp + - '*' + type: string + source: + description: Source specifies the CIDR + or source IP range. Asterix '*' can + also be used to match all source IPs. + Default tags such as 'VirtualNetwork', + 'AzureLoadBalancer' and 'Internet' + can also be used. If this is an ingress + rule, specifies where network traffic + originates from. + type: string + sourcePorts: + description: SourcePorts specifies source + port or range. Integer or range between + 0 and 65535. Asterix '*' can also + be used to match all ports. + type: string + required: + - description + - direction + - name + - protocol + type: object + type: array + tags: + additionalProperties: + type: string + description: Tags defines a map of tags. + type: object + type: object + required: + - role + type: object + type: object + type: object + cloudProviderConfigOverrides: + description: 'CloudProviderConfigOverrides is an optional + set of configuration values that can be overridden in azure + cloud provider config. This is only a subset of options + that are available in azure cloud provider config. Some + values for the cloud provider config are inferred from other + parts of cluster api provider azure spec, and may not be + available for overrides. See: https://kubernetes-sigs.github.io/cloud-provider-azure/install/configs + Note: All cloud provider config values can be customized + by creating the secret beforehand. CloudProviderConfigOverrides + is only used when the secret is managed by the Azure Provider.' + properties: + backOffs: + description: BackOffConfig indicates the back-off config + options. + properties: + cloudProviderBackoff: + type: boolean + cloudProviderBackoffDuration: + type: integer + cloudProviderBackoffExponent: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cloudProviderBackoffJitter: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cloudProviderBackoffRetries: + type: integer + type: object + rateLimits: + items: + description: 'RateLimitSpec represents the rate limit + configuration for a particular kind of resource. Eg. + loadBalancerRateLimit is used to configure rate limits + for load balancers. This eventually gets converted + to CloudProviderRateLimitConfig that cloud-provider-azure + expects. See: https://github.com/kubernetes-sigs/cloud-provider-azure/blob/d585c2031925b39c925624302f22f8856e29e352/pkg/provider/azure_ratelimit.go#L25 + We cannot use CloudProviderRateLimitConfig directly + because floating point values are not supported in + controller-tools. See: https://github.com/kubernetes-sigs/controller-tools/issues/245' + properties: + config: + description: RateLimitConfig indicates the rate + limit config options. + properties: + cloudProviderRateLimit: + type: boolean + cloudProviderRateLimitBucket: + type: integer + cloudProviderRateLimitBucketWrite: + type: integer + cloudProviderRateLimitQPS: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cloudProviderRateLimitQPSWrite: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + name: + description: Name is the name of the rate limit + spec. + enum: + - defaultRateLimit + - routeRateLimit + - subnetsRateLimit + - interfaceRateLimit + - routeTableRateLimit + - loadBalancerRateLimit + - publicIPAddressRateLimit + - securityGroupRateLimit + - virtualMachineRateLimit + - storageAccountRateLimit + - diskRateLimit + - snapshotRateLimit + - virtualMachineScaleSetRateLimit + - virtualMachineSizesRateLimit + - availabilitySetRateLimit + type: string + required: + - name + type: object + type: array + type: object + identityRef: + description: IdentityRef is a reference to an AzureIdentity + to be used when reconciling this cluster + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead + of an entire object, this string should contain a valid + JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that + triggered the event) or if no container name is specified + "spec.containers[2]" (container with index 2 in this + pod). This syntax is chosen only to have some well-defined + way of referencing a part of an object. TODO: this design + is not final and this field is subject to change in + the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + location: + type: string + networkSpec: + description: NetworkSpec encapsulates all things related to + Azure network. + properties: + apiServerLB: + description: APIServerLB is the configuration for the + control-plane load balancer. + properties: + idleTimeoutInMinutes: + description: IdleTimeoutInMinutes specifies the timeout + for the TCP idle connection. + format: int32 + type: integer + ipAllocationMethod: + type: string + sku: + description: SKU defines an Azure load balancer SKU. + type: string + type: + description: LBType defines an Azure load balancer + Type. + type: string + type: object + controlPlaneOutboundLB: + description: ControlPlaneOutboundLB is the configuration + for the control-plane outbound load balancer. This is + different from APIServerLB, and is used only in private + clusters (optionally) for enabling outbound traffic. + properties: + idleTimeoutInMinutes: + description: IdleTimeoutInMinutes specifies the timeout + for the TCP idle connection. + format: int32 + type: integer + ipAllocationMethod: + type: string + sku: + description: SKU defines an Azure load balancer SKU. + type: string + type: + description: LBType defines an Azure load balancer + Type. + type: string + type: object + nodeOutboundLB: + description: NodeOutboundLB is the configuration for the + node outbound load balancer. + properties: + idleTimeoutInMinutes: + description: IdleTimeoutInMinutes specifies the timeout + for the TCP idle connection. + format: int32 + type: integer + ipAllocationMethod: + type: string + sku: + description: SKU defines an Azure load balancer SKU. + type: string + type: + description: LBType defines an Azure load balancer + Type. + type: string + type: object + privateDNSZoneName: + description: PrivateDNSZoneName defines the zone name + for the Azure Private DNS. + type: string + subnets: + description: Subnets is the configuration for the control-plane + subnet and the node subnet. + items: + properties: + cidrBlocks: + description: CIDRBlocks defines the subnet's address + space, specified as one or more address prefixes + in CIDR notation. + items: + type: string + type: array + natGateway: + description: NatGateway associated with this subnet. + properties: + name: + type: string + required: + - name + type: object + role: + description: Role defines the subnet role (eg. Node, + ControlPlane) + enum: + - node + - control-plane + - bastion + - all + type: string + securityGroup: + description: SecurityGroup defines the NSG (network + security group) that should be attached to this + subnet. + properties: + securityRules: + description: SecurityRules is a slice of Azure + security rules for security groups. + items: + description: SecurityRule defines an Azure + security rule for security groups. + properties: + description: + description: A description for this rule. + Restricted to 140 chars. + type: string + destination: + description: Destination is the destination + address prefix. CIDR or destination + IP range. Asterix '*' can also be used + to match all source IPs. Default tags + such as 'VirtualNetwork', 'AzureLoadBalancer' + and 'Internet' can also be used. + type: string + destinationPorts: + description: DestinationPorts specifies + the destination port or range. Integer + or range between 0 and 65535. Asterix + '*' can also be used to match all ports. + type: string + direction: + description: Direction indicates whether + the rule applies to inbound, or outbound + traffic. "Inbound" or "Outbound". + enum: + - Inbound + - Outbound + type: string + name: + description: Name is a unique name within + the network security group. + type: string + priority: + description: Priority is a number between + 100 and 4096. Each rule should have + a unique value for priority. Rules are + processed in priority order, with lower + numbers processed before higher numbers. + Once traffic matches a rule, processing + stops. + format: int32 + type: integer + protocol: + description: Protocol specifies the protocol + type. "Tcp", "Udp", "Icmp", or "*". + enum: + - Tcp + - Udp + - Icmp + - '*' + type: string + source: + description: Source specifies the CIDR + or source IP range. Asterix '*' can + also be used to match all source IPs. + Default tags such as 'VirtualNetwork', + 'AzureLoadBalancer' and 'Internet' can + also be used. If this is an ingress + rule, specifies where network traffic + originates from. + type: string + sourcePorts: + description: SourcePorts specifies source + port or range. Integer or range between + 0 and 65535. Asterix '*' can also be + used to match all ports. + type: string + required: + - description + - direction + - name + - protocol + type: object + type: array + tags: + additionalProperties: + type: string + description: Tags defines a map of tags. + type: object + type: object + required: + - role + type: object + type: array + vnet: + description: Vnet is the configuration for the Azure virtual + network. + properties: + cidrBlocks: + description: CIDRBlocks defines the virtual network's + address space, specified as one or more address + prefixes in CIDR notation. + items: + type: string + type: array + peerings: + description: Peerings defines a list of peerings of + the newly created virtual network with existing + virtual networks. + items: + properties: + remoteVnetName: + description: RemoteVnetName defines name of + the remote virtual network. + type: string + required: + - remoteVnetName + type: object + type: array + tags: + additionalProperties: + type: string + description: Tags is a collection of tags describing + the resource. + type: object + type: object + type: object + subscriptionID: + type: string + required: + - location + type: object + required: + - spec + type: object + required: + - template + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: unapproved + controller-gen.kubebuilder.io/version: v0.5.0 + labels: + cluster.x-k8s.io/provider: infrastructure-azure + name: azureidentities.aadpodidentity.k8s.io +spec: + group: aadpodidentity.k8s.io + names: + kind: AzureIdentity + listKind: AzureIdentityList + plural: azureidentities + singular: azureidentity + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: AzureIdentity is the specification of the identity data structure. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureIdentitySpec describes the credential specifications + of an identity on Azure. + properties: + adEndpoint: + type: string + adResourceID: + description: For service principal. Option param for specifying the AD + details. + type: string + auxiliaryTenantIDs: + description: Service principal auxiliary tenant ids + items: + type: string + nullable: true + type: array + clientID: + description: Both User Assigned MSI and SP can use this field. + type: string + clientPassword: + description: Used for service principal + properties: + name: + description: Name is unique within a namespace to reference a + secret resource. + type: string + namespace: + description: Namespace defines the space within which the secret + name must be unique. + type: string + type: object + metadata: + type: object + replicas: + format: int32 + nullable: true + type: integer + resourceID: + description: User assigned MSI resource id. + type: string + tenantID: + description: Service principal primary tenant id. + type: string + type: + description: UserAssignedMSI or Service Principal + type: integer + type: object + status: + description: AzureIdentityStatus contains the replica status of the resource. + properties: + availableReplicas: + format: int32 + type: integer + metadata: + type: object + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: unapproved + controller-gen.kubebuilder.io/version: v0.5.0 + labels: + cluster.x-k8s.io/provider: infrastructure-azure + name: azureidentitybindings.aadpodidentity.k8s.io +spec: + group: aadpodidentity.k8s.io + names: + kind: AzureIdentityBinding + listKind: AzureIdentityBindingList + plural: azureidentitybindings + singular: azureidentitybinding + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: AzureIdentityBinding brings together the spec of matching pods + and the identity which they can use. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureIdentityBindingSpec matches the pod with the Identity. + Used to indicate the potential matches to look for between the pod/deployment + and the identities present. + properties: + azureIdentity: + type: string + metadata: + type: object + selector: + type: string + weight: + description: Weight is used to figure out which of the matching identities + would be selected. + type: integer + type: object + status: + description: AzureIdentityBindingStatus contains the status of an AzureIdentityBinding. + properties: + availableReplicas: + format: int32 + type: integer + metadata: + type: object + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-azure + cluster.x-k8s.io/v1beta1: v1beta1 + name: azuremachinepoolmachines.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureMachinePoolMachine + listKind: AzureMachinePoolMachineList + plural: azuremachinepoolmachines + shortNames: + - ampm + singular: azuremachinepoolmachine + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Kubernetes version + jsonPath: .status.version + name: Version + type: string + - description: Flag indicating infrastructure is successfully provisioned + jsonPath: .status.ready + name: Ready + type: string + - description: Azure VMSS VM provisioning state + jsonPath: .status.provisioningState + name: State + type: string + - description: Cluster to which this AzureMachinePoolMachine belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + priority: 1 + type: string + - description: Azure VMSS VM ID + jsonPath: .spec.providerID + name: VMSS VM ID + priority: 1 + type: string + name: v1alpha4 + schema: + openAPIV3Schema: + description: AzureMachinePoolMachine is the Schema for the azuremachinepoolmachines + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureMachinePoolMachineSpec defines the desired state of + AzureMachinePoolMachine. + properties: + instanceID: + description: InstanceID is the identification of the Machine Instance + within the VMSS + type: string + providerID: + description: ProviderID is the identification ID of the Virtual Machine + Scale Set + type: string + required: + - instanceID + - providerID + type: object + status: + description: AzureMachinePoolMachineStatus defines the observed state + of AzureMachinePoolMachine. + properties: + conditions: + description: Conditions defines current service state of the AzureMachinePool. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + failureMessage: + description: "FailureMessage will be set in the event that there is + a terminal problem reconciling the MachinePool and will contain + a more verbose string suitable for logging and human consumption. + \n Any transient errors that occur during the reconciliation of + MachinePools can be added as events to the MachinePool object and/or + logged in the controller's output." + type: string + failureReason: + description: "FailureReason will be set in the event that there is + a terminal problem reconciling the MachinePool machine and will + contain a succinct value suitable for machine interpretation. \n + Any transient errors that occur during the reconciliation of MachinePools + can be added as events to the MachinePool object and/or logged in + the controller's output." + type: string + instanceName: + description: InstanceName is the name of the Machine Instance within + the VMSS + type: string + latestModelApplied: + description: LatestModelApplied indicates the instance is running + the most up-to-date VMSS model. A VMSS model describes the image + version the VM is running. If the instance is not running the latest + model, it means the instance may not be running the version of Kubernetes + the Machine Pool has specified and needs to be updated. + type: boolean + longRunningOperationStates: + description: LongRunningOperationStates saves the state for Azure + long running operations so they can be continued on the next reconciliation + loop. + items: + description: Future contains the data needed for an Azure long-running + operation to continue across reconcile loops. + properties: + data: + description: Data is the base64 url encoded json Azure AutoRest + Future. + type: string + name: + description: Name is the name of the Azure resource. Together + with the service name, this forms the unique identifier for + the future. + type: string + resourceGroup: + description: ResourceGroup is the Azure resource group for the + resource. + type: string + serviceName: + description: ServiceName is the name of the Azure service. Together + with the name of the resource, this forms the unique identifier + for the future. + type: string + type: + description: Type describes the type of future, such as update, + create, delete, etc. + type: string + required: + - name + - serviceName + - type + type: object + type: array + nodeRef: + description: NodeRef will point to the corresponding Node if it exists. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + provisioningState: + description: ProvisioningState is the provisioning state of the Azure + virtual machine instance. + type: string + ready: + description: Ready is true when the provider resource is ready. + type: boolean + version: + description: Version defines the Kubernetes version for the VM Instance + type: string + required: + - latestModelApplied + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Kubernetes version + jsonPath: .status.version + name: Version + type: string + - description: Flag indicating infrastructure is successfully provisioned + jsonPath: .status.ready + name: Ready + type: string + - description: Azure VMSS VM provisioning state + jsonPath: .status.provisioningState + name: State + type: string + - description: Cluster to which this AzureMachinePoolMachine belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + priority: 1 + type: string + - description: Azure VMSS VM ID + jsonPath: .spec.providerID + name: VMSS VM ID + priority: 1 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: AzureMachinePoolMachine is the Schema for the azuremachinepoolmachines + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureMachinePoolMachineSpec defines the desired state of + AzureMachinePoolMachine. + properties: + instanceID: + description: InstanceID is the identification of the Machine Instance + within the VMSS + type: string + providerID: + description: ProviderID is the identification ID of the Virtual Machine + Scale Set + type: string + required: + - instanceID + - providerID + type: object + status: + description: AzureMachinePoolMachineStatus defines the observed state + of AzureMachinePoolMachine. + properties: + conditions: + description: Conditions defines current service state of the AzureMachinePool. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + failureMessage: + description: "FailureMessage will be set in the event that there is + a terminal problem reconciling the MachinePool and will contain + a more verbose string suitable for logging and human consumption. + \n Any transient errors that occur during the reconciliation of + MachinePools can be added as events to the MachinePool object and/or + logged in the controller's output." + type: string + failureReason: + description: "FailureReason will be set in the event that there is + a terminal problem reconciling the MachinePool machine and will + contain a succinct value suitable for machine interpretation. \n + Any transient errors that occur during the reconciliation of MachinePools + can be added as events to the MachinePool object and/or logged in + the controller's output." + type: string + instanceName: + description: InstanceName is the name of the Machine Instance within + the VMSS + type: string + latestModelApplied: + description: LatestModelApplied indicates the instance is running + the most up-to-date VMSS model. A VMSS model describes the image + version the VM is running. If the instance is not running the latest + model, it means the instance may not be running the version of Kubernetes + the Machine Pool has specified and needs to be updated. + type: boolean + longRunningOperationStates: + description: LongRunningOperationStates saves the state for Azure + long running operations so they can be continued on the next reconciliation + loop. + items: + description: Future contains the data needed for an Azure long-running + operation to continue across reconcile loops. + properties: + data: + description: Data is the base64 url encoded json Azure AutoRest + Future. + type: string + name: + description: Name is the name of the Azure resource. Together + with the service name, this forms the unique identifier for + the future. + type: string + resourceGroup: + description: ResourceGroup is the Azure resource group for the + resource. + type: string + serviceName: + description: ServiceName is the name of the Azure service. Together + with the name of the resource, this forms the unique identifier + for the future. + type: string + type: + description: Type describes the type of future, such as update, + create, delete, etc. + type: string + required: + - data + - name + - serviceName + - type + type: object + type: array + nodeRef: + description: NodeRef will point to the corresponding Node if it exists. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + provisioningState: + description: ProvisioningState is the provisioning state of the Azure + virtual machine instance. + type: string + ready: + description: Ready is true when the provider resource is ready. + type: boolean + version: + description: Version defines the Kubernetes version for the VM Instance + type: string + required: + - latestModelApplied + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-azure + cluster.x-k8s.io/v1beta1: v1beta1 + name: azuremachinepools.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureMachinePool + listKind: AzureMachinePoolList + plural: azuremachinepools + shortNames: + - amp + singular: azuremachinepool + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: AzureMachinePool replicas count + jsonPath: .status.replicas + name: Replicas + type: string + - description: AzureMachinePool replicas count + jsonPath: .status.ready + name: Ready + type: string + - description: Azure VMSS provisioning state + jsonPath: .status.provisioningState + name: State + type: string + - description: Cluster to which this AzureMachinePool belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + priority: 1 + type: string + - description: MachinePool object to which this AzureMachinePool belongs + jsonPath: .metadata.ownerReferences[?(@.kind=="MachinePool")].name + name: MachinePool + priority: 1 + type: string + - description: Azure VMSS ID + jsonPath: .spec.providerID + name: VMSS ID + priority: 1 + type: string + - description: Azure VM Size + jsonPath: .spec.template.vmSize + name: VM Size + priority: 1 + type: string + name: v1alpha3 + schema: + openAPIV3Schema: + description: AzureMachinePool is the Schema for the azuremachinepools API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureMachinePoolSpec defines the desired state of AzureMachinePool. + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to an + instance, in addition to the ones added by default by the Azure + provider. If both the AzureCluster and the AzureMachine specify + the same tag name with different values, the AzureMachine's value + takes precedence. + type: object + identity: + default: None + description: Identity is the type of identity used for the Virtual + Machine Scale Set. The type 'SystemAssigned' is an implicitly created + identity. The generated identity will be assigned a Subscription + contributor role. The type 'UserAssigned' is a standalone Azure + resource provided by the user and assigned to the VM + enum: + - None + - SystemAssigned + - UserAssigned + type: string + location: + description: Location is the Azure region location e.g. westus2 + type: string + providerID: + description: ProviderID is the identification ID of the Virtual Machine + Scale Set + type: string + providerIDList: + description: ProviderIDList are the identification IDs of machine + instances provided by the provider. This field must match the provider + IDs as seen on the node objects corresponding to a machine pool's + machine instances. + items: + type: string + type: array + roleAssignmentName: + description: RoleAssignmentName is the name of the role assignment + to create for a system assigned identity. It can be any valid GUID. + If not specified, a random GUID will be generated. + type: string + template: + description: Template contains the details used to build a replica + virtual machine within the Machine Pool. + properties: + acceleratedNetworking: + description: AcceleratedNetworking enables or disables Azure accelerated + networking. If omitted, it will be set based on whether the + requested VMSize supports accelerated networking. If AcceleratedNetworking + is set to true with a VMSize that does not support it, Azure + will return an error. + type: boolean + dataDisks: + description: DataDisks specifies the list of data disks to be + created for a Virtual Machine + items: + description: DataDisk specifies the parameters that are used + to add one or more data disks to the machine. + properties: + cachingType: + type: string + diskSizeGB: + description: DiskSizeGB is the size in GB to assign to the + data disk. + format: int32 + type: integer + lun: + description: Lun Specifies the logical unit number of the + data disk. This value is used to identify data disks within + the VM and therefore must be unique for each data disk + attached to a VM. The value must be between 0 and 63. + format: int32 + type: integer + managedDisk: + description: ManagedDisk defines the managed disk options + for a VM. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk + encryption options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + required: + - storageAccountType + type: object + nameSuffix: + description: NameSuffix is the suffix to be appended to + the machine name to generate the disk name. Each disk + name will be in format _. + type: string + required: + - diskSizeGB + - nameSuffix + type: object + type: array + image: + description: Image is used to provide details of an image to use + during Virtual Machine creation. If image details are omitted + the image will default the Azure Marketplace "capi" offer, which + is based on Ubuntu. + properties: + id: + description: ID specifies an image to use by ID + type: string + marketplace: + description: Marketplace specifies an image to use from the + Azure Marketplace + properties: + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization + that created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such + as a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter + minLength: 1 + type: string + thirdPartyImage: + default: false + description: ThirdPartyImage indicates the image is published + by a third party publisher and a Plan will be generated + for it. + type: boolean + version: + description: Version specifies the version of an image + sku. The allowed formats are Major.Minor.Build or 'latest'. + Major, Minor, and Build are decimal numbers. Specify + 'latest' to use the latest version of an image available + at deploy time. Even if you use 'latest', the VM image + will not automatically update after deploy time even + if a new version becomes available. + minLength: 1 + type: string + required: + - offer + - publisher + - sku + - version + type: object + sharedGallery: + description: SharedGallery specifies an image to use from + an Azure Shared Image Gallery + properties: + gallery: + description: Gallery specifies the name of the shared + image gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + resourceGroup: + description: ResourceGroup specifies the resource group + containing the shared image gallery + minLength: 1 + type: string + subscriptionID: + description: SubscriptionID is the identifier of the subscription + that contains the shared image gallery + minLength: 1 + type: string + version: + description: Version specifies the version of the marketplace + image. The allowed formats are Major.Minor.Build or + 'latest'. Major, Minor, and Build are decimal numbers. + Specify 'latest' to use the latest version of an image + available at deploy time. Even if you use 'latest', + the VM image will not automatically update after deploy + time even if a new version becomes available. + minLength: 1 + type: string + required: + - gallery + - name + - resourceGroup + - subscriptionID + - version + type: object + type: object + osDisk: + description: OSDisk contains the operating system disk information + for a Virtual Machine + properties: + cachingType: + type: string + diffDiskSettings: + description: DiffDiskSettings describe ephemeral disk settings + for the os disk. + properties: + option: + description: Option enables ephemeral OS when set to "Local" + See https://docs.microsoft.com/en-us/azure/virtual-machines/ephemeral-os-disks + for full details + enum: + - Local + type: string + required: + - option + type: object + diskSizeGB: + format: int32 + type: integer + managedDisk: + description: ManagedDisk defines the managed disk options + for a VM. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk + encryption options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + required: + - storageAccountType + type: object + osType: + type: string + required: + - diskSizeGB + - managedDisk + - osType + type: object + securityProfile: + description: SecurityProfile specifies the Security profile settings + for a virtual machine. + properties: + encryptionAtHost: + description: This field indicates whether Host Encryption + should be enabled or disabled for a virtual machine or virtual + machine scale set. Default is disabled. + type: boolean + type: object + spotVMOptions: + description: SpotVMOptions allows the ability to specify the Machine + should use a Spot VM + properties: + maxPrice: + anyOf: + - type: integer + - type: string + description: MaxPrice defines the maximum price the user is + willing to pay for Spot VM instances + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + sshPublicKey: + description: SSHPublicKey is the SSH public key string base64 + encoded to add to a Virtual Machine + type: string + terminateNotificationTimeout: + description: TerminateNotificationTimeout enables or disables + VMSS scheduled events termination notification with specified + timeout allowed values are between 5 and 15 (mins) + type: integer + vmSize: + description: VMSize is the size of the Virtual Machine to build. + See https://docs.microsoft.com/en-us/rest/api/compute/virtualmachines/createorupdate#virtualmachinesizetypes + type: string + required: + - osDisk + - sshPublicKey + - vmSize + type: object + userAssignedIdentities: + description: UserAssignedIdentities is a list of standalone Azure + identities provided by the user The lifecycle of a user-assigned + identity is managed separately from the lifecycle of the AzureMachinePool. + See https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-manage-ua-identity-cli + items: + description: UserAssignedIdentity defines the user-assigned identities + provided by the user to be assigned to Azure resources. + properties: + providerID: + description: 'ProviderID is the identification ID of the user-assigned + Identity, the format of an identity is: ''azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}''' + type: string + required: + - providerID + type: object + type: array + required: + - location + - template + type: object + status: + description: AzureMachinePoolStatus defines the observed state of AzureMachinePool. + properties: + conditions: + description: Conditions defines current service state of the AzureMachinePool. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + failureMessage: + description: "FailureMessage will be set in the event that there is + a terminal problem reconciling the MachinePool and will contain + a more verbose string suitable for logging and human consumption. + \n This field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the MachinePool's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of MachinePools can be added as + events to the MachinePool object and/or logged in the controller's + output." + type: string + failureReason: + description: "FailureReason will be set in the event that there is + a terminal problem reconciling the MachinePool and will contain + a succinct value suitable for machine interpretation. \n This field + should not be set for transitive errors that a controller faces + that are expected to be fixed automatically over time (like service + outages), but instead indicate that something is fundamentally wrong + with the MachinePool's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of MachinePools can be added as + events to the MachinePool object and/or logged in the controller's + output." + type: string + instances: + description: Instances is the VM instance status for each VM in the + VMSS + items: + description: AzureMachinePoolInstanceStatus provides status information + for each instance in the VMSS. + properties: + instanceID: + description: InstanceID is the identification of the Machine + Instance within the VMSS + type: string + instanceName: + description: InstanceName is the name of the Machine Instance + within the VMSS + type: string + latestModelApplied: + description: LatestModelApplied indicates the instance is running + the most up-to-date VMSS model. A VMSS model describes the + image version the VM is running. If the instance is not running + the latest model, it means the instance may not be running + the version of Kubernetes the Machine Pool has specified and + needs to be updated. + type: boolean + providerID: + description: ProviderID is the provider identification of the + VMSS Instance + type: string + provisioningState: + description: ProvisioningState is the provisioning state of + the Azure virtual machine instance. + type: string + version: + description: Version defines the Kubernetes version for the + VM Instance + type: string + required: + - latestModelApplied + type: object + type: array + longRunningOperationState: + description: LongRunningOperationState saves the state for an Azure + long running operations so it can be continued on the next reconciliation + loop. + properties: + futureData: + description: FutureData is the base64 url encoded json Azure AutoRest + Future. + type: string + name: + description: Name is the name of the Azure resource. + type: string + resourceGroup: + description: ResourceGroup is the Azure resource group for the + resource. + type: string + type: + description: Type describes the type of future, update, create, + delete, etc. + type: string + required: + - type + type: object + provisioningState: + description: ProvisioningState is the provisioning state of the Azure + virtual machine. + type: string + ready: + description: Ready is true when the provider resource is ready. + type: boolean + replicas: + description: Replicas is the most recently observed number of replicas. + format: int32 + type: integer + version: + description: Version is the Kubernetes version for the current VMSS + model + type: string + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: AzureMachinePool replicas count + jsonPath: .status.replicas + name: Replicas + type: string + - description: AzureMachinePool replicas count + jsonPath: .status.ready + name: Ready + type: string + - description: Azure VMSS provisioning state + jsonPath: .status.provisioningState + name: State + type: string + - description: Cluster to which this AzureMachinePool belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + priority: 1 + type: string + - description: MachinePool object to which this AzureMachinePool belongs + jsonPath: .metadata.ownerReferences[?(@.kind=="MachinePool")].name + name: MachinePool + priority: 1 + type: string + - description: Azure VMSS ID + jsonPath: .spec.providerID + name: VMSS ID + priority: 1 + type: string + - description: Azure VM Size + jsonPath: .spec.template.vmSize + name: VM Size + priority: 1 + type: string + name: v1alpha4 + schema: + openAPIV3Schema: + description: AzureMachinePool is the Schema for the azuremachinepools API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureMachinePoolSpec defines the desired state of AzureMachinePool. + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to an + instance, in addition to the ones added by default by the Azure + provider. If both the AzureCluster and the AzureMachine specify + the same tag name with different values, the AzureMachine's value + takes precedence. + type: object + identity: + default: None + description: Identity is the type of identity used for the Virtual + Machine Scale Set. The type 'SystemAssigned' is an implicitly created + identity. The generated identity will be assigned a Subscription + contributor role. The type 'UserAssigned' is a standalone Azure + resource provided by the user and assigned to the VM + enum: + - None + - SystemAssigned + - UserAssigned + type: string + location: + description: Location is the Azure region location e.g. westus2 + type: string + nodeDrainTimeout: + description: 'NodeDrainTimeout is the total amount of time that the + controller will spend on draining a node. The default value is 0, + meaning that the node can be drained without any time limitations. + NOTE: NodeDrainTimeout is different from `kubectl drain --timeout`' + type: string + providerID: + description: ProviderID is the identification ID of the Virtual Machine + Scale Set + type: string + providerIDList: + description: ProviderIDList are the identification IDs of machine + instances provided by the provider. This field must match the provider + IDs as seen on the node objects corresponding to a machine pool's + machine instances. + items: + type: string + type: array + roleAssignmentName: + description: RoleAssignmentName is the name of the role assignment + to create for a system assigned identity. It can be any valid GUID. + If not specified, a random GUID will be generated. + type: string + strategy: + default: + rollingUpdate: + deletePolicy: Oldest + maxSurge: 1 + maxUnavailable: 0 + type: RollingUpdate + description: The deployment strategy to use to replace existing AzureMachinePoolMachines + with new ones. + properties: + rollingUpdate: + description: Rolling update config params. Present only if MachineDeploymentStrategyType + = RollingUpdate. + properties: + deletePolicy: + default: Oldest + description: DeletePolicy defines the policy used by the MachineDeployment + to identify nodes to delete when downscaling. Valid values + are "Random, "Newest", "Oldest" When no value is supplied, + the default is Oldest + enum: + - Random + - Newest + - Oldest + type: string + maxSurge: + anyOf: + - type: integer + - type: string + default: 1 + description: 'The maximum number of machines that can be scheduled + above the desired number of machines. Value can be an absolute + number (ex: 5) or a percentage of desired machines (ex: + 10%). This can not be 0 if MaxUnavailable is 0. Absolute + number is calculated from percentage by rounding up. Defaults + to 1. Example: when this is set to 30%, the new MachineSet + can be scaled up immediately when the rolling update starts, + such that the total number of old and new machines do not + exceed 130% of desired machines. Once old machines have + been killed, new MachineSet can be scaled up further, ensuring + that total number of machines running at any time during + the update is at most 130% of desired machines.' + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + default: 0 + description: 'The maximum number of machines that can be unavailable + during the update. Value can be an absolute number (ex: + 5) or a percentage of desired machines (ex: 10%). Absolute + number is calculated from percentage by rounding down. This + can not be 0 if MaxSurge is 0. Defaults to 0. Example: when + this is set to 30%, the old MachineSet can be scaled down + to 70% of desired machines immediately when the rolling + update starts. Once new machines are ready, old MachineSet + can be scaled down further, followed by scaling up the new + MachineSet, ensuring that the total number of machines available + at all times during the update is at least 70% of desired + machines.' + x-kubernetes-int-or-string: true + type: object + type: + default: RollingUpdate + description: Type of deployment. Currently the only supported + strategy is RollingUpdate + enum: + - RollingUpdate + type: string + type: object + template: + description: Template contains the details used to build a replica + virtual machine within the Machine Pool + properties: + acceleratedNetworking: + description: AcceleratedNetworking enables or disables Azure accelerated + networking. If omitted, it will be set based on whether the + requested VMSize supports accelerated networking. If AcceleratedNetworking + is set to true with a VMSize that does not support it, Azure + will return an error. + type: boolean + dataDisks: + description: DataDisks specifies the list of data disks to be + created for a Virtual Machine + items: + description: DataDisk specifies the parameters that are used + to add one or more data disks to the machine. + properties: + cachingType: + description: CachingType specifies the caching requirements. + enum: + - None + - ReadOnly + - ReadWrite + type: string + diskSizeGB: + description: DiskSizeGB is the size in GB to assign to the + data disk. + format: int32 + type: integer + lun: + description: Lun Specifies the logical unit number of the + data disk. This value is used to identify data disks within + the VM and therefore must be unique for each data disk + attached to a VM. The value must be between 0 and 63. + format: int32 + type: integer + managedDisk: + description: ManagedDisk specifies the Managed Disk parameters + for the data disk. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk + encryption options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + type: object + nameSuffix: + description: NameSuffix is the suffix to be appended to + the machine name to generate the disk name. Each disk + name will be in format _. + type: string + required: + - diskSizeGB + - nameSuffix + type: object + type: array + image: + description: Image is used to provide details of an image to use + during VM creation. If image details are omitted the image will + default the Azure Marketplace "capi" offer, which is based on + Ubuntu. + properties: + id: + description: ID specifies an image to use by ID + type: string + marketplace: + description: Marketplace specifies an image to use from the + Azure Marketplace + properties: + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization + that created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such + as a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter + minLength: 1 + type: string + thirdPartyImage: + default: false + description: ThirdPartyImage indicates the image is published + by a third party publisher and a Plan will be generated + for it. + type: boolean + version: + description: Version specifies the version of an image + sku. The allowed formats are Major.Minor.Build or 'latest'. + Major, Minor, and Build are decimal numbers. Specify + 'latest' to use the latest version of an image available + at deploy time. Even if you use 'latest', the VM image + will not automatically update after deploy time even + if a new version becomes available. + minLength: 1 + type: string + required: + - offer + - publisher + - sku + - version + type: object + sharedGallery: + description: SharedGallery specifies an image to use from + an Azure Shared Image Gallery + properties: + gallery: + description: Gallery specifies the name of the shared + image gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer This value will be used to add a `Plan` + in the API request when creating the VM/VMSS resource. + This is needed when the source image from which this + SIG image was built requires the `Plan` to be used. + type: string + publisher: + description: Publisher is the name of the organization + that created the image. This value will be used to add + a `Plan` in the API request when creating the VM/VMSS + resource. This is needed when the source image from + which this SIG image was built requires the `Plan` to + be used. + type: string + resourceGroup: + description: ResourceGroup specifies the resource group + containing the shared image gallery + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such + as a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter This value will be used to add a `Plan` + in the API request when creating the VM/VMSS resource. + This is needed when the source image from which this + SIG image was built requires the `Plan` to be used. + type: string + subscriptionID: + description: SubscriptionID is the identifier of the subscription + that contains the shared image gallery + minLength: 1 + type: string + version: + description: Version specifies the version of the marketplace + image. The allowed formats are Major.Minor.Build or + 'latest'. Major, Minor, and Build are decimal numbers. + Specify 'latest' to use the latest version of an image + available at deploy time. Even if you use 'latest', + the VM image will not automatically update after deploy + time even if a new version becomes available. + minLength: 1 + type: string + required: + - gallery + - name + - resourceGroup + - subscriptionID + - version + type: object + type: object + osDisk: + description: OSDisk contains the operating system disk information + for a Virtual Machine + properties: + cachingType: + description: CachingType specifies the caching requirements. + enum: + - None + - ReadOnly + - ReadWrite + type: string + diffDiskSettings: + description: DiffDiskSettings describe ephemeral disk settings + for the os disk. + properties: + option: + description: Option enables ephemeral OS when set to "Local" + See https://docs.microsoft.com/en-us/azure/virtual-machines/ephemeral-os-disks + for full details + enum: + - Local + type: string + required: + - option + type: object + diskSizeGB: + description: DiskSizeGB is the size in GB to assign to the + OS disk. Will have a default of 30GB if not provided + format: int32 + type: integer + managedDisk: + description: ManagedDisk specifies the Managed Disk parameters + for the OS disk. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk + encryption options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + type: object + osType: + type: string + required: + - osType + type: object + securityProfile: + description: SecurityProfile specifies the Security profile settings + for a virtual machine. + properties: + encryptionAtHost: + description: This field indicates whether Host Encryption + should be enabled or disabled for a virtual machine or virtual + machine scale set. Default is disabled. + type: boolean + type: object + spotVMOptions: + description: SpotVMOptions allows the ability to specify the Machine + should use a Spot VM + properties: + maxPrice: + anyOf: + - type: integer + - type: string + description: MaxPrice defines the maximum price the user is + willing to pay for Spot VM instances + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + sshPublicKey: + description: SSHPublicKey is the SSH public key string base64 + encoded to add to a Virtual Machine + type: string + subnetName: + description: SubnetName selects the Subnet where the VMSS will + be placed + type: string + terminateNotificationTimeout: + description: TerminateNotificationTimeout enables or disables + VMSS scheduled events termination notification with specified + timeout allowed values are between 5 and 15 (mins) + type: integer + vmSize: + description: VMSize is the size of the Virtual Machine to build. + See https://docs.microsoft.com/en-us/rest/api/compute/virtualmachines/createorupdate#virtualmachinesizetypes + type: string + required: + - osDisk + - sshPublicKey + - vmSize + type: object + userAssignedIdentities: + description: UserAssignedIdentities is a list of standalone Azure + identities provided by the user The lifecycle of a user-assigned + identity is managed separately from the lifecycle of the AzureMachinePool. + See https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-manage-ua-identity-cli + items: + description: UserAssignedIdentity defines the user-assigned identities + provided by the user to be assigned to Azure resources. + properties: + providerID: + description: 'ProviderID is the identification ID of the user-assigned + Identity, the format of an identity is: ''azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}''' + type: string + required: + - providerID + type: object + type: array + required: + - location + - template + type: object + status: + description: AzureMachinePoolStatus defines the observed state of AzureMachinePool. + properties: + conditions: + description: Conditions defines current service state of the AzureMachinePool. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + failureMessage: + description: "FailureMessage will be set in the event that there is + a terminal problem reconciling the MachinePool and will contain + a more verbose string suitable for logging and human consumption. + \n This field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the MachinePool's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of MachinePools can be added as + events to the MachinePool object and/or logged in the controller's + output." + type: string + failureReason: + description: "FailureReason will be set in the event that there is + a terminal problem reconciling the MachinePool and will contain + a succinct value suitable for machine interpretation. \n This field + should not be set for transitive errors that a controller faces + that are expected to be fixed automatically over time (like service + outages), but instead indicate that something is fundamentally wrong + with the MachinePool's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of MachinePools can be added as + events to the MachinePool object and/or logged in the controller's + output." + type: string + image: + description: Image is the current image used in the AzureMachinePool. + When the spec image is nil, this image is populated with the details + of the defaulted Azure Marketplace "capi" offer. + properties: + id: + description: ID specifies an image to use by ID + type: string + marketplace: + description: Marketplace specifies an image to use from the Azure + Marketplace + properties: + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization that + created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such as + a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter + minLength: 1 + type: string + thirdPartyImage: + default: false + description: ThirdPartyImage indicates the image is published + by a third party publisher and a Plan will be generated + for it. + type: boolean + version: + description: Version specifies the version of an image sku. + The allowed formats are Major.Minor.Build or 'latest'. Major, + Minor, and Build are decimal numbers. Specify 'latest' to + use the latest version of an image available at deploy time. + Even if you use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes available. + minLength: 1 + type: string + required: + - offer + - publisher + - sku + - version + type: object + sharedGallery: + description: SharedGallery specifies an image to use from an Azure + Shared Image Gallery + properties: + gallery: + description: Gallery specifies the name of the shared image + gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer This value will be used to add a `Plan` in + the API request when creating the VM/VMSS resource. This + is needed when the source image from which this SIG image + was built requires the `Plan` to be used. + type: string + publisher: + description: Publisher is the name of the organization that + created the image. This value will be used to add a `Plan` + in the API request when creating the VM/VMSS resource. This + is needed when the source image from which this SIG image + was built requires the `Plan` to be used. + type: string + resourceGroup: + description: ResourceGroup specifies the resource group containing + the shared image gallery + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such as + a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter This value will be used to add a `Plan` + in the API request when creating the VM/VMSS resource. This + is needed when the source image from which this SIG image + was built requires the `Plan` to be used. + type: string + subscriptionID: + description: SubscriptionID is the identifier of the subscription + that contains the shared image gallery + minLength: 1 + type: string + version: + description: Version specifies the version of the marketplace + image. The allowed formats are Major.Minor.Build or 'latest'. + Major, Minor, and Build are decimal numbers. Specify 'latest' + to use the latest version of an image available at deploy + time. Even if you use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes available. + minLength: 1 + type: string + required: + - gallery + - name + - resourceGroup + - subscriptionID + - version + type: object + type: object + instances: + description: Instances is the VM instance status for each VM in the + VMSS + items: + description: AzureMachinePoolInstanceStatus provides status information + for each instance in the VMSS. + properties: + instanceID: + description: InstanceID is the identification of the Machine + Instance within the VMSS + type: string + instanceName: + description: InstanceName is the name of the Machine Instance + within the VMSS + type: string + latestModelApplied: + description: LatestModelApplied indicates the instance is running + the most up-to-date VMSS model. A VMSS model describes the + image version the VM is running. If the instance is not running + the latest model, it means the instance may not be running + the version of Kubernetes the Machine Pool has specified and + needs to be updated. + type: boolean + providerID: + description: ProviderID is the provider identification of the + VMSS Instance + type: string + provisioningState: + description: ProvisioningState is the provisioning state of + the Azure virtual machine instance. + type: string + version: + description: Version defines the Kubernetes version for the + VM Instance + type: string + required: + - latestModelApplied + type: object + type: array + longRunningOperationStates: + description: LongRunningOperationStates saves the state for Azure + long-running operations so they can be continued on the next reconciliation + loop. + items: + description: Future contains the data needed for an Azure long-running + operation to continue across reconcile loops. + properties: + data: + description: Data is the base64 url encoded json Azure AutoRest + Future. + type: string + name: + description: Name is the name of the Azure resource. Together + with the service name, this forms the unique identifier for + the future. + type: string + resourceGroup: + description: ResourceGroup is the Azure resource group for the + resource. + type: string + serviceName: + description: ServiceName is the name of the Azure service. Together + with the name of the resource, this forms the unique identifier + for the future. + type: string + type: + description: Type describes the type of future, such as update, + create, delete, etc. + type: string + required: + - name + - serviceName + - type + type: object + type: array + provisioningState: + description: ProvisioningState is the provisioning state of the Azure + virtual machine. + type: string + ready: + description: Ready is true when the provider resource is ready. + type: boolean + replicas: + description: Replicas is the most recently observed number of replicas. + format: int32 + type: integer + version: + description: Version is the Kubernetes version for the current VMSS + model + type: string + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: AzureMachinePool replicas count + jsonPath: .status.replicas + name: Replicas + type: string + - description: AzureMachinePool replicas count + jsonPath: .status.ready + name: Ready + type: string + - description: Azure VMSS provisioning state + jsonPath: .status.provisioningState + name: State + type: string + - description: Cluster to which this AzureMachinePool belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + priority: 1 + type: string + - description: MachinePool object to which this AzureMachinePool belongs + jsonPath: .metadata.ownerReferences[?(@.kind=="MachinePool")].name + name: MachinePool + priority: 1 + type: string + - description: Azure VMSS ID + jsonPath: .spec.providerID + name: VMSS ID + priority: 1 + type: string + - description: Azure VM Size + jsonPath: .spec.template.vmSize + name: VM Size + priority: 1 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: AzureMachinePool is the Schema for the azuremachinepools API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureMachinePoolSpec defines the desired state of AzureMachinePool. + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to an + instance, in addition to the ones added by default by the Azure + provider. If both the AzureCluster and the AzureMachine specify + the same tag name with different values, the AzureMachine's value + takes precedence. + type: object + identity: + default: None + description: Identity is the type of identity used for the Virtual + Machine Scale Set. The type 'SystemAssigned' is an implicitly created + identity. The generated identity will be assigned a Subscription + contributor role. The type 'UserAssigned' is a standalone Azure + resource provided by the user and assigned to the VM + enum: + - None + - SystemAssigned + - UserAssigned + type: string + location: + description: Location is the Azure region location e.g. westus2 + type: string + nodeDrainTimeout: + description: 'NodeDrainTimeout is the total amount of time that the + controller will spend on draining a node. The default value is 0, + meaning that the node can be drained without any time limitations. + NOTE: NodeDrainTimeout is different from `kubectl drain --timeout`' + type: string + providerID: + description: ProviderID is the identification ID of the Virtual Machine + Scale Set + type: string + providerIDList: + description: ProviderIDList are the identification IDs of machine + instances provided by the provider. This field must match the provider + IDs as seen on the node objects corresponding to a machine pool's + machine instances. + items: + type: string + type: array + roleAssignmentName: + description: RoleAssignmentName is the name of the role assignment + to create for a system assigned identity. It can be any valid GUID. + If not specified, a random GUID will be generated. + type: string + strategy: + default: + rollingUpdate: + deletePolicy: Oldest + maxSurge: 1 + maxUnavailable: 0 + type: RollingUpdate + description: The deployment strategy to use to replace existing AzureMachinePoolMachines + with new ones. + properties: + rollingUpdate: + description: Rolling update config params. Present only if MachineDeploymentStrategyType + = RollingUpdate. + properties: + deletePolicy: + default: Oldest + description: DeletePolicy defines the policy used by the MachineDeployment + to identify nodes to delete when downscaling. Valid values + are "Random, "Newest", "Oldest" When no value is supplied, + the default is Oldest + enum: + - Random + - Newest + - Oldest + type: string + maxSurge: + anyOf: + - type: integer + - type: string + default: 1 + description: 'The maximum number of machines that can be scheduled + above the desired number of machines. Value can be an absolute + number (ex: 5) or a percentage of desired machines (ex: + 10%). This can not be 0 if MaxUnavailable is 0. Absolute + number is calculated from percentage by rounding up. Defaults + to 1. Example: when this is set to 30%, the new MachineSet + can be scaled up immediately when the rolling update starts, + such that the total number of old and new machines do not + exceed 130% of desired machines. Once old machines have + been killed, new MachineSet can be scaled up further, ensuring + that total number of machines running at any time during + the update is at most 130% of desired machines.' + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + default: 0 + description: 'The maximum number of machines that can be unavailable + during the update. Value can be an absolute number (ex: + 5) or a percentage of desired machines (ex: 10%). Absolute + number is calculated from percentage by rounding down. This + can not be 0 if MaxSurge is 0. Defaults to 0. Example: when + this is set to 30%, the old MachineSet can be scaled down + to 70% of desired machines immediately when the rolling + update starts. Once new machines are ready, old MachineSet + can be scaled down further, followed by scaling up the new + MachineSet, ensuring that the total number of machines available + at all times during the update is at least 70% of desired + machines.' + x-kubernetes-int-or-string: true + type: object + type: + default: RollingUpdate + description: Type of deployment. Currently the only supported + strategy is RollingUpdate + enum: + - RollingUpdate + type: string + type: object + template: + description: Template contains the details used to build a replica + virtual machine within the Machine Pool + properties: + acceleratedNetworking: + description: AcceleratedNetworking enables or disables Azure accelerated + networking. If omitted, it will be set based on whether the + requested VMSize supports accelerated networking. If AcceleratedNetworking + is set to true with a VMSize that does not support it, Azure + will return an error. + type: boolean + dataDisks: + description: DataDisks specifies the list of data disks to be + created for a Virtual Machine + items: + description: DataDisk specifies the parameters that are used + to add one or more data disks to the machine. + properties: + cachingType: + description: CachingType specifies the caching requirements. + enum: + - None + - ReadOnly + - ReadWrite + type: string + diskSizeGB: + description: DiskSizeGB is the size in GB to assign to the + data disk. + format: int32 + type: integer + lun: + description: Lun Specifies the logical unit number of the + data disk. This value is used to identify data disks within + the VM and therefore must be unique for each data disk + attached to a VM. The value must be between 0 and 63. + format: int32 + type: integer + managedDisk: + description: ManagedDisk specifies the Managed Disk parameters + for the data disk. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk + encryption options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + type: object + nameSuffix: + description: NameSuffix is the suffix to be appended to + the machine name to generate the disk name. Each disk + name will be in format _. + type: string + required: + - diskSizeGB + - nameSuffix + type: object + type: array + image: + description: Image is used to provide details of an image to use + during VM creation. If image details are omitted the image will + default the Azure Marketplace "capi" offer, which is based on + Ubuntu. + properties: + computeGallery: + description: ComputeGallery specifies an image to use from + the Azure Compute Gallery + properties: + gallery: + description: Gallery specifies the name of the compute + image gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + plan: + description: Plan contains plan information. + properties: + offer: + description: Offer specifies the name of a group of + related images created by the publisher. For example, + UbuntuServer, WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization + that created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, + such as a major release of a distribution. For example, + 18.04-LTS, 2019-Datacenter + minLength: 1 + type: string + required: + - offer + - publisher + - sku + type: object + resourceGroup: + description: ResourceGroup specifies the resource group + containing the private compute gallery. + type: string + subscriptionID: + description: SubscriptionID is the identifier of the subscription + that contains the private compute gallery. + type: string + version: + description: Version specifies the version of the marketplace + image. The allowed formats are Major.Minor.Build or + 'latest'. Major, Minor, and Build are decimal numbers. + Specify 'latest' to use the latest version of an image + available at deploy time. Even if you use 'latest', + the VM image will not automatically update after deploy + time even if a new version becomes available. + minLength: 1 + type: string + required: + - gallery + - name + - version + type: object + id: + description: ID specifies an image to use by ID + type: string + marketplace: + description: Marketplace specifies an image to use from the + Azure Marketplace + properties: + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization + that created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such + as a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter + minLength: 1 + type: string + thirdPartyImage: + default: false + description: ThirdPartyImage indicates the image is published + by a third party publisher and a Plan will be generated + for it. + type: boolean + version: + description: Version specifies the version of an image + sku. The allowed formats are Major.Minor.Build or 'latest'. + Major, Minor, and Build are decimal numbers. Specify + 'latest' to use the latest version of an image available + at deploy time. Even if you use 'latest', the VM image + will not automatically update after deploy time even + if a new version becomes available. + minLength: 1 + type: string + required: + - offer + - publisher + - sku + - version + type: object + sharedGallery: + description: 'SharedGallery specifies an image to use from + an Azure Shared Image Gallery Deprecated: use ComputeGallery + instead.' + properties: + gallery: + description: Gallery specifies the name of the shared + image gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer This value will be used to add a `Plan` + in the API request when creating the VM/VMSS resource. + This is needed when the source image from which this + SIG image was built requires the `Plan` to be used. + type: string + publisher: + description: Publisher is the name of the organization + that created the image. This value will be used to add + a `Plan` in the API request when creating the VM/VMSS + resource. This is needed when the source image from + which this SIG image was built requires the `Plan` to + be used. + type: string + resourceGroup: + description: ResourceGroup specifies the resource group + containing the shared image gallery + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such + as a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter This value will be used to add a `Plan` + in the API request when creating the VM/VMSS resource. + This is needed when the source image from which this + SIG image was built requires the `Plan` to be used. + type: string + subscriptionID: + description: SubscriptionID is the identifier of the subscription + that contains the shared image gallery + minLength: 1 + type: string + version: + description: Version specifies the version of the marketplace + image. The allowed formats are Major.Minor.Build or + 'latest'. Major, Minor, and Build are decimal numbers. + Specify 'latest' to use the latest version of an image + available at deploy time. Even if you use 'latest', + the VM image will not automatically update after deploy + time even if a new version becomes available. + minLength: 1 + type: string + required: + - gallery + - name + - resourceGroup + - subscriptionID + - version + type: object + type: object + osDisk: + description: OSDisk contains the operating system disk information + for a Virtual Machine + properties: + cachingType: + description: CachingType specifies the caching requirements. + enum: + - None + - ReadOnly + - ReadWrite + type: string + diffDiskSettings: + description: DiffDiskSettings describe ephemeral disk settings + for the os disk. + properties: + option: + description: Option enables ephemeral OS when set to "Local" + See https://docs.microsoft.com/en-us/azure/virtual-machines/ephemeral-os-disks + for full details + enum: + - Local + type: string + required: + - option + type: object + diskSizeGB: + description: DiskSizeGB is the size in GB to assign to the + OS disk. Will have a default of 30GB if not provided + format: int32 + type: integer + managedDisk: + description: ManagedDisk specifies the Managed Disk parameters + for the OS disk. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk + encryption options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + type: object + osType: + type: string + required: + - osType + type: object + securityProfile: + description: SecurityProfile specifies the Security profile settings + for a virtual machine. + properties: + encryptionAtHost: + description: This field indicates whether Host Encryption + should be enabled or disabled for a virtual machine or virtual + machine scale set. Default is disabled. + type: boolean + type: object + spotVMOptions: + description: SpotVMOptions allows the ability to specify the Machine + should use a Spot VM + properties: + maxPrice: + anyOf: + - type: integer + - type: string + description: MaxPrice defines the maximum price the user is + willing to pay for Spot VM instances + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + sshPublicKey: + description: SSHPublicKey is the SSH public key string base64 + encoded to add to a Virtual Machine + type: string + subnetName: + description: SubnetName selects the Subnet where the VMSS will + be placed + type: string + terminateNotificationTimeout: + description: TerminateNotificationTimeout enables or disables + VMSS scheduled events termination notification with specified + timeout allowed values are between 5 and 15 (mins) + type: integer + vmSize: + description: VMSize is the size of the Virtual Machine to build. + See https://docs.microsoft.com/en-us/rest/api/compute/virtualmachines/createorupdate#virtualmachinesizetypes + type: string + required: + - osDisk + - sshPublicKey + - vmSize + type: object + userAssignedIdentities: + description: UserAssignedIdentities is a list of standalone Azure + identities provided by the user The lifecycle of a user-assigned + identity is managed separately from the lifecycle of the AzureMachinePool. + See https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-manage-ua-identity-cli + items: + description: UserAssignedIdentity defines the user-assigned identities + provided by the user to be assigned to Azure resources. + properties: + providerID: + description: 'ProviderID is the identification ID of the user-assigned + Identity, the format of an identity is: ''azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}''' + type: string + required: + - providerID + type: object + type: array + required: + - location + - template + type: object + status: + description: AzureMachinePoolStatus defines the observed state of AzureMachinePool. + properties: + conditions: + description: Conditions defines current service state of the AzureMachinePool. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + failureMessage: + description: "FailureMessage will be set in the event that there is + a terminal problem reconciling the MachinePool and will contain + a more verbose string suitable for logging and human consumption. + \n This field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the MachinePool's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of MachinePools can be added as + events to the MachinePool object and/or logged in the controller's + output." + type: string + failureReason: + description: "FailureReason will be set in the event that there is + a terminal problem reconciling the MachinePool and will contain + a succinct value suitable for machine interpretation. \n This field + should not be set for transitive errors that a controller faces + that are expected to be fixed automatically over time (like service + outages), but instead indicate that something is fundamentally wrong + with the MachinePool's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of MachinePools can be added as + events to the MachinePool object and/or logged in the controller's + output." + type: string + image: + description: Image is the current image used in the AzureMachinePool. + When the spec image is nil, this image is populated with the details + of the defaulted Azure Marketplace "capi" offer. + properties: + computeGallery: + description: ComputeGallery specifies an image to use from the + Azure Compute Gallery + properties: + gallery: + description: Gallery specifies the name of the compute image + gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + plan: + description: Plan contains plan information. + properties: + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization + that created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such + as a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter + minLength: 1 + type: string + required: + - offer + - publisher + - sku + type: object + resourceGroup: + description: ResourceGroup specifies the resource group containing + the private compute gallery. + type: string + subscriptionID: + description: SubscriptionID is the identifier of the subscription + that contains the private compute gallery. + type: string + version: + description: Version specifies the version of the marketplace + image. The allowed formats are Major.Minor.Build or 'latest'. + Major, Minor, and Build are decimal numbers. Specify 'latest' + to use the latest version of an image available at deploy + time. Even if you use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes available. + minLength: 1 + type: string + required: + - gallery + - name + - version + type: object + id: + description: ID specifies an image to use by ID + type: string + marketplace: + description: Marketplace specifies an image to use from the Azure + Marketplace + properties: + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization that + created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such as + a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter + minLength: 1 + type: string + thirdPartyImage: + default: false + description: ThirdPartyImage indicates the image is published + by a third party publisher and a Plan will be generated + for it. + type: boolean + version: + description: Version specifies the version of an image sku. + The allowed formats are Major.Minor.Build or 'latest'. Major, + Minor, and Build are decimal numbers. Specify 'latest' to + use the latest version of an image available at deploy time. + Even if you use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes available. + minLength: 1 + type: string + required: + - offer + - publisher + - sku + - version + type: object + sharedGallery: + description: 'SharedGallery specifies an image to use from an + Azure Shared Image Gallery Deprecated: use ComputeGallery instead.' + properties: + gallery: + description: Gallery specifies the name of the shared image + gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer This value will be used to add a `Plan` in + the API request when creating the VM/VMSS resource. This + is needed when the source image from which this SIG image + was built requires the `Plan` to be used. + type: string + publisher: + description: Publisher is the name of the organization that + created the image. This value will be used to add a `Plan` + in the API request when creating the VM/VMSS resource. This + is needed when the source image from which this SIG image + was built requires the `Plan` to be used. + type: string + resourceGroup: + description: ResourceGroup specifies the resource group containing + the shared image gallery + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such as + a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter This value will be used to add a `Plan` + in the API request when creating the VM/VMSS resource. This + is needed when the source image from which this SIG image + was built requires the `Plan` to be used. + type: string + subscriptionID: + description: SubscriptionID is the identifier of the subscription + that contains the shared image gallery + minLength: 1 + type: string + version: + description: Version specifies the version of the marketplace + image. The allowed formats are Major.Minor.Build or 'latest'. + Major, Minor, and Build are decimal numbers. Specify 'latest' + to use the latest version of an image available at deploy + time. Even if you use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes available. + minLength: 1 + type: string + required: + - gallery + - name + - resourceGroup + - subscriptionID + - version + type: object + type: object + instances: + description: Instances is the VM instance status for each VM in the + VMSS + items: + description: AzureMachinePoolInstanceStatus provides status information + for each instance in the VMSS. + properties: + instanceID: + description: InstanceID is the identification of the Machine + Instance within the VMSS + type: string + instanceName: + description: InstanceName is the name of the Machine Instance + within the VMSS + type: string + latestModelApplied: + description: LatestModelApplied indicates the instance is running + the most up-to-date VMSS model. A VMSS model describes the + image version the VM is running. If the instance is not running + the latest model, it means the instance may not be running + the version of Kubernetes the Machine Pool has specified and + needs to be updated. + type: boolean + providerID: + description: ProviderID is the provider identification of the + VMSS Instance + type: string + provisioningState: + description: ProvisioningState is the provisioning state of + the Azure virtual machine instance. + type: string + version: + description: Version defines the Kubernetes version for the + VM Instance + type: string + required: + - latestModelApplied + type: object + type: array + longRunningOperationStates: + description: LongRunningOperationStates saves the state for Azure + long-running operations so they can be continued on the next reconciliation + loop. + items: + description: Future contains the data needed for an Azure long-running + operation to continue across reconcile loops. + properties: + data: + description: Data is the base64 url encoded json Azure AutoRest + Future. + type: string + name: + description: Name is the name of the Azure resource. Together + with the service name, this forms the unique identifier for + the future. + type: string + resourceGroup: + description: ResourceGroup is the Azure resource group for the + resource. + type: string + serviceName: + description: ServiceName is the name of the Azure service. Together + with the name of the resource, this forms the unique identifier + for the future. + type: string + type: + description: Type describes the type of future, such as update, + create, delete, etc. + type: string + required: + - data + - name + - serviceName + - type + type: object + type: array + provisioningState: + description: ProvisioningState is the provisioning state of the Azure + virtual machine. + type: string + ready: + description: Ready is true when the provider resource is ready. + type: boolean + replicas: + description: Replicas is the most recently observed number of replicas. + format: int32 + type: integer + version: + description: Version is the Kubernetes version for the current VMSS + model + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-azure + cluster.x-k8s.io/v1beta1: v1beta1 + name: azuremachines.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureMachine + listKind: AzureMachineList + plural: azuremachines + singular: azuremachine + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: AzureMachine ready status + jsonPath: .status.ready + name: Ready + type: string + - description: Azure VM provisioning state + jsonPath: .status.vmState + name: State + type: string + - description: Cluster to which this AzureMachine belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + priority: 1 + type: string + - description: Machine object to which this AzureMachine belongs + jsonPath: .metadata.ownerReferences[?(@.kind=="Machine")].name + name: Machine + priority: 1 + type: string + - description: Azure VM ID + jsonPath: .spec.providerID + name: VM ID + priority: 1 + type: string + - description: Azure VM Size + jsonPath: .spec.vmSize + name: VM Size + priority: 1 + type: string + name: v1alpha3 + schema: + openAPIV3Schema: + description: AzureMachine is the Schema for the azuremachines API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureMachineSpec defines the desired state of AzureMachine. + properties: + acceleratedNetworking: + description: AcceleratedNetworking enables or disables Azure accelerated + networking. If omitted, it will be set based on whether the requested + VMSize supports accelerated networking. If AcceleratedNetworking + is set to true with a VMSize that does not support it, Azure will + return an error. + type: boolean + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to an + instance, in addition to the ones added by default by the Azure + provider. If both the AzureCluster and the AzureMachine specify + the same tag name with different values, the AzureMachine's value + takes precedence. + type: object + allocatePublicIP: + description: AllocatePublicIP allows the ability to create dynamic + public ips for machines where this value is true. + type: boolean + availabilityZone: + description: 'Deprecated: use FailureDomain instead' + properties: + enabled: + type: boolean + id: + type: string + type: object + dataDisks: + description: DataDisk specifies the parameters that are used to add + one or more data disks to the machine + items: + description: DataDisk specifies the parameters that are used to + add one or more data disks to the machine. + properties: + cachingType: + type: string + diskSizeGB: + description: DiskSizeGB is the size in GB to assign to the data + disk. + format: int32 + type: integer + lun: + description: Lun Specifies the logical unit number of the data + disk. This value is used to identify data disks within the + VM and therefore must be unique for each data disk attached + to a VM. The value must be between 0 and 63. + format: int32 + type: integer + managedDisk: + description: ManagedDisk defines the managed disk options for + a VM. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk encryption + options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + required: + - storageAccountType + type: object + nameSuffix: + description: NameSuffix is the suffix to be appended to the + machine name to generate the disk name. Each disk name will + be in format _. + type: string + required: + - diskSizeGB + - nameSuffix + type: object + type: array + enableIPForwarding: + description: EnableIPForwarding enables IP Forwarding in Azure which + is required for some CNI's to send traffic from a pods on one machine + to another. This is required for IpV6 with Calico in combination + with User Defined Routes (set by the Azure Cloud Controller manager). + Default is false for disabled. + type: boolean + failureDomain: + description: FailureDomain is the failure domain unique identifier + this Machine should be attached to, as defined in Cluster API. This + relates to an Azure Availability Zone + type: string + identity: + default: None + description: Identity is the type of identity used for the virtual + machine. The type 'SystemAssigned' is an implicitly created identity. + The generated identity will be assigned a Subscription contributor + role. The type 'UserAssigned' is a standalone Azure resource provided + by the user and assigned to the VM + enum: + - None + - SystemAssigned + - UserAssigned + type: string + image: + description: Image is used to provide details of an image to use during + VM creation. If image details are omitted the image will default + the Azure Marketplace "capi" offer, which is based on Ubuntu. + properties: + id: + description: ID specifies an image to use by ID + type: string + marketplace: + description: Marketplace specifies an image to use from the Azure + Marketplace + properties: + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization that + created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such as + a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter + minLength: 1 + type: string + thirdPartyImage: + default: false + description: ThirdPartyImage indicates the image is published + by a third party publisher and a Plan will be generated + for it. + type: boolean + version: + description: Version specifies the version of an image sku. + The allowed formats are Major.Minor.Build or 'latest'. Major, + Minor, and Build are decimal numbers. Specify 'latest' to + use the latest version of an image available at deploy time. + Even if you use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes available. + minLength: 1 + type: string + required: + - offer + - publisher + - sku + - version + type: object + sharedGallery: + description: SharedGallery specifies an image to use from an Azure + Shared Image Gallery + properties: + gallery: + description: Gallery specifies the name of the shared image + gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + resourceGroup: + description: ResourceGroup specifies the resource group containing + the shared image gallery + minLength: 1 + type: string + subscriptionID: + description: SubscriptionID is the identifier of the subscription + that contains the shared image gallery + minLength: 1 + type: string + version: + description: Version specifies the version of the marketplace + image. The allowed formats are Major.Minor.Build or 'latest'. + Major, Minor, and Build are decimal numbers. Specify 'latest' + to use the latest version of an image available at deploy + time. Even if you use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes available. + minLength: 1 + type: string + required: + - gallery + - name + - resourceGroup + - subscriptionID + - version + type: object + type: object + location: + description: 'Deprecated: to support old clients, will be removed + in v1alpha4/v1beta1' + type: string + osDisk: + description: OSDisk specifies the parameters for the operating system + disk of the machine + properties: + cachingType: + type: string + diffDiskSettings: + description: DiffDiskSettings describe ephemeral disk settings + for the os disk. + properties: + option: + description: Option enables ephemeral OS when set to "Local" + See https://docs.microsoft.com/en-us/azure/virtual-machines/ephemeral-os-disks + for full details + enum: + - Local + type: string + required: + - option + type: object + diskSizeGB: + format: int32 + type: integer + managedDisk: + description: ManagedDisk defines the managed disk options for + a VM. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk encryption + options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + required: + - storageAccountType + type: object + osType: + type: string + required: + - diskSizeGB + - managedDisk + - osType + type: object + providerID: + description: ProviderID is the unique identifier as specified by the + cloud provider. + type: string + roleAssignmentName: + description: RoleAssignmentName is the name of the role assignment + to create for a system assigned identity. It can be any valid GUID. + If not specified, a random GUID will be generated. + type: string + securityProfile: + description: SecurityProfile specifies the Security profile settings + for a virtual machine. + properties: + encryptionAtHost: + description: This field indicates whether Host Encryption should + be enabled or disabled for a virtual machine or virtual machine + scale set. Default is disabled. + type: boolean + type: object + spotVMOptions: + description: SpotVMOptions allows the ability to specify the Machine + should use a Spot VM. + properties: + maxPrice: + anyOf: + - type: integer + - type: string + description: MaxPrice defines the maximum price the user is willing + to pay for Spot VM instances + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + sshPublicKey: + type: string + userAssignedIdentities: + description: UserAssignedIdentities is a list of standalone Azure + identities provided by the user The lifecycle of a user-assigned + identity is managed separately from the lifecycle of the AzureMachine. + See https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-manage-ua-identity-cli + items: + description: UserAssignedIdentity defines the user-assigned identities + provided by the user to be assigned to Azure resources. + properties: + providerID: + description: 'ProviderID is the identification ID of the user-assigned + Identity, the format of an identity is: ''azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}''' + type: string + required: + - providerID + type: object + type: array + vmSize: + type: string + required: + - location + - osDisk + - sshPublicKey + - vmSize + type: object + status: + description: AzureMachineStatus defines the observed state of AzureMachine. + properties: + addresses: + description: Addresses contains the Azure instance associated addresses. + items: + description: NodeAddress contains information for the node's address. + properties: + address: + description: The node address. + type: string + type: + description: Node address type, one of Hostname, ExternalIP + or InternalIP. + type: string + required: + - address + - type + type: object + type: array + conditions: + description: Conditions defines current service state of the AzureMachine. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + failureMessage: + description: "ErrorMessage will be set in the event that there is + a terminal problem reconciling the Machine and will contain a more + verbose string suitable for logging and human consumption. \n This + field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the Machine's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of Machines can be added as events + to the Machine object and/or logged in the controller's output." + type: string + failureReason: + description: "ErrorReason will be set in the event that there is a + terminal problem reconciling the Machine and will contain a succinct + value suitable for machine interpretation. \n This field should + not be set for transitive errors that a controller faces that are + expected to be fixed automatically over time (like service outages), + but instead indicate that something is fundamentally wrong with + the Machine's spec or the configuration of the controller, and that + manual intervention is required. Examples of terminal errors would + be invalid combinations of settings in the spec, values that are + unsupported by the controller, or the responsible controller itself + being critically misconfigured. \n Any transient errors that occur + during the reconciliation of Machines can be added as events to + the Machine object and/or logged in the controller's output." + type: string + ready: + description: Ready is true when the provider resource is ready. + type: boolean + vmState: + description: VMState is the provisioning state of the Azure virtual + machine. + type: string + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: AzureMachine ready status + jsonPath: .status.ready + name: Ready + type: string + - description: Azure VM provisioning state + jsonPath: .status.vmState + name: State + type: string + - description: Cluster to which this AzureMachine belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + priority: 1 + type: string + - description: Machine object to which this AzureMachine belongs + jsonPath: .metadata.ownerReferences[?(@.kind=="Machine")].name + name: Machine + priority: 1 + type: string + - description: Azure VM ID + jsonPath: .spec.providerID + name: VM ID + priority: 1 + type: string + - description: Azure VM Size + jsonPath: .spec.vmSize + name: VM Size + priority: 1 + type: string + name: v1alpha4 + schema: + openAPIV3Schema: + description: AzureMachine is the Schema for the azuremachines API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureMachineSpec defines the desired state of AzureMachine. + properties: + acceleratedNetworking: + description: AcceleratedNetworking enables or disables Azure accelerated + networking. If omitted, it will be set based on whether the requested + VMSize supports accelerated networking. If AcceleratedNetworking + is set to true with a VMSize that does not support it, Azure will + return an error. + type: boolean + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to an + instance, in addition to the ones added by default by the Azure + provider. If both the AzureCluster and the AzureMachine specify + the same tag name with different values, the AzureMachine's value + takes precedence. + type: object + allocatePublicIP: + description: AllocatePublicIP allows the ability to create dynamic + public ips for machines where this value is true. + type: boolean + dataDisks: + description: DataDisk specifies the parameters that are used to add + one or more data disks to the machine + items: + description: DataDisk specifies the parameters that are used to + add one or more data disks to the machine. + properties: + cachingType: + description: CachingType specifies the caching requirements. + enum: + - None + - ReadOnly + - ReadWrite + type: string + diskSizeGB: + description: DiskSizeGB is the size in GB to assign to the data + disk. + format: int32 + type: integer + lun: + description: Lun Specifies the logical unit number of the data + disk. This value is used to identify data disks within the + VM and therefore must be unique for each data disk attached + to a VM. The value must be between 0 and 63. + format: int32 + type: integer + managedDisk: + description: ManagedDisk specifies the Managed Disk parameters + for the data disk. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk encryption + options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + type: object + nameSuffix: + description: NameSuffix is the suffix to be appended to the + machine name to generate the disk name. Each disk name will + be in format _. + type: string + required: + - diskSizeGB + - nameSuffix + type: object + type: array + enableIPForwarding: + description: EnableIPForwarding enables IP Forwarding in Azure which + is required for some CNI's to send traffic from a pods on one machine + to another. This is required for IpV6 with Calico in combination + with User Defined Routes (set by the Azure Cloud Controller manager). + Default is false for disabled. + type: boolean + failureDomain: + description: FailureDomain is the failure domain unique identifier + this Machine should be attached to, as defined in Cluster API. This + relates to an Azure Availability Zone + type: string + identity: + default: None + description: Identity is the type of identity used for the virtual + machine. The type 'SystemAssigned' is an implicitly created identity. + The generated identity will be assigned a Subscription contributor + role. The type 'UserAssigned' is a standalone Azure resource provided + by the user and assigned to the VM + enum: + - None + - SystemAssigned + - UserAssigned + type: string + image: + description: Image is used to provide details of an image to use during + VM creation. If image details are omitted the image will default + the Azure Marketplace "capi" offer, which is based on Ubuntu. + properties: + id: + description: ID specifies an image to use by ID + type: string + marketplace: + description: Marketplace specifies an image to use from the Azure + Marketplace + properties: + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization that + created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such as + a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter + minLength: 1 + type: string + thirdPartyImage: + default: false + description: ThirdPartyImage indicates the image is published + by a third party publisher and a Plan will be generated + for it. + type: boolean + version: + description: Version specifies the version of an image sku. + The allowed formats are Major.Minor.Build or 'latest'. Major, + Minor, and Build are decimal numbers. Specify 'latest' to + use the latest version of an image available at deploy time. + Even if you use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes available. + minLength: 1 + type: string + required: + - offer + - publisher + - sku + - version + type: object + sharedGallery: + description: SharedGallery specifies an image to use from an Azure + Shared Image Gallery + properties: + gallery: + description: Gallery specifies the name of the shared image + gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer This value will be used to add a `Plan` in + the API request when creating the VM/VMSS resource. This + is needed when the source image from which this SIG image + was built requires the `Plan` to be used. + type: string + publisher: + description: Publisher is the name of the organization that + created the image. This value will be used to add a `Plan` + in the API request when creating the VM/VMSS resource. This + is needed when the source image from which this SIG image + was built requires the `Plan` to be used. + type: string + resourceGroup: + description: ResourceGroup specifies the resource group containing + the shared image gallery + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such as + a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter This value will be used to add a `Plan` + in the API request when creating the VM/VMSS resource. This + is needed when the source image from which this SIG image + was built requires the `Plan` to be used. + type: string + subscriptionID: + description: SubscriptionID is the identifier of the subscription + that contains the shared image gallery + minLength: 1 + type: string + version: + description: Version specifies the version of the marketplace + image. The allowed formats are Major.Minor.Build or 'latest'. + Major, Minor, and Build are decimal numbers. Specify 'latest' + to use the latest version of an image available at deploy + time. Even if you use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes available. + minLength: 1 + type: string + required: + - gallery + - name + - resourceGroup + - subscriptionID + - version + type: object + type: object + osDisk: + description: OSDisk specifies the parameters for the operating system + disk of the machine + properties: + cachingType: + description: CachingType specifies the caching requirements. + enum: + - None + - ReadOnly + - ReadWrite + type: string + diffDiskSettings: + description: DiffDiskSettings describe ephemeral disk settings + for the os disk. + properties: + option: + description: Option enables ephemeral OS when set to "Local" + See https://docs.microsoft.com/en-us/azure/virtual-machines/ephemeral-os-disks + for full details + enum: + - Local + type: string + required: + - option + type: object + diskSizeGB: + description: DiskSizeGB is the size in GB to assign to the OS + disk. Will have a default of 30GB if not provided + format: int32 + type: integer + managedDisk: + description: ManagedDisk specifies the Managed Disk parameters + for the OS disk. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk encryption + options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + type: object + osType: + type: string + required: + - osType + type: object + providerID: + description: ProviderID is the unique identifier as specified by the + cloud provider. + type: string + roleAssignmentName: + description: RoleAssignmentName is the name of the role assignment + to create for a system assigned identity. It can be any valid GUID. + If not specified, a random GUID will be generated. + type: string + securityProfile: + description: SecurityProfile specifies the Security profile settings + for a virtual machine. + properties: + encryptionAtHost: + description: This field indicates whether Host Encryption should + be enabled or disabled for a virtual machine or virtual machine + scale set. Default is disabled. + type: boolean + type: object + spotVMOptions: + description: SpotVMOptions allows the ability to specify the Machine + should use a Spot VM + properties: + maxPrice: + anyOf: + - type: integer + - type: string + description: MaxPrice defines the maximum price the user is willing + to pay for Spot VM instances + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + sshPublicKey: + type: string + subnetName: + description: SubnetName selects the Subnet where the VM will be placed + type: string + userAssignedIdentities: + description: UserAssignedIdentities is a list of standalone Azure + identities provided by the user The lifecycle of a user-assigned + identity is managed separately from the lifecycle of the AzureMachine. + See https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-manage-ua-identity-cli + items: + description: UserAssignedIdentity defines the user-assigned identities + provided by the user to be assigned to Azure resources. + properties: + providerID: + description: 'ProviderID is the identification ID of the user-assigned + Identity, the format of an identity is: ''azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}''' + type: string + required: + - providerID + type: object + type: array + vmSize: + type: string + required: + - osDisk + - sshPublicKey + - vmSize + type: object + status: + description: AzureMachineStatus defines the observed state of AzureMachine. + properties: + addresses: + description: Addresses contains the Azure instance associated addresses. + items: + description: NodeAddress contains information for the node's address. + properties: + address: + description: The node address. + type: string + type: + description: Node address type, one of Hostname, ExternalIP + or InternalIP. + type: string + required: + - address + - type + type: object + type: array + conditions: + description: Conditions defines current service state of the AzureMachine. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + failureMessage: + description: "ErrorMessage will be set in the event that there is + a terminal problem reconciling the Machine and will contain a more + verbose string suitable for logging and human consumption. \n This + field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the Machine's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of Machines can be added as events + to the Machine object and/or logged in the controller's output." + type: string + failureReason: + description: "ErrorReason will be set in the event that there is a + terminal problem reconciling the Machine and will contain a succinct + value suitable for machine interpretation. \n This field should + not be set for transitive errors that a controller faces that are + expected to be fixed automatically over time (like service outages), + but instead indicate that something is fundamentally wrong with + the Machine's spec or the configuration of the controller, and that + manual intervention is required. Examples of terminal errors would + be invalid combinations of settings in the spec, values that are + unsupported by the controller, or the responsible controller itself + being critically misconfigured. \n Any transient errors that occur + during the reconciliation of Machines can be added as events to + the Machine object and/or logged in the controller's output." + type: string + longRunningOperationStates: + description: LongRunningOperationStates saves the states for Azure + long-running operations so they can be continued on the next reconciliation + loop. + items: + description: Future contains the data needed for an Azure long-running + operation to continue across reconcile loops. + properties: + data: + description: Data is the base64 url encoded json Azure AutoRest + Future. + type: string + name: + description: Name is the name of the Azure resource. Together + with the service name, this forms the unique identifier for + the future. + type: string + resourceGroup: + description: ResourceGroup is the Azure resource group for the + resource. + type: string + serviceName: + description: ServiceName is the name of the Azure service. Together + with the name of the resource, this forms the unique identifier + for the future. + type: string + type: + description: Type describes the type of future, such as update, + create, delete, etc. + type: string + required: + - name + - serviceName + - type + type: object + type: array + ready: + description: Ready is true when the provider resource is ready. + type: boolean + vmState: + description: VMState is the provisioning state of the Azure virtual + machine. + type: string + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].reason + name: Reason + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].message + name: Message + priority: 1 + type: string + - description: Azure VM provisioning state + jsonPath: .status.vmState + name: State + type: string + - description: Cluster to which this AzureMachine belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + priority: 1 + type: string + - description: Machine object to which this AzureMachine belongs + jsonPath: .metadata.ownerReferences[?(@.kind=="Machine")].name + name: Machine + priority: 1 + type: string + - description: Azure VM ID + jsonPath: .spec.providerID + name: VM ID + priority: 1 + type: string + - description: Azure VM Size + jsonPath: .spec.vmSize + name: VM Size + priority: 1 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: AzureMachine is the Schema for the azuremachines API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureMachineSpec defines the desired state of AzureMachine. + properties: + acceleratedNetworking: + description: AcceleratedNetworking enables or disables Azure accelerated + networking. If omitted, it will be set based on whether the requested + VMSize supports accelerated networking. If AcceleratedNetworking + is set to true with a VMSize that does not support it, Azure will + return an error. + type: boolean + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to an + instance, in addition to the ones added by default by the Azure + provider. If both the AzureCluster and the AzureMachine specify + the same tag name with different values, the AzureMachine's value + takes precedence. + type: object + allocatePublicIP: + description: AllocatePublicIP allows the ability to create dynamic + public ips for machines where this value is true. + type: boolean + dataDisks: + description: DataDisk specifies the parameters that are used to add + one or more data disks to the machine + items: + description: DataDisk specifies the parameters that are used to + add one or more data disks to the machine. + properties: + cachingType: + description: CachingType specifies the caching requirements. + enum: + - None + - ReadOnly + - ReadWrite + type: string + diskSizeGB: + description: DiskSizeGB is the size in GB to assign to the data + disk. + format: int32 + type: integer + lun: + description: Lun Specifies the logical unit number of the data + disk. This value is used to identify data disks within the + VM and therefore must be unique for each data disk attached + to a VM. The value must be between 0 and 63. + format: int32 + type: integer + managedDisk: + description: ManagedDisk specifies the Managed Disk parameters + for the data disk. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk encryption + options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + type: object + nameSuffix: + description: NameSuffix is the suffix to be appended to the + machine name to generate the disk name. Each disk name will + be in format _. + type: string + required: + - diskSizeGB + - nameSuffix + type: object + type: array + enableIPForwarding: + description: EnableIPForwarding enables IP Forwarding in Azure which + is required for some CNI's to send traffic from a pods on one machine + to another. This is required for IpV6 with Calico in combination + with User Defined Routes (set by the Azure Cloud Controller manager). + Default is false for disabled. + type: boolean + failureDomain: + description: FailureDomain is the failure domain unique identifier + this Machine should be attached to, as defined in Cluster API. This + relates to an Azure Availability Zone + type: string + identity: + default: None + description: Identity is the type of identity used for the virtual + machine. The type 'SystemAssigned' is an implicitly created identity. + The generated identity will be assigned a Subscription contributor + role. The type 'UserAssigned' is a standalone Azure resource provided + by the user and assigned to the VM + enum: + - None + - SystemAssigned + - UserAssigned + type: string + image: + description: Image is used to provide details of an image to use during + VM creation. If image details are omitted the image will default + the Azure Marketplace "capi" offer, which is based on Ubuntu. + properties: + computeGallery: + description: ComputeGallery specifies an image to use from the + Azure Compute Gallery + properties: + gallery: + description: Gallery specifies the name of the compute image + gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + plan: + description: Plan contains plan information. + properties: + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization + that created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such + as a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter + minLength: 1 + type: string + required: + - offer + - publisher + - sku + type: object + resourceGroup: + description: ResourceGroup specifies the resource group containing + the private compute gallery. + type: string + subscriptionID: + description: SubscriptionID is the identifier of the subscription + that contains the private compute gallery. + type: string + version: + description: Version specifies the version of the marketplace + image. The allowed formats are Major.Minor.Build or 'latest'. + Major, Minor, and Build are decimal numbers. Specify 'latest' + to use the latest version of an image available at deploy + time. Even if you use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes available. + minLength: 1 + type: string + required: + - gallery + - name + - version + type: object + id: + description: ID specifies an image to use by ID + type: string + marketplace: + description: Marketplace specifies an image to use from the Azure + Marketplace + properties: + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization that + created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such as + a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter + minLength: 1 + type: string + thirdPartyImage: + default: false + description: ThirdPartyImage indicates the image is published + by a third party publisher and a Plan will be generated + for it. + type: boolean + version: + description: Version specifies the version of an image sku. + The allowed formats are Major.Minor.Build or 'latest'. Major, + Minor, and Build are decimal numbers. Specify 'latest' to + use the latest version of an image available at deploy time. + Even if you use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes available. + minLength: 1 + type: string + required: + - offer + - publisher + - sku + - version + type: object + sharedGallery: + description: 'SharedGallery specifies an image to use from an + Azure Shared Image Gallery Deprecated: use ComputeGallery instead.' + properties: + gallery: + description: Gallery specifies the name of the shared image + gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer This value will be used to add a `Plan` in + the API request when creating the VM/VMSS resource. This + is needed when the source image from which this SIG image + was built requires the `Plan` to be used. + type: string + publisher: + description: Publisher is the name of the organization that + created the image. This value will be used to add a `Plan` + in the API request when creating the VM/VMSS resource. This + is needed when the source image from which this SIG image + was built requires the `Plan` to be used. + type: string + resourceGroup: + description: ResourceGroup specifies the resource group containing + the shared image gallery + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such as + a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter This value will be used to add a `Plan` + in the API request when creating the VM/VMSS resource. This + is needed when the source image from which this SIG image + was built requires the `Plan` to be used. + type: string + subscriptionID: + description: SubscriptionID is the identifier of the subscription + that contains the shared image gallery + minLength: 1 + type: string + version: + description: Version specifies the version of the marketplace + image. The allowed formats are Major.Minor.Build or 'latest'. + Major, Minor, and Build are decimal numbers. Specify 'latest' + to use the latest version of an image available at deploy + time. Even if you use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes available. + minLength: 1 + type: string + required: + - gallery + - name + - resourceGroup + - subscriptionID + - version + type: object + type: object + osDisk: + description: OSDisk specifies the parameters for the operating system + disk of the machine + properties: + cachingType: + description: CachingType specifies the caching requirements. + enum: + - None + - ReadOnly + - ReadWrite + type: string + diffDiskSettings: + description: DiffDiskSettings describe ephemeral disk settings + for the os disk. + properties: + option: + description: Option enables ephemeral OS when set to "Local" + See https://docs.microsoft.com/en-us/azure/virtual-machines/ephemeral-os-disks + for full details + enum: + - Local + type: string + required: + - option + type: object + diskSizeGB: + description: DiskSizeGB is the size in GB to assign to the OS + disk. Will have a default of 30GB if not provided + format: int32 + type: integer + managedDisk: + description: ManagedDisk specifies the Managed Disk parameters + for the OS disk. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk encryption + options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + type: object + osType: + type: string + required: + - osType + type: object + providerID: + description: ProviderID is the unique identifier as specified by the + cloud provider. + type: string + roleAssignmentName: + description: RoleAssignmentName is the name of the role assignment + to create for a system assigned identity. It can be any valid GUID. + If not specified, a random GUID will be generated. + type: string + securityProfile: + description: SecurityProfile specifies the Security profile settings + for a virtual machine. + properties: + encryptionAtHost: + description: This field indicates whether Host Encryption should + be enabled or disabled for a virtual machine or virtual machine + scale set. Default is disabled. + type: boolean + type: object + spotVMOptions: + description: SpotVMOptions allows the ability to specify the Machine + should use a Spot VM + properties: + maxPrice: + anyOf: + - type: integer + - type: string + description: MaxPrice defines the maximum price the user is willing + to pay for Spot VM instances + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + sshPublicKey: + type: string + subnetName: + description: SubnetName selects the Subnet where the VM will be placed + type: string + userAssignedIdentities: + description: UserAssignedIdentities is a list of standalone Azure + identities provided by the user The lifecycle of a user-assigned + identity is managed separately from the lifecycle of the AzureMachine. + See https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-manage-ua-identity-cli + items: + description: UserAssignedIdentity defines the user-assigned identities + provided by the user to be assigned to Azure resources. + properties: + providerID: + description: 'ProviderID is the identification ID of the user-assigned + Identity, the format of an identity is: ''azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}''' + type: string + required: + - providerID + type: object + type: array + vmSize: + type: string + required: + - osDisk + - sshPublicKey + - vmSize + type: object + status: + description: AzureMachineStatus defines the observed state of AzureMachine. + properties: + addresses: + description: Addresses contains the Azure instance associated addresses. + items: + description: NodeAddress contains information for the node's address. + properties: + address: + description: The node address. + type: string + type: + description: Node address type, one of Hostname, ExternalIP + or InternalIP. + type: string + required: + - address + - type + type: object + type: array + conditions: + description: Conditions defines current service state of the AzureMachine. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + failureMessage: + description: "ErrorMessage will be set in the event that there is + a terminal problem reconciling the Machine and will contain a more + verbose string suitable for logging and human consumption. \n This + field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the Machine's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of Machines can be added as events + to the Machine object and/or logged in the controller's output." + type: string + failureReason: + description: "ErrorReason will be set in the event that there is a + terminal problem reconciling the Machine and will contain a succinct + value suitable for machine interpretation. \n This field should + not be set for transitive errors that a controller faces that are + expected to be fixed automatically over time (like service outages), + but instead indicate that something is fundamentally wrong with + the Machine's spec or the configuration of the controller, and that + manual intervention is required. Examples of terminal errors would + be invalid combinations of settings in the spec, values that are + unsupported by the controller, or the responsible controller itself + being critically misconfigured. \n Any transient errors that occur + during the reconciliation of Machines can be added as events to + the Machine object and/or logged in the controller's output." + type: string + longRunningOperationStates: + description: LongRunningOperationStates saves the states for Azure + long-running operations so they can be continued on the next reconciliation + loop. + items: + description: Future contains the data needed for an Azure long-running + operation to continue across reconcile loops. + properties: + data: + description: Data is the base64 url encoded json Azure AutoRest + Future. + type: string + name: + description: Name is the name of the Azure resource. Together + with the service name, this forms the unique identifier for + the future. + type: string + resourceGroup: + description: ResourceGroup is the Azure resource group for the + resource. + type: string + serviceName: + description: ServiceName is the name of the Azure service. Together + with the name of the resource, this forms the unique identifier + for the future. + type: string + type: + description: Type describes the type of future, such as update, + create, delete, etc. + type: string + required: + - data + - name + - serviceName + - type + type: object + type: array + ready: + description: Ready is true when the provider resource is ready. + type: boolean + vmState: + description: VMState is the provisioning state of the Azure virtual + machine. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-azure + cluster.x-k8s.io/v1beta1: v1beta1 + name: azuremachinetemplates.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureMachineTemplate + listKind: AzureMachineTemplateList + plural: azuremachinetemplates + singular: azuremachinetemplate + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: AzureMachineTemplate is the Schema for the azuremachinetemplates + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureMachineTemplateSpec defines the desired state of AzureMachineTemplate. + properties: + template: + description: AzureMachineTemplateResource describes the data needed + to create an AzureMachine from a template. + properties: + spec: + description: Spec is the specification of the desired behavior + of the machine. + properties: + acceleratedNetworking: + description: AcceleratedNetworking enables or disables Azure + accelerated networking. If omitted, it will be set based + on whether the requested VMSize supports accelerated networking. + If AcceleratedNetworking is set to true with a VMSize that + does not support it, Azure will return an error. + type: boolean + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to + add to an instance, in addition to the ones added by default + by the Azure provider. If both the AzureCluster and the + AzureMachine specify the same tag name with different values, + the AzureMachine's value takes precedence. + type: object + allocatePublicIP: + description: AllocatePublicIP allows the ability to create + dynamic public ips for machines where this value is true. + type: boolean + availabilityZone: + description: 'Deprecated: use FailureDomain instead' + properties: + enabled: + type: boolean + id: + type: string + type: object + dataDisks: + description: DataDisk specifies the parameters that are used + to add one or more data disks to the machine + items: + description: DataDisk specifies the parameters that are + used to add one or more data disks to the machine. + properties: + cachingType: + type: string + diskSizeGB: + description: DiskSizeGB is the size in GB to assign + to the data disk. + format: int32 + type: integer + lun: + description: Lun Specifies the logical unit number of + the data disk. This value is used to identify data + disks within the VM and therefore must be unique for + each data disk attached to a VM. The value must be + between 0 and 63. + format: int32 + type: integer + managedDisk: + description: ManagedDisk defines the managed disk options + for a VM. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines + disk encryption options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + required: + - storageAccountType + type: object + nameSuffix: + description: NameSuffix is the suffix to be appended + to the machine name to generate the disk name. Each + disk name will be in format _. + type: string + required: + - diskSizeGB + - nameSuffix + type: object + type: array + enableIPForwarding: + description: EnableIPForwarding enables IP Forwarding in Azure + which is required for some CNI's to send traffic from a + pods on one machine to another. This is required for IpV6 + with Calico in combination with User Defined Routes (set + by the Azure Cloud Controller manager). Default is false + for disabled. + type: boolean + failureDomain: + description: FailureDomain is the failure domain unique identifier + this Machine should be attached to, as defined in Cluster + API. This relates to an Azure Availability Zone + type: string + identity: + default: None + description: Identity is the type of identity used for the + virtual machine. The type 'SystemAssigned' is an implicitly + created identity. The generated identity will be assigned + a Subscription contributor role. The type 'UserAssigned' + is a standalone Azure resource provided by the user and + assigned to the VM + enum: + - None + - SystemAssigned + - UserAssigned + type: string + image: + description: Image is used to provide details of an image + to use during VM creation. If image details are omitted + the image will default the Azure Marketplace "capi" offer, + which is based on Ubuntu. + properties: + id: + description: ID specifies an image to use by ID + type: string + marketplace: + description: Marketplace specifies an image to use from + the Azure Marketplace + properties: + offer: + description: Offer specifies the name of a group of + related images created by the publisher. For example, + UbuntuServer, WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization + that created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, + such as a major release of a distribution. For example, + 18.04-LTS, 2019-Datacenter + minLength: 1 + type: string + thirdPartyImage: + default: false + description: ThirdPartyImage indicates the image is + published by a third party publisher and a Plan + will be generated for it. + type: boolean + version: + description: Version specifies the version of an image + sku. The allowed formats are Major.Minor.Build or + 'latest'. Major, Minor, and Build are decimal numbers. + Specify 'latest' to use the latest version of an + image available at deploy time. Even if you use + 'latest', the VM image will not automatically update + after deploy time even if a new version becomes + available. + minLength: 1 + type: string + required: + - offer + - publisher + - sku + - version + type: object + sharedGallery: + description: SharedGallery specifies an image to use from + an Azure Shared Image Gallery + properties: + gallery: + description: Gallery specifies the name of the shared + image gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + resourceGroup: + description: ResourceGroup specifies the resource + group containing the shared image gallery + minLength: 1 + type: string + subscriptionID: + description: SubscriptionID is the identifier of the + subscription that contains the shared image gallery + minLength: 1 + type: string + version: + description: Version specifies the version of the + marketplace image. The allowed formats are Major.Minor.Build + or 'latest'. Major, Minor, and Build are decimal + numbers. Specify 'latest' to use the latest version + of an image available at deploy time. Even if you + use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes + available. + minLength: 1 + type: string + required: + - gallery + - name + - resourceGroup + - subscriptionID + - version + type: object + type: object + location: + description: 'Deprecated: to support old clients, will be + removed in v1alpha4/v1beta1' + type: string + osDisk: + description: OSDisk specifies the parameters for the operating + system disk of the machine + properties: + cachingType: + type: string + diffDiskSettings: + description: DiffDiskSettings describe ephemeral disk + settings for the os disk. + properties: + option: + description: Option enables ephemeral OS when set + to "Local" See https://docs.microsoft.com/en-us/azure/virtual-machines/ephemeral-os-disks + for full details + enum: + - Local + type: string + required: + - option + type: object + diskSizeGB: + format: int32 + type: integer + managedDisk: + description: ManagedDisk defines the managed disk options + for a VM. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk + encryption options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + required: + - storageAccountType + type: object + osType: + type: string + required: + - diskSizeGB + - managedDisk + - osType + type: object + providerID: + description: ProviderID is the unique identifier as specified + by the cloud provider. + type: string + roleAssignmentName: + description: RoleAssignmentName is the name of the role assignment + to create for a system assigned identity. It can be any + valid GUID. If not specified, a random GUID will be generated. + type: string + securityProfile: + description: SecurityProfile specifies the Security profile + settings for a virtual machine. + properties: + encryptionAtHost: + description: This field indicates whether Host Encryption + should be enabled or disabled for a virtual machine + or virtual machine scale set. Default is disabled. + type: boolean + type: object + spotVMOptions: + description: SpotVMOptions allows the ability to specify the + Machine should use a Spot VM. + properties: + maxPrice: + anyOf: + - type: integer + - type: string + description: MaxPrice defines the maximum price the user + is willing to pay for Spot VM instances + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + sshPublicKey: + type: string + userAssignedIdentities: + description: UserAssignedIdentities is a list of standalone + Azure identities provided by the user The lifecycle of a + user-assigned identity is managed separately from the lifecycle + of the AzureMachine. See https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-manage-ua-identity-cli + items: + description: UserAssignedIdentity defines the user-assigned + identities provided by the user to be assigned to Azure + resources. + properties: + providerID: + description: 'ProviderID is the identification ID of + the user-assigned Identity, the format of an identity + is: ''azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}''' + type: string + required: + - providerID + type: object + type: array + vmSize: + type: string + required: + - location + - osDisk + - sshPublicKey + - vmSize + type: object + required: + - spec + type: object + required: + - template + type: object + type: object + served: true + storage: false + - name: v1alpha4 + schema: + openAPIV3Schema: + description: AzureMachineTemplate is the Schema for the azuremachinetemplates + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureMachineTemplateSpec defines the desired state of AzureMachineTemplate. + properties: + template: + description: AzureMachineTemplateResource describes the data needed + to create an AzureMachine from a template. + properties: + spec: + description: Spec is the specification of the desired behavior + of the machine. + properties: + acceleratedNetworking: + description: AcceleratedNetworking enables or disables Azure + accelerated networking. If omitted, it will be set based + on whether the requested VMSize supports accelerated networking. + If AcceleratedNetworking is set to true with a VMSize that + does not support it, Azure will return an error. + type: boolean + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to + add to an instance, in addition to the ones added by default + by the Azure provider. If both the AzureCluster and the + AzureMachine specify the same tag name with different values, + the AzureMachine's value takes precedence. + type: object + allocatePublicIP: + description: AllocatePublicIP allows the ability to create + dynamic public ips for machines where this value is true. + type: boolean + dataDisks: + description: DataDisk specifies the parameters that are used + to add one or more data disks to the machine + items: + description: DataDisk specifies the parameters that are + used to add one or more data disks to the machine. + properties: + cachingType: + description: CachingType specifies the caching requirements. + enum: + - None + - ReadOnly + - ReadWrite + type: string + diskSizeGB: + description: DiskSizeGB is the size in GB to assign + to the data disk. + format: int32 + type: integer + lun: + description: Lun Specifies the logical unit number of + the data disk. This value is used to identify data + disks within the VM and therefore must be unique for + each data disk attached to a VM. The value must be + between 0 and 63. + format: int32 + type: integer + managedDisk: + description: ManagedDisk specifies the Managed Disk + parameters for the data disk. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines + disk encryption options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + type: object + nameSuffix: + description: NameSuffix is the suffix to be appended + to the machine name to generate the disk name. Each + disk name will be in format _. + type: string + required: + - diskSizeGB + - nameSuffix + type: object + type: array + enableIPForwarding: + description: EnableIPForwarding enables IP Forwarding in Azure + which is required for some CNI's to send traffic from a + pods on one machine to another. This is required for IpV6 + with Calico in combination with User Defined Routes (set + by the Azure Cloud Controller manager). Default is false + for disabled. + type: boolean + failureDomain: + description: FailureDomain is the failure domain unique identifier + this Machine should be attached to, as defined in Cluster + API. This relates to an Azure Availability Zone + type: string + identity: + default: None + description: Identity is the type of identity used for the + virtual machine. The type 'SystemAssigned' is an implicitly + created identity. The generated identity will be assigned + a Subscription contributor role. The type 'UserAssigned' + is a standalone Azure resource provided by the user and + assigned to the VM + enum: + - None + - SystemAssigned + - UserAssigned + type: string + image: + description: Image is used to provide details of an image + to use during VM creation. If image details are omitted + the image will default the Azure Marketplace "capi" offer, + which is based on Ubuntu. + properties: + id: + description: ID specifies an image to use by ID + type: string + marketplace: + description: Marketplace specifies an image to use from + the Azure Marketplace + properties: + offer: + description: Offer specifies the name of a group of + related images created by the publisher. For example, + UbuntuServer, WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization + that created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, + such as a major release of a distribution. For example, + 18.04-LTS, 2019-Datacenter + minLength: 1 + type: string + thirdPartyImage: + default: false + description: ThirdPartyImage indicates the image is + published by a third party publisher and a Plan + will be generated for it. + type: boolean + version: + description: Version specifies the version of an image + sku. The allowed formats are Major.Minor.Build or + 'latest'. Major, Minor, and Build are decimal numbers. + Specify 'latest' to use the latest version of an + image available at deploy time. Even if you use + 'latest', the VM image will not automatically update + after deploy time even if a new version becomes + available. + minLength: 1 + type: string + required: + - offer + - publisher + - sku + - version + type: object + sharedGallery: + description: SharedGallery specifies an image to use from + an Azure Shared Image Gallery + properties: + gallery: + description: Gallery specifies the name of the shared + image gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + offer: + description: Offer specifies the name of a group of + related images created by the publisher. For example, + UbuntuServer, WindowsServer This value will be used + to add a `Plan` in the API request when creating + the VM/VMSS resource. This is needed when the source + image from which this SIG image was built requires + the `Plan` to be used. + type: string + publisher: + description: Publisher is the name of the organization + that created the image. This value will be used + to add a `Plan` in the API request when creating + the VM/VMSS resource. This is needed when the source + image from which this SIG image was built requires + the `Plan` to be used. + type: string + resourceGroup: + description: ResourceGroup specifies the resource + group containing the shared image gallery + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, + such as a major release of a distribution. For example, + 18.04-LTS, 2019-Datacenter This value will be used + to add a `Plan` in the API request when creating + the VM/VMSS resource. This is needed when the source + image from which this SIG image was built requires + the `Plan` to be used. + type: string + subscriptionID: + description: SubscriptionID is the identifier of the + subscription that contains the shared image gallery + minLength: 1 + type: string + version: + description: Version specifies the version of the + marketplace image. The allowed formats are Major.Minor.Build + or 'latest'. Major, Minor, and Build are decimal + numbers. Specify 'latest' to use the latest version + of an image available at deploy time. Even if you + use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes + available. + minLength: 1 + type: string + required: + - gallery + - name + - resourceGroup + - subscriptionID + - version + type: object + type: object + osDisk: + description: OSDisk specifies the parameters for the operating + system disk of the machine + properties: + cachingType: + description: CachingType specifies the caching requirements. + enum: + - None + - ReadOnly + - ReadWrite + type: string + diffDiskSettings: + description: DiffDiskSettings describe ephemeral disk + settings for the os disk. + properties: + option: + description: Option enables ephemeral OS when set + to "Local" See https://docs.microsoft.com/en-us/azure/virtual-machines/ephemeral-os-disks + for full details + enum: + - Local + type: string + required: + - option + type: object + diskSizeGB: + description: DiskSizeGB is the size in GB to assign to + the OS disk. Will have a default of 30GB if not provided + format: int32 + type: integer + managedDisk: + description: ManagedDisk specifies the Managed Disk parameters + for the OS disk. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk + encryption options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + type: object + osType: + type: string + required: + - osType + type: object + providerID: + description: ProviderID is the unique identifier as specified + by the cloud provider. + type: string + roleAssignmentName: + description: RoleAssignmentName is the name of the role assignment + to create for a system assigned identity. It can be any + valid GUID. If not specified, a random GUID will be generated. + type: string + securityProfile: + description: SecurityProfile specifies the Security profile + settings for a virtual machine. + properties: + encryptionAtHost: + description: This field indicates whether Host Encryption + should be enabled or disabled for a virtual machine + or virtual machine scale set. Default is disabled. + type: boolean + type: object + spotVMOptions: + description: SpotVMOptions allows the ability to specify the + Machine should use a Spot VM + properties: + maxPrice: + anyOf: + - type: integer + - type: string + description: MaxPrice defines the maximum price the user + is willing to pay for Spot VM instances + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + sshPublicKey: + type: string + subnetName: + description: SubnetName selects the Subnet where the VM will + be placed + type: string + userAssignedIdentities: + description: UserAssignedIdentities is a list of standalone + Azure identities provided by the user The lifecycle of a + user-assigned identity is managed separately from the lifecycle + of the AzureMachine. See https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-manage-ua-identity-cli + items: + description: UserAssignedIdentity defines the user-assigned + identities provided by the user to be assigned to Azure + resources. + properties: + providerID: + description: 'ProviderID is the identification ID of + the user-assigned Identity, the format of an identity + is: ''azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}''' + type: string + required: + - providerID + type: object + type: array + vmSize: + type: string + required: + - osDisk + - sshPublicKey + - vmSize + type: object + required: + - spec + type: object + required: + - template + type: object + type: object + served: true + storage: false + - name: v1beta1 + schema: + openAPIV3Schema: + description: AzureMachineTemplate is the Schema for the azuremachinetemplates + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureMachineTemplateSpec defines the desired state of AzureMachineTemplate. + properties: + template: + description: AzureMachineTemplateResource describes the data needed + to create an AzureMachine from a template. + properties: + metadata: + description: "ObjectMeta is metadata that all persisted resources + must have, which includes all objects users must create. This + is a copy of customizable fields from metav1.ObjectMeta. \n + ObjectMeta is embedded in `Machine.Spec`, `MachineDeployment.Template` + and `MachineSet.Template`, which are not top-level Kubernetes + objects. Given that metav1.ObjectMeta has lots of special cases + and read-only fields which end up in the generated CRD validation, + having it as a subset simplifies the API and some issues that + can impact user experience. \n During the [upgrade to controller-tools@v2](https://github.com/kubernetes-sigs/cluster-api/pull/1054) + for v1alpha2, we noticed a failure would occur running Cluster + API test suite against the new CRDs, specifically `spec.metadata.creationTimestamp + in body must be of type string: \"null\"`. The investigation + showed that `controller-tools@v2` behaves differently than its + previous version when handling types from [metav1](k8s.io/apimachinery/pkg/apis/meta/v1) + package. \n In more details, we found that embedded (non-top + level) types that embedded `metav1.ObjectMeta` had validation + properties, including for `creationTimestamp` (metav1.Time). + The `metav1.Time` type specifies a custom json marshaller that, + when IsZero() is true, returns `null` which breaks validation + because the field isn't marked as nullable. \n In future versions, + controller-tools@v2 might allow overriding the type and validation + for embedded types. When that happens, this hack should be revisited." + properties: + annotations: + additionalProperties: + type: string + description: 'Annotations is an unstructured key value map + stored with a resource that may be set by external tools + to store and retrieve arbitrary metadata. They are not queryable + and should be preserved when modifying objects. More info: + http://kubernetes.io/docs/user-guide/annotations' + type: object + labels: + additionalProperties: + type: string + description: 'Map of string keys and values that can be used + to organize and categorize (scope and select) objects. May + match selectors of replication controllers and services. + More info: http://kubernetes.io/docs/user-guide/labels' + type: object + type: object + spec: + description: Spec is the specification of the desired behavior + of the machine. + properties: + acceleratedNetworking: + description: AcceleratedNetworking enables or disables Azure + accelerated networking. If omitted, it will be set based + on whether the requested VMSize supports accelerated networking. + If AcceleratedNetworking is set to true with a VMSize that + does not support it, Azure will return an error. + type: boolean + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to + add to an instance, in addition to the ones added by default + by the Azure provider. If both the AzureCluster and the + AzureMachine specify the same tag name with different values, + the AzureMachine's value takes precedence. + type: object + allocatePublicIP: + description: AllocatePublicIP allows the ability to create + dynamic public ips for machines where this value is true. + type: boolean + dataDisks: + description: DataDisk specifies the parameters that are used + to add one or more data disks to the machine + items: + description: DataDisk specifies the parameters that are + used to add one or more data disks to the machine. + properties: + cachingType: + description: CachingType specifies the caching requirements. + enum: + - None + - ReadOnly + - ReadWrite + type: string + diskSizeGB: + description: DiskSizeGB is the size in GB to assign + to the data disk. + format: int32 + type: integer + lun: + description: Lun Specifies the logical unit number of + the data disk. This value is used to identify data + disks within the VM and therefore must be unique for + each data disk attached to a VM. The value must be + between 0 and 63. + format: int32 + type: integer + managedDisk: + description: ManagedDisk specifies the Managed Disk + parameters for the data disk. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines + disk encryption options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + type: object + nameSuffix: + description: NameSuffix is the suffix to be appended + to the machine name to generate the disk name. Each + disk name will be in format _. + type: string + required: + - diskSizeGB + - nameSuffix + type: object + type: array + enableIPForwarding: + description: EnableIPForwarding enables IP Forwarding in Azure + which is required for some CNI's to send traffic from a + pods on one machine to another. This is required for IpV6 + with Calico in combination with User Defined Routes (set + by the Azure Cloud Controller manager). Default is false + for disabled. + type: boolean + failureDomain: + description: FailureDomain is the failure domain unique identifier + this Machine should be attached to, as defined in Cluster + API. This relates to an Azure Availability Zone + type: string + identity: + default: None + description: Identity is the type of identity used for the + virtual machine. The type 'SystemAssigned' is an implicitly + created identity. The generated identity will be assigned + a Subscription contributor role. The type 'UserAssigned' + is a standalone Azure resource provided by the user and + assigned to the VM + enum: + - None + - SystemAssigned + - UserAssigned + type: string + image: + description: Image is used to provide details of an image + to use during VM creation. If image details are omitted + the image will default the Azure Marketplace "capi" offer, + which is based on Ubuntu. + properties: + computeGallery: + description: ComputeGallery specifies an image to use + from the Azure Compute Gallery + properties: + gallery: + description: Gallery specifies the name of the compute + image gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + plan: + description: Plan contains plan information. + properties: + offer: + description: Offer specifies the name of a group + of related images created by the publisher. + For example, UbuntuServer, WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization + that created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, + such as a major release of a distribution. For + example, 18.04-LTS, 2019-Datacenter + minLength: 1 + type: string + required: + - offer + - publisher + - sku + type: object + resourceGroup: + description: ResourceGroup specifies the resource + group containing the private compute gallery. + type: string + subscriptionID: + description: SubscriptionID is the identifier of the + subscription that contains the private compute gallery. + type: string + version: + description: Version specifies the version of the + marketplace image. The allowed formats are Major.Minor.Build + or 'latest'. Major, Minor, and Build are decimal + numbers. Specify 'latest' to use the latest version + of an image available at deploy time. Even if you + use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes + available. + minLength: 1 + type: string + required: + - gallery + - name + - version + type: object + id: + description: ID specifies an image to use by ID + type: string + marketplace: + description: Marketplace specifies an image to use from + the Azure Marketplace + properties: + offer: + description: Offer specifies the name of a group of + related images created by the publisher. For example, + UbuntuServer, WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization + that created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, + such as a major release of a distribution. For example, + 18.04-LTS, 2019-Datacenter + minLength: 1 + type: string + thirdPartyImage: + default: false + description: ThirdPartyImage indicates the image is + published by a third party publisher and a Plan + will be generated for it. + type: boolean + version: + description: Version specifies the version of an image + sku. The allowed formats are Major.Minor.Build or + 'latest'. Major, Minor, and Build are decimal numbers. + Specify 'latest' to use the latest version of an + image available at deploy time. Even if you use + 'latest', the VM image will not automatically update + after deploy time even if a new version becomes + available. + minLength: 1 + type: string + required: + - offer + - publisher + - sku + - version + type: object + sharedGallery: + description: 'SharedGallery specifies an image to use + from an Azure Shared Image Gallery Deprecated: use ComputeGallery + instead.' + properties: + gallery: + description: Gallery specifies the name of the shared + image gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + offer: + description: Offer specifies the name of a group of + related images created by the publisher. For example, + UbuntuServer, WindowsServer This value will be used + to add a `Plan` in the API request when creating + the VM/VMSS resource. This is needed when the source + image from which this SIG image was built requires + the `Plan` to be used. + type: string + publisher: + description: Publisher is the name of the organization + that created the image. This value will be used + to add a `Plan` in the API request when creating + the VM/VMSS resource. This is needed when the source + image from which this SIG image was built requires + the `Plan` to be used. + type: string + resourceGroup: + description: ResourceGroup specifies the resource + group containing the shared image gallery + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, + such as a major release of a distribution. For example, + 18.04-LTS, 2019-Datacenter This value will be used + to add a `Plan` in the API request when creating + the VM/VMSS resource. This is needed when the source + image from which this SIG image was built requires + the `Plan` to be used. + type: string + subscriptionID: + description: SubscriptionID is the identifier of the + subscription that contains the shared image gallery + minLength: 1 + type: string + version: + description: Version specifies the version of the + marketplace image. The allowed formats are Major.Minor.Build + or 'latest'. Major, Minor, and Build are decimal + numbers. Specify 'latest' to use the latest version + of an image available at deploy time. Even if you + use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes + available. + minLength: 1 + type: string + required: + - gallery + - name + - resourceGroup + - subscriptionID + - version + type: object + type: object + osDisk: + description: OSDisk specifies the parameters for the operating + system disk of the machine + properties: + cachingType: + description: CachingType specifies the caching requirements. + enum: + - None + - ReadOnly + - ReadWrite + type: string + diffDiskSettings: + description: DiffDiskSettings describe ephemeral disk + settings for the os disk. + properties: + option: + description: Option enables ephemeral OS when set + to "Local" See https://docs.microsoft.com/en-us/azure/virtual-machines/ephemeral-os-disks + for full details + enum: + - Local + type: string + required: + - option + type: object + diskSizeGB: + description: DiskSizeGB is the size in GB to assign to + the OS disk. Will have a default of 30GB if not provided + format: int32 + type: integer + managedDisk: + description: ManagedDisk specifies the Managed Disk parameters + for the OS disk. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk + encryption options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + type: object + osType: + type: string + required: + - osType + type: object + providerID: + description: ProviderID is the unique identifier as specified + by the cloud provider. + type: string + roleAssignmentName: + description: RoleAssignmentName is the name of the role assignment + to create for a system assigned identity. It can be any + valid GUID. If not specified, a random GUID will be generated. + type: string + securityProfile: + description: SecurityProfile specifies the Security profile + settings for a virtual machine. + properties: + encryptionAtHost: + description: This field indicates whether Host Encryption + should be enabled or disabled for a virtual machine + or virtual machine scale set. Default is disabled. + type: boolean + type: object + spotVMOptions: + description: SpotVMOptions allows the ability to specify the + Machine should use a Spot VM + properties: + maxPrice: + anyOf: + - type: integer + - type: string + description: MaxPrice defines the maximum price the user + is willing to pay for Spot VM instances + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + sshPublicKey: + type: string + subnetName: + description: SubnetName selects the Subnet where the VM will + be placed + type: string + userAssignedIdentities: + description: UserAssignedIdentities is a list of standalone + Azure identities provided by the user The lifecycle of a + user-assigned identity is managed separately from the lifecycle + of the AzureMachine. See https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-manage-ua-identity-cli + items: + description: UserAssignedIdentity defines the user-assigned + identities provided by the user to be assigned to Azure + resources. + properties: + providerID: + description: 'ProviderID is the identification ID of + the user-assigned Identity, the format of an identity + is: ''azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}''' + type: string + required: + - providerID + type: object + type: array + vmSize: + type: string + required: + - osDisk + - sshPublicKey + - vmSize + type: object + required: + - spec + type: object + required: + - template + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-azure + cluster.x-k8s.io/v1beta1: v1beta1 + name: azuremanagedclusters.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureManagedCluster + listKind: AzureManagedClusterList + plural: azuremanagedclusters + shortNames: + - amc + singular: azuremanagedcluster + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: AzureManagedCluster is the Schema for the azuremanagedclusters + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureManagedClusterSpec defines the desired state of AzureManagedCluster. + properties: + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + type: object + status: + description: AzureManagedClusterStatus defines the observed state of AzureManagedCluster. + properties: + ready: + description: Ready is true when the provider resource is ready. + type: boolean + type: object + type: object + served: true + storage: false + subresources: + status: {} + - name: v1alpha4 + schema: + openAPIV3Schema: + description: AzureManagedCluster is the Schema for the azuremanagedclusters + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureManagedClusterSpec defines the desired state of AzureManagedCluster. + properties: + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + type: object + status: + description: AzureManagedClusterStatus defines the observed state of AzureManagedCluster. + properties: + ready: + description: Ready is true when the provider resource is ready. + type: boolean + type: object + type: object + served: true + storage: false + subresources: + status: {} + - name: v1beta1 + schema: + openAPIV3Schema: + description: AzureManagedCluster is the Schema for the azuremanagedclusters + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureManagedClusterSpec defines the desired state of AzureManagedCluster. + properties: + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + type: object + status: + description: AzureManagedClusterStatus defines the observed state of AzureManagedCluster. + properties: + ready: + description: Ready is true when the provider resource is ready. + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-azure + cluster.x-k8s.io/v1beta1: v1beta1 + name: azuremanagedcontrolplanes.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureManagedControlPlane + listKind: AzureManagedControlPlaneList + plural: azuremanagedcontrolplanes + shortNames: + - amcp + singular: azuremanagedcontrolplane + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: AzureManagedControlPlane is the Schema for the azuremanagedcontrolplanes + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureManagedControlPlaneSpec defines the desired state of + AzureManagedControlPlane. + properties: + aadProfile: + description: AadProfile is Azure Active Directory configuration to + integrate with AKS for aad authentication. + properties: + adminGroupObjectIDs: + description: AdminGroupObjectIDs - AAD group object IDs that will + have admin role of the cluster. + items: + type: string + type: array + managed: + description: Managed - Whether to enable managed AAD. + type: boolean + required: + - adminGroupObjectIDs + - managed + type: object + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to Azure + resources managed by the Azure provider, in addition to the ones + added by default. + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + dnsServiceIP: + description: DNSServiceIP is an IP address assigned to the Kubernetes + DNS service. It must be within the Kubernetes service address range + specified in serviceCidr. + type: string + loadBalancerSKU: + description: LoadBalancerSKU is the SKU of the loadBalancer to be + provisioned. + enum: + - Basic + - Standard + type: string + location: + description: 'Location is a string matching one of the canonical Azure + region names. Examples: "westus2", "eastus".' + type: string + networkPlugin: + description: NetworkPlugin used for building Kubernetes network. + enum: + - azure + - kubenet + type: string + networkPolicy: + description: NetworkPolicy used for building Kubernetes network. + enum: + - azure + - calico + type: string + nodeResourceGroupName: + description: NodeResourceGroupName is the name of the resource group + containing cluster IaaS resources. Will be populated to default + in webhook. + type: string + resourceGroupName: + description: ResourceGroupName is the name of the Azure resource group + for this AKS Cluster. + type: string + sshPublicKey: + description: SSHPublicKey is a string literal containing an ssh public + key base64 encoded. + type: string + subscriptionID: + description: SubscriotionID is the GUID of the Azure subscription + to hold this cluster. + type: string + version: + description: Version defines the desired Kubernetes version. + minLength: 2 + type: string + virtualNetwork: + description: VirtualNetwork describes the vnet for the AKS cluster. + Will be created if it does not exist. + properties: + cidrBlock: + type: string + name: + type: string + subnet: + description: ManagedControlPlaneSubnet describes a subnet for + an AKS cluster. + properties: + cidrBlock: + type: string + name: + type: string + required: + - cidrBlock + - name + type: object + required: + - cidrBlock + - name + type: object + required: + - location + - nodeResourceGroupName + - resourceGroupName + - sshPublicKey + - version + type: object + status: + description: AzureManagedControlPlaneStatus defines the observed state + of AzureManagedControlPlane. + properties: + initialized: + description: Initialized is true when the the control plane is available + for initial contact. This may occur before the control plane is + fully ready. In the AzureManagedControlPlane implementation, these + are identical. + type: boolean + ready: + description: Ready is true when the provider resource is ready. + type: boolean + type: object + type: object + served: true + storage: false + subresources: + status: {} + - name: v1alpha4 + schema: + openAPIV3Schema: + description: AzureManagedControlPlane is the Schema for the azuremanagedcontrolplanes + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureManagedControlPlaneSpec defines the desired state of + AzureManagedControlPlane. + properties: + aadProfile: + description: AadProfile is Azure Active Directory configuration to + integrate with AKS for aad authentication. + properties: + adminGroupObjectIDs: + description: AdminGroupObjectIDs - AAD group object IDs that will + have admin role of the cluster. + items: + type: string + type: array + managed: + description: Managed - Whether to enable managed AAD. + type: boolean + required: + - adminGroupObjectIDs + - managed + type: object + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to Azure + resources managed by the Azure provider, in addition to the ones + added by default. + type: object + apiServerAccessProfile: + description: APIServerAccessProfile is the access profile for AKS + API server. + properties: + authorizedIPRanges: + description: AuthorizedIPRanges - Authorized IP Ranges to kubernetes + API server. + items: + type: string + type: array + enablePrivateCluster: + description: EnablePrivateCluster - Whether to create the cluster + as a private cluster or not. + type: boolean + enablePrivateClusterPublicFQDN: + description: EnablePrivateClusterPublicFQDN - Whether to create + additional public FQDN for private cluster or not. + type: boolean + privateDNSZone: + description: PrivateDNSZone - Private dns zone mode for private + cluster. + enum: + - System + - None + type: string + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + dnsServiceIP: + description: DNSServiceIP is an IP address assigned to the Kubernetes + DNS service. It must be within the Kubernetes service address range + specified in serviceCidr. + type: string + identityRef: + description: IdentityRef is a reference to a AzureClusterIdentity + to be used when reconciling this cluster + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + loadBalancerProfile: + description: LoadBalancerProfile is the profile of the cluster load + balancer. + properties: + allocatedOutboundPorts: + description: AllocatedOutboundPorts - Desired number of allocated + SNAT ports per VM. Allowed values must be in the range of 0 + to 64000 (inclusive). The default value is 0 which results in + Azure dynamically allocating ports. + format: int32 + type: integer + idleTimeoutInMinutes: + description: IdleTimeoutInMinutes - Desired outbound flow idle + timeout in minutes. Allowed values must be in the range of 4 + to 120 (inclusive). The default value is 30 minutes. + format: int32 + type: integer + managedOutboundIPs: + description: ManagedOutboundIPs - Desired managed outbound IPs + for the cluster load balancer. + format: int32 + type: integer + outboundIPPrefixes: + description: OutboundIPPrefixes - Desired outbound IP Prefix resources + for the cluster load balancer. + items: + type: string + type: array + outboundIPs: + description: OutboundIPs - Desired outbound IP resources for the + cluster load balancer. + items: + type: string + type: array + type: object + loadBalancerSKU: + description: LoadBalancerSKU is the SKU of the loadBalancer to be + provisioned. + enum: + - Basic + - Standard + type: string + location: + description: 'Location is a string matching one of the canonical Azure + region names. Examples: "westus2", "eastus".' + type: string + networkPlugin: + description: NetworkPlugin used for building Kubernetes network. + enum: + - azure + - kubenet + type: string + networkPolicy: + description: NetworkPolicy used for building Kubernetes network. + enum: + - azure + - calico + type: string + nodeResourceGroupName: + description: NodeResourceGroupName is the name of the resource group + containing cluster IaaS resources. Will be populated to default + in webhook. + type: string + resourceGroupName: + description: ResourceGroupName is the name of the Azure resource group + for this AKS Cluster. + type: string + sku: + description: SKU is the SKU of the AKS to be provisioned. + properties: + tier: + description: Tier - Tier of a managed cluster SKU. + enum: + - Free + - Paid + type: string + required: + - tier + type: object + sshPublicKey: + description: SSHPublicKey is a string literal containing an ssh public + key base64 encoded. + type: string + subscriptionID: + description: SubscriptionID is the GUID of the Azure subscription + to hold this cluster. + type: string + version: + description: Version defines the desired Kubernetes version. + minLength: 2 + type: string + virtualNetwork: + description: VirtualNetwork describes the vnet for the AKS cluster. + Will be created if it does not exist. + properties: + cidrBlock: + type: string + name: + type: string + resourceGroupName: + description: ResourceGroupName is the name of the Azure resource + group for the VNet and Subnet. + type: string + subnet: + description: ManagedControlPlaneSubnet describes a subnet for + an AKS cluster. + properties: + cidrBlock: + type: string + name: + type: string + required: + - cidrBlock + - name + type: object + required: + - cidrBlock + - name + type: object + required: + - location + - resourceGroupName + - sshPublicKey + - version + type: object + status: + description: AzureManagedControlPlaneStatus defines the observed state + of AzureManagedControlPlane. + properties: + initialized: + description: Initialized is true when the the control plane is available + for initial contact. This may occur before the control plane is + fully ready. In the AzureManagedControlPlane implementation, these + are identical. + type: boolean + longRunningOperationStates: + description: LongRunningOperationStates saves the states for Azure + long-running operations so they can be continued on the next reconciliation + loop. + items: + description: Future contains the data needed for an Azure long-running + operation to continue across reconcile loops. + properties: + data: + description: Data is the base64 url encoded json Azure AutoRest + Future. + type: string + name: + description: Name is the name of the Azure resource. Together + with the service name, this forms the unique identifier for + the future. + type: string + resourceGroup: + description: ResourceGroup is the Azure resource group for the + resource. + type: string + serviceName: + description: ServiceName is the name of the Azure service. Together + with the name of the resource, this forms the unique identifier + for the future. + type: string + type: + description: Type describes the type of future, such as update, + create, delete, etc. + type: string + required: + - name + - serviceName + - type + type: object + type: array + ready: + description: Ready is true when the provider resource is ready. + type: boolean + type: object + type: object + served: true + storage: false + subresources: + status: {} + - name: v1beta1 + schema: + openAPIV3Schema: + description: AzureManagedControlPlane is the Schema for the azuremanagedcontrolplanes + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureManagedControlPlaneSpec defines the desired state of + AzureManagedControlPlane. + properties: + aadProfile: + description: AadProfile is Azure Active Directory configuration to + integrate with AKS for aad authentication. + properties: + adminGroupObjectIDs: + description: AdminGroupObjectIDs - AAD group object IDs that will + have admin role of the cluster. + items: + type: string + type: array + managed: + description: Managed - Whether to enable managed AAD. + type: boolean + required: + - adminGroupObjectIDs + - managed + type: object + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to Azure + resources managed by the Azure provider, in addition to the ones + added by default. + type: object + addonProfiles: + description: AddonProfiles are the profiles of managed cluster add-on. + items: + properties: + config: + additionalProperties: + type: string + description: Config - Key-value pairs for configuring an add-on. + type: object + enabled: + description: Enabled - Whether the add-on is enabled or not. + type: boolean + name: + description: Name- The name of managed cluster add-on. + type: string + required: + - enabled + - name + type: object + type: array + apiServerAccessProfile: + description: APIServerAccessProfile is the access profile for AKS + API server. + properties: + authorizedIPRanges: + description: AuthorizedIPRanges - Authorized IP Ranges to kubernetes + API server. + items: + type: string + type: array + enablePrivateCluster: + description: EnablePrivateCluster - Whether to create the cluster + as a private cluster or not. + type: boolean + enablePrivateClusterPublicFQDN: + description: EnablePrivateClusterPublicFQDN - Whether to create + additional public FQDN for private cluster or not. + type: boolean + privateDNSZone: + description: PrivateDNSZone - Private dns zone mode for private + cluster. + type: string + type: object + autoUpgradeProfile: + description: AutoUpgradeProfile - Profile of auto upgrade configuration. + properties: + upgradeChannel: + description: 'UpgradeChannel - upgrade channel for auto upgrade. + Possible values include: "node-image","none","patch","rapid","stable"' + enum: + - node-image + - none + - patch + - rapid + - stable + type: string + required: + - upgradeChannel + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + disableLocalAccounts: + description: DisableLocalAccounts - If set to true, getting static + credential will be disabled for this cluster. Expected to only be + used for AAD clusters. + type: boolean + dnsPrefix: + description: DNSPrefix - DNS prefix specified when creating the managed + cluster. + type: string + dnsServiceIP: + description: DNSServiceIP is an IP address assigned to the Kubernetes + DNS service. It must be within the Kubernetes service address range + specified in serviceCidr. + type: string + dockerBridgeCidr: + description: DockerBridgeCidr - A CIDR notation IP range assigned + to the Docker bridge network. It must not overlap with any Subnet + IP ranges or the Kubernetes service address range. + type: string + fqdnSubdomain: + description: FqdnSubdomain - FQDN subdomain specified when creating + private cluster with custom private dns zone. + type: string + identityRef: + description: IdentityRef is a reference to a AzureClusterIdentity + to be used when reconciling this cluster + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + loadBalancerProfile: + description: LoadBalancerProfile is the profile of the cluster load + balancer. + properties: + allocatedOutboundPorts: + description: AllocatedOutboundPorts - Desired number of allocated + SNAT ports per VM. Allowed values must be in the range of 0 + to 64000 (inclusive). The default value is 0 which results in + Azure dynamically allocating ports. + format: int32 + type: integer + idleTimeoutInMinutes: + description: IdleTimeoutInMinutes - Desired outbound flow idle + timeout in minutes. Allowed values must be in the range of 4 + to 120 (inclusive). The default value is 30 minutes. + format: int32 + type: integer + managedOutboundIPs: + description: ManagedOutboundIPs - Desired managed outbound IPs + for the cluster load balancer. + format: int32 + type: integer + outboundIPPrefixes: + description: OutboundIPPrefixes - Desired outbound IP Prefix resources + for the cluster load balancer. + items: + type: string + type: array + outboundIPs: + description: OutboundIPs - Desired outbound IP resources for the + cluster load balancer. + items: + type: string + type: array + type: object + loadBalancerSKU: + description: LoadBalancerSKU is the SKU of the loadBalancer to be + provisioned. + enum: + - Basic + - Standard + type: string + location: + description: 'Location is a string matching one of the canonical Azure + region names. Examples: "westus2", "eastus".' + type: string + networkPlugin: + description: NetworkPlugin used for building Kubernetes network. + enum: + - azure + - kubenet + type: string + networkPolicy: + description: NetworkPolicy used for building Kubernetes network. + enum: + - azure + - calico + type: string + nodeResourceGroupName: + description: NodeResourceGroupName is the name of the resource group + containing cluster IaaS resources. Will be populated to default + in webhook. + type: string + outboundType: + description: Outbound configuration used by Nodes. + enum: + - loadBalancer + - managedNATGateway + - userAssignedNATGateway + - userDefinedRouting + type: string + resourceGroupName: + description: ResourceGroupName is the name of the Azure resource group + for this AKS Cluster. + type: string + sku: + description: SKU is the SKU of the AKS to be provisioned. + properties: + tier: + description: Tier - Tier of a managed cluster SKU. + enum: + - Free + - Paid + type: string + required: + - tier + type: object + sshPublicKey: + description: SSHPublicKey is a string literal containing an ssh public + key base64 encoded. + type: string + subscriptionID: + description: SubscriptionID is the GUID of the Azure subscription + to hold this cluster. + type: string + userAssignedIdentities: + description: UserAssignedIdentities is a list of standalone Azure + identities provided by the user to assign the cluster + items: + description: UserAssignedIdentity defines the user-assigned identities + provided by the user to be assigned to Azure resources. + properties: + providerID: + description: 'ProviderID is the identification ID of the user-assigned + Identity, the format of an identity is: ''azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}''' + type: string + required: + - providerID + type: object + type: array + version: + description: Version defines the desired Kubernetes version. + minLength: 2 + type: string + virtualNetwork: + description: VirtualNetwork describes the vnet for the AKS cluster. + Will be created if it does not exist. + properties: + cidrBlock: + type: string + name: + type: string + resourceGroupName: + description: ResourceGroupName is the name of the Azure resource + group for the VNet and Subnet. + type: string + subnet: + description: ManagedControlPlaneSubnet describes a subnet for + an AKS cluster. + properties: + cidrBlock: + type: string + name: + type: string + required: + - cidrBlock + - name + type: object + required: + - cidrBlock + - name + type: object + required: + - location + - resourceGroupName + - sshPublicKey + - version + type: object + status: + description: AzureManagedControlPlaneStatus defines the observed state + of AzureManagedControlPlane. + properties: + autoUpgradeVersion: + description: AutoUpgradeVersion is the Kubernetes version populated + after autoupgrade based on the upgrade channel. + minLength: 2 + type: string + conditions: + description: Conditions defines current service state of the AzureManagedControlPlane. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + initialized: + description: Initialized is true when the control plane is available + for initial contact. This may occur before the control plane is + fully ready. In the AzureManagedControlPlane implementation, these + are identical. + type: boolean + longRunningOperationStates: + description: LongRunningOperationStates saves the states for Azure + long-running operations so they can be continued on the next reconciliation + loop. + items: + description: Future contains the data needed for an Azure long-running + operation to continue across reconcile loops. + properties: + data: + description: Data is the base64 url encoded json Azure AutoRest + Future. + type: string + name: + description: Name is the name of the Azure resource. Together + with the service name, this forms the unique identifier for + the future. + type: string + resourceGroup: + description: ResourceGroup is the Azure resource group for the + resource. + type: string + serviceName: + description: ServiceName is the name of the Azure service. Together + with the name of the resource, this forms the unique identifier + for the future. + type: string + type: + description: Type describes the type of future, such as update, + create, delete, etc. + type: string + required: + - data + - name + - serviceName + - type + type: object + type: array + ready: + description: Ready is true when the provider resource is ready. + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-azure + cluster.x-k8s.io/v1beta1: v1beta1 + name: azuremanagedmachinepools.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureManagedMachinePool + listKind: AzureManagedMachinePoolList + plural: azuremanagedmachinepools + shortNames: + - ammp + singular: azuremanagedmachinepool + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: AzureManagedMachinePool is the Schema for the azuremanagedmachinepools + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureManagedMachinePoolSpec defines the desired state of + AzureManagedMachinePool. + properties: + mode: + description: 'Mode - represents mode of an agent pool. Possible values + include: System, User.' + enum: + - System + - User + type: string + osDiskSizeGB: + description: OSDiskSizeGB is the disk size for every machine in this + agent pool. If you specify 0, it will apply the default osDisk size + according to the vmSize specified. + format: int32 + type: integer + providerIDList: + description: ProviderIDList is the unique identifier as specified + by the cloud provider. + items: + type: string + type: array + sku: + description: SKU is the size of the VMs in the node pool. + type: string + required: + - mode + - sku + type: object + status: + description: AzureManagedMachinePoolStatus defines the observed state + of AzureManagedMachinePool. + properties: + errorMessage: + description: Any transient errors that occur during the reconciliation + of Machines can be added as events to the Machine object and/or + logged in the controller's output. + type: string + errorReason: + description: Any transient errors that occur during the reconciliation + of Machines can be added as events to the Machine object and/or + logged in the controller's output. + type: string + ready: + description: Ready is true when the provider resource is ready. + type: boolean + replicas: + description: Replicas is the most recently observed number of replicas. + format: int32 + type: integer + type: object + type: object + served: true + storage: false + subresources: + status: {} + - name: v1alpha4 + schema: + openAPIV3Schema: + description: AzureManagedMachinePool is the Schema for the azuremanagedmachinepools + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureManagedMachinePoolSpec defines the desired state of + AzureManagedMachinePool. + properties: + mode: + description: 'Mode - represents mode of an agent pool. Possible values + include: System, User.' + enum: + - System + - User + type: string + name: + description: Name - name of the agent pool. If not specified, CAPZ + uses the name of the CR as the agent pool name. + type: string + osDiskSizeGB: + description: OSDiskSizeGB is the disk size for every machine in this + agent pool. If you specify 0, it will apply the default osDisk size + according to the vmSize specified. + format: int32 + type: integer + osType: + description: 'OSType specifies the virtual machine operating system. + Default to Linux. Possible values include: ''Linux'', ''Windows''' + enum: + - Linux + - Windows + type: string + providerIDList: + description: ProviderIDList is the unique identifier as specified + by the cloud provider. + items: + type: string + type: array + sku: + description: SKU is the size of the VMs in the node pool. + type: string + required: + - mode + - sku + type: object + status: + description: AzureManagedMachinePoolStatus defines the observed state + of AzureManagedMachinePool. + properties: + errorMessage: + description: Any transient errors that occur during the reconciliation + of Machines can be added as events to the Machine object and/or + logged in the controller's output. + type: string + errorReason: + description: Any transient errors that occur during the reconciliation + of Machines can be added as events to the Machine object and/or + logged in the controller's output. + type: string + ready: + description: Ready is true when the provider resource is ready. + type: boolean + replicas: + description: Replicas is the most recently observed number of replicas. + format: int32 + type: integer + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .spec.mode + name: Mode + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: AzureManagedMachinePool is the Schema for the azuremanagedmachinepools + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureManagedMachinePoolSpec defines the desired state of + AzureManagedMachinePool. + properties: + availabilityZones: + description: AvailabilityZones - Availability zones for nodes. Must + use VirtualMachineScaleSets AgentPoolType. + items: + type: string + type: array + enableUltraSSD: + description: EnableUltraSSD enables the storage type UltraSSD_LRS + for the agent pool. + type: boolean + maxPods: + description: MaxPods specifies the kubelet --max-pods configuration + for the node pool. + format: int32 + type: integer + mode: + description: 'Mode - represents mode of an agent pool. Possible values + include: System, User.' + enum: + - System + - User + type: string + name: + description: Name - name of the agent pool. If not specified, CAPZ + uses the name of the CR as the agent pool name. + type: string + nodeLabels: + additionalProperties: + type: string + description: Node labels - labels for all of the nodes present in + node pool + type: object + osDiskSizeGB: + description: OSDiskSizeGB is the disk size for every machine in this + agent pool. If you specify 0, it will apply the default osDisk size + according to the vmSize specified. + format: int32 + type: integer + osDiskType: + default: Managed + description: OsDiskType specifies the OS disk type for each node in + the pool. Allowed values are 'Ephemeral' and 'Managed'. + enum: + - Ephemeral + - Managed + type: string + osType: + description: 'OSType specifies the virtual machine operating system. + Default to Linux. Possible values include: ''Linux'', ''Windows''' + enum: + - Linux + - Windows + type: string + providerIDList: + description: ProviderIDList is the unique identifier as specified + by the cloud provider. + items: + type: string + type: array + scaling: + description: Scaling specifies the autoscaling parameters for the + node pool. + properties: + maxSize: + format: int32 + type: integer + minSize: + format: int32 + type: integer + type: object + sku: + description: SKU is the size of the VMs in the node pool. + type: string + subnetName: + description: SubnetName selects the Subnet where the MachinePool will + be placed + type: string + taints: + description: Taints specifies the taints for nodes present in this + agent pool. + items: + properties: + effect: + description: Effect specifies the effect for the taint + enum: + - NoSchedule + - NoExecute + - PreferNoSchedule + type: string + key: + description: Key is the key of the taint + type: string + value: + description: Value is the value of the taint + type: string + required: + - effect + - key + - value + type: object + type: array + required: + - mode + - sku + type: object + status: + description: AzureManagedMachinePoolStatus defines the observed state + of AzureManagedMachinePool. + properties: + conditions: + description: Conditions defines current service state of the AzureManagedControlPlane. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + errorMessage: + description: Any transient errors that occur during the reconciliation + of Machines can be added as events to the Machine object and/or + logged in the controller's output. + type: string + errorReason: + description: Any transient errors that occur during the reconciliation + of Machines can be added as events to the Machine object and/or + logged in the controller's output. + type: string + longRunningOperationStates: + description: LongRunningOperationStates saves the states for Azure + long-running operations so they can be continued on the next reconciliation + loop. + items: + description: Future contains the data needed for an Azure long-running + operation to continue across reconcile loops. + properties: + data: + description: Data is the base64 url encoded json Azure AutoRest + Future. + type: string + name: + description: Name is the name of the Azure resource. Together + with the service name, this forms the unique identifier for + the future. + type: string + resourceGroup: + description: ResourceGroup is the Azure resource group for the + resource. + type: string + serviceName: + description: ServiceName is the name of the Azure service. Together + with the name of the resource, this forms the unique identifier + for the future. + type: string + type: + description: Type describes the type of future, such as update, + create, delete, etc. + type: string + required: + - data + - name + - serviceName + - type + type: object + type: array + ready: + description: Ready is true when the provider resource is ready. + type: boolean + replicas: + description: Replicas is the most recently observed number of replicas. + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: unapproved + controller-gen.kubebuilder.io/version: v0.5.0 + labels: + cluster.x-k8s.io/provider: infrastructure-azure + name: azurepodidentityexceptions.aadpodidentity.k8s.io +spec: + group: aadpodidentity.k8s.io + names: + kind: AzurePodIdentityException + listKind: AzurePodIdentityExceptionList + plural: azurepodidentityexceptions + singular: azurepodidentityexception + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: AzurePodIdentityException contains the pod selectors for all + pods that don't require NMI to process and request token on their behalf. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzurePodIdentityExceptionSpec matches pods with the selector + defined. If request originates from a pod that matches the selector, + nmi will proxy the request and send response back without any validation. + properties: + metadata: + type: object + podLabels: + additionalProperties: + type: string + type: object + type: object + status: + description: AzurePodIdentityExceptionStatus contains the status of an + AzurePodIdentityException. + properties: + metadata: + type: object + status: + type: string + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-azure + name: capz-mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azurecluster + failurePolicy: Fail + matchPolicy: Equivalent + name: default.azurecluster.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azureclusters + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azureclustertemplate + failurePolicy: Fail + matchPolicy: Equivalent + name: default.azureclustertemplate.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azureclustertemplates + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachine + failurePolicy: Fail + matchPolicy: Equivalent + name: default.azuremachine.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremachines + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachinetemplate + failurePolicy: Fail + matchPolicy: Equivalent + name: default.azuremachinetemplate.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremachinetemplates + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachinepool + failurePolicy: Fail + name: default.azuremachinepool.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremachinepools + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedcontrolplane + failurePolicy: Fail + name: default.azuremanagedcontrolplanes.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremanagedcontrolplanes + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedmachinepool + failurePolicy: Fail + matchPolicy: Equivalent + name: default.azuremanagedmachinepools.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremanagedmachinepools + sideEffects: None +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-azure + name: capz-aad-pod-id-nmi-role +rules: +- apiGroups: + - cluster.x-k8s.io + resources: + - clusters + verbs: + - get + - list + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - azuremanagedclusters + - azuremanagedcontrolplanes + verbs: + - '*' +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get +- apiGroups: + - aadpodidentity.k8s.io + resources: + - azureidentitybindings + - azureidentities + - azurepodidentityexceptions + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-azure + k8s-app: capz-aad-pod-id-nmi-binding + name: capz-aad-pod-id-nmi-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: capz-aad-pod-id-nmi-role +subjects: +- kind: ServiceAccount + name: default + namespace: capi-webhook-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-azure + name: capz-webhook-service + namespace: capi-webhook-system +spec: + ports: + - port: 443 + targetPort: webhook-server + selector: + cluster.x-k8s.io/provider: infrastructure-azure +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + aadpodidbinding: capz-controller-aadpodidentity-selector + cluster.x-k8s.io/provider: infrastructure-azure + control-plane: capz-controller-manager + name: capz-controller-manager + namespace: capi-webhook-system +spec: + replicas: 1 + selector: + matchLabels: + cluster.x-k8s.io/provider: infrastructure-azure + control-plane: capz-controller-manager + template: + metadata: + annotations: + kubectl.kubernetes.io/default-logs-container: manager + labels: + aadpodidbinding: capz-controller-aadpodidentity-selector + cluster.x-k8s.io/provider: infrastructure-azure + control-plane: capz-controller-manager + spec: + containers: + - args: + - --leader-elect + - --metrics-bind-addr=localhost:8080 + - --feature-gates=MachinePool=${EXP_MACHINE_POOL:=false},AKS=${EXP_AKS:=false} + - --v=0 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/spectro-dev-public/ubuntu/release/cluster-api-azure-controller:v1.3.2-spectro-4.0.0-dev + imagePullPolicy: Always + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + - containerPort: 9440 + name: healthz + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + terminationGracePeriodSeconds: 10 + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: capz-webhook-service-cert +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-azure + component: nmi + k8s-app: aad-pod-id + tier: node + name: capz-nmi + namespace: capi-webhook-system +spec: + selector: + matchLabels: + cluster.x-k8s.io/provider: infrastructure-azure + component: nmi + tier: node + template: + metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-azure + component: nmi + tier: node + spec: + containers: + - args: + - --node=$(NODE_NAME) + - --operation-mode=managed + - --forceNamespaced + - --http-probe-port=8085 + env: + - name: FORCENAMESPACED + value: "true" + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: LOG_LEVEL + value: DEBUG + image: mcr.microsoft.com/oss/azure/aad-pod-identity/nmi:v1.8.6 + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /healthz + port: 8085 + initialDelaySeconds: 10 + periodSeconds: 5 + name: nmi + resources: + limits: + cpu: 200m + memory: 512Mi + requests: + cpu: 100m + memory: 256Mi + securityContext: + capabilities: + add: + - DAC_READ_SEARCH + - NET_ADMIN + - NET_RAW + drop: + - ALL + runAsUser: 0 + volumeMounts: + - mountPath: /run/xtables.lock + name: iptableslock + - mountPath: /etc/default/kubelet + name: kubelet-config + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + volumes: + - hostPath: + path: /run/xtables.lock + type: FileOrCreate + name: iptableslock + - hostPath: + path: /etc/default/kubelet + type: FileOrCreate + name: kubelet-config + updateStrategy: + type: RollingUpdate +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-azure + name: capz-serving-cert + namespace: capi-webhook-system +spec: + dnsNames: + - capz-webhook-service.capi-webhook-system.svc + - capz-webhook-service.capi-webhook-system.svc.cluster.local + issuerRef: + kind: Issuer + name: capz-selfsigned-issuer + secretName: capz-webhook-service-cert +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-azure + name: capz-selfsigned-issuer + namespace: capi-webhook-system +spec: + selfSigned: {} +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-azure + name: capz-validating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-azurecluster + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.azurecluster.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azureclusters + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-azureclustertemplate + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.azureclustertemplate.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azureclustertemplates + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachine + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.azuremachine.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremachines + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachinetemplate + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.azuremachinetemplate.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremachinetemplates + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachinepool + failurePolicy: Fail + name: validation.azuremachinepool.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremachinepools + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachinepoolmachine + failurePolicy: Fail + name: azuremachinepoolmachine.kb.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremachinepoolmachines + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedcluster + failurePolicy: Fail + name: validation.azuremanagedclusters.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - UPDATE + resources: + - azuremanagedclusters + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedcontrolplane + failurePolicy: Fail + name: validation.azuremanagedcontrolplanes.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremanagedcontrolplanes + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedmachinepool + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.azuremanagedmachinepools.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - UPDATE + - DELETE + resources: + - azuremanagedmachinepools + sideEffects: None diff --git a/spectro/global/kustomization.yaml b/spectro/global/kustomization.yaml new file mode 100644 index 00000000000..1b70f5d7c4b --- /dev/null +++ b/spectro/global/kustomization.yaml @@ -0,0 +1,139 @@ +namespace: capi-webhook-system + +namePrefix: capz- + +# Labels to add to all resources and selectors. +commonLabels: + cluster.x-k8s.io/provider: "infrastructure-azure" + +resources: + - ../../config/default/namespace.yaml + - ../../config/default/aad-pod-identity-deployment.yaml + - ../../config/crd + - ../../config/webhook + - ../../config/manager + - ../../config/certmanager + +patchesStrategicMerge: + - ../../config/default/manager_image_patch.yaml + - ../../config/default/manager_pull_policy.yaml + - ../../config/default/manager_webhook_patch.yaml + - ../../config/default/webhookcainjection_patch.yaml + +vars: + - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + fieldref: + fieldpath: metadata.namespace + - name: CERTIFICATE_NAME + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + - name: SERVICE_NAMESPACE # namespace of the service + objref: + kind: Service + version: v1 + name: webhook-service + fieldref: + fieldpath: metadata.namespace + - name: SERVICE_NAME + objref: + kind: Service + version: v1 + name: webhook-service + +configurations: + - ../../config/default/kustomizeconfig.yaml + +patchesJson6902: + - target: + group: rbac.authorization.k8s.io + version: v1 + kind: ClusterRole + name: capz-aad-pod-id-nmi-role + path: patch_cluster_role.yaml + - target: + kind: Namespace + name: system + version: v1 + path: patch_namespace.yaml + - target: + group: apps + kind: Deployment + name: controller-manager + namespace: system + version: v1 + path: patch_service_account.yaml + - target: + group: apps + kind: DaemonSet + name: capz-nmi + namespace: capi-webhook-system + version: v1 + path: patch_service_account.yaml + - target: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: azureclusteridentities.infrastructure.cluster.x-k8s.io + path: patch_crd_webhook_namespace.yaml + - target: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: azureclusters.infrastructure.cluster.x-k8s.io + path: patch_crd_webhook_namespace.yaml + - target: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: azuremachines.infrastructure.cluster.x-k8s.io + path: patch_crd_webhook_namespace.yaml + - target: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: azuremachinetemplates.infrastructure.cluster.x-k8s.io + path: patch_crd_webhook_namespace.yaml + - target: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: azuremachinepools.infrastructure.cluster.x-k8s.io + path: patch_crd_webhook_namespace.yaml + - target: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: azuremachinepoolmachines.infrastructure.cluster.x-k8s.io + path: patch_crd_webhook_namespace.yaml + - target: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: azuremanagedclusters.infrastructure.cluster.x-k8s.io + path: patch_crd_webhook_namespace.yaml + - target: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: azuremanagedcontrolplanes.infrastructure.cluster.x-k8s.io + path: patch_crd_webhook_namespace.yaml + - target: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: azuremanagedmachinepools.infrastructure.cluster.x-k8s.io + path: patch_crd_webhook_namespace.yaml + - target: + group: rbac.authorization.k8s.io + version: v1 + kind: ClusterRoleBinding + name: capz-aad-pod-id-nmi-binding + path: patch_nmi_cluster_role_binding.yaml diff --git a/spectro/global/patch_cluster_role.yaml b/spectro/global/patch_cluster_role.yaml new file mode 100644 index 00000000000..8d81d5ed7f7 --- /dev/null +++ b/spectro/global/patch_cluster_role.yaml @@ -0,0 +1,21 @@ +- op: add + path: /rules/0 + value: + apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - azuremanagedclusters + - azuremanagedcontrolplanes + verbs: + - '*' +- op: add + path: /rules/0 + value: + apiGroups: + - cluster.x-k8s.io + resources: + - clusters + verbs: + - get + - list + - watch \ No newline at end of file diff --git a/spectro/global/patch_crd_webhook_namespace.yaml b/spectro/global/patch_crd_webhook_namespace.yaml new file mode 100644 index 00000000000..e40df94ba2c --- /dev/null +++ b/spectro/global/patch_crd_webhook_namespace.yaml @@ -0,0 +1,3 @@ +- op: replace + path: "/spec/conversion/webhook/clientConfig/service/namespace" + value: capi-webhook-system \ No newline at end of file diff --git a/spectro/global/patch_namespace.yaml b/spectro/global/patch_namespace.yaml new file mode 100644 index 00000000000..0de288003c7 --- /dev/null +++ b/spectro/global/patch_namespace.yaml @@ -0,0 +1,3 @@ +- op: replace + path: "/metadata/name" + value: capi-webhook-system \ No newline at end of file diff --git a/spectro/global/patch_nmi_cluster_role_binding.yaml b/spectro/global/patch_nmi_cluster_role_binding.yaml new file mode 100644 index 00000000000..e854a54e527 --- /dev/null +++ b/spectro/global/patch_nmi_cluster_role_binding.yaml @@ -0,0 +1,6 @@ +- op: replace + path: "/subjects/0/namespace" + value: capi-webhook-system +- op: replace + path: "/subjects/0/name" + value: default \ No newline at end of file diff --git a/spectro/global/patch_service_account.yaml b/spectro/global/patch_service_account.yaml new file mode 100644 index 00000000000..99d7e697b56 --- /dev/null +++ b/spectro/global/patch_service_account.yaml @@ -0,0 +1,2 @@ +- op: remove + path: "/spec/template/spec/serviceAccountName" \ No newline at end of file diff --git a/spectro/run.sh b/spectro/run.sh new file mode 100755 index 00000000000..95acb764911 --- /dev/null +++ b/spectro/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +rm generated/* + +kustomize build --load_restrictor none global > ./generated/core-global.yaml +kustomize build --load_restrictor none base > ./generated/core-base.yaml diff --git a/util/azure/azure.go b/util/azure/azure.go new file mode 100644 index 00000000000..79b9f5eedd2 --- /dev/null +++ b/util/azure/azure.go @@ -0,0 +1,37 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "fmt" + "regexp" + "strings" +) + +var azureResourceGroupNameRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/(?:.*)`) + +// ConvertResourceGroupNameToLower converts the resource group name in the resource ID to be lowered. +// Inspired by https://github.com/kubernetes-sigs/cloud-provider-azure/blob/88c9b89611e7c1fcbd39266928cce8406eb0e728/pkg/provider/azure_wrap.go#L409 +func ConvertResourceGroupNameToLower(resourceID string) (string, error) { + matches := azureResourceGroupNameRE.FindStringSubmatch(resourceID) + if len(matches) != 2 { + return "", fmt.Errorf("%q isn't in Azure resource ID format %q", resourceID, azureResourceGroupNameRE.String()) + } + + resourceGroup := matches[1] + return strings.Replace(resourceID, resourceGroup, strings.ToLower(resourceGroup), 1), nil +} diff --git a/util/azure/azure_test.go b/util/azure/azure_test.go new file mode 100644 index 00000000000..d2a6622c9ab --- /dev/null +++ b/util/azure/azure_test.go @@ -0,0 +1,85 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "testing" + + . "github.com/onsi/gomega" +) + +func TestConvertResourceGroupNameToLower(t *testing.T) { + tests := []struct { + desc string + resourceID string + expected string + expectError bool + }{ + { + desc: "empty string should report error", + resourceID: "", + expectError: true, + }, + { + desc: "resourceID not in Azure format should report error", + resourceID: "invalid-id", + expectError: true, + }, + { + desc: "providerID not in Azure format should report error", + resourceID: "azure://invalid-id", + expectError: true, + }, + { + desc: "resource group name in VM providerID should be converted", + resourceID: "azure:///subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myResourceGroupName/providers/Microsoft.Compute/virtualMachines/k8s-agent-AAAAAAAA-0", + expected: "azure:///subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myresourcegroupname/providers/Microsoft.Compute/virtualMachines/k8s-agent-AAAAAAAA-0", + }, + { + desc: "resource group name in VM resourceID should be converted", + resourceID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myResourceGroupName/providers/Microsoft.Compute/virtualMachines/k8s-agent-AAAAAAAA-0", + expected: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myresourcegroupname/providers/Microsoft.Compute/virtualMachines/k8s-agent-AAAAAAAA-0", + }, + { + desc: "resource group name in VMSS providerID should be converted", + resourceID: "azure:///subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myResourceGroupName/providers/Microsoft.Compute/virtualMachineScaleSets/myScaleSetName/virtualMachines/156", + expected: "azure:///subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myresourcegroupname/providers/Microsoft.Compute/virtualMachineScaleSets/myScaleSetName/virtualMachines/156", + }, + { + desc: "resource group name in VMSS resourceID should be converted", + resourceID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myResourceGroupName/providers/Microsoft.Compute/virtualMachineScaleSets/myScaleSetName/virtualMachines/156", + expected: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myresourcegroupname/providers/Microsoft.Compute/virtualMachineScaleSets/myScaleSetName/virtualMachines/156", + }, + } + + for _, test := range tests { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + var actual string + var err error + actual, err = ConvertResourceGroupNameToLower(test.resourceID) + if test.expectError { + g.Expect(err).NotTo(BeNil()) + } else { + g.Expect(err).To(BeNil()) + g.Expect(actual).To(Equal(test.expected)) + } + }) + } +} diff --git a/util/versions/version.go b/util/versions/version.go new file mode 100644 index 00000000000..4646c3c8cfe --- /dev/null +++ b/util/versions/version.go @@ -0,0 +1,26 @@ +package versions + +import ( + semverv4 "github.com/blang/semver" + "github.com/pkg/errors" +) + +// GetHigherK8sVersion returns the higher k8s version out of a and b. +func GetHigherK8sVersion(a, b string) (string, error) { + v1, errv1 := semverv4.ParseTolerant(a) + v2, errv2 := semverv4.ParseTolerant(b) + if errv1 != nil && errv2 != nil { + return "", errors.Wrapf(errv1, "error parsing k8s version %s, %v error parsing k8s version %s", a, errv2, b) + } + if errv1 != nil { + return b, nil + } + if errv2 != nil { + return a, nil + } + + if v1.GTE(v2) { + return a, nil + } + return b, nil +} diff --git a/util/versions/version_test.go b/util/versions/version_test.go new file mode 100644 index 00000000000..738b39e6ba4 --- /dev/null +++ b/util/versions/version_test.go @@ -0,0 +1,100 @@ +package versions + +import ( + "testing" + + . "github.com/onsi/gomega" +) + +func TestGetHigherK8sVersion(t *testing.T) { + cases := []struct { + name string + a string + b string + output string + expectErr bool + }{ + { + name: "a is greater than b", + a: "v1.17.8", + b: "v1.18.8", + output: "v1.18.8", + expectErr: false, + }, + { + name: "b is greater than a", + a: "v1.18.9", + b: "v1.18.8", + output: "v1.18.9", + expectErr: false, + }, + { + name: "b is greater than a", + a: "v1.18", + b: "v1.18.8", + output: "v1.18.8", + expectErr: false, + }, + { + name: "a is equal to b", + a: "v1.18.8", + b: "v1.18.8", + output: "v1.18.8", + expectErr: false, + }, + { + name: "a is greater than b and a is major.minor", + a: "v1.18", + b: "v1.17.8", + output: "v1.18", + expectErr: false, + }, + { + name: "a is greater than b and a is major.minor", + a: "1.18", + b: "1.17.8", + output: "1.18", + expectErr: false, + }, + { + name: "a is invalid", + a: "1.18.", + b: "v1.17.8", + output: "v1.17.8", + expectErr: false, + }, + { + name: "b is invalid", + a: "1.18.1", + b: "v1.17.8.", + output: "1.18.1", + expectErr: false, + }, + { + name: "b is invalid", + a: "9.99.9999", + b: "v1.17.8.", + output: "9.99.9999", + expectErr: false, + }, + { + name: "a & b is invalid", + a: "", + b: "v1.17.8.", + output: "", + expectErr: true, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + g := NewWithT(t) + output, err := GetHigherK8sVersion(c.a, c.b) + g.Expect(output).To(Equal(c.output)) + if c.expectErr { + g.Expect(err).NotTo(BeNil()) + } else { + g.Expect(err).To(BeNil()) + } + }) + } +}