use workload identity

Signed-off-by: Somtochi Onyekwere <[email protected]>

Author: somtochiama

go.mod

@@ -33,8 +33,8 @@ replace golang.org/x/text => golang.org/x/text v0.4.0
 require (
 	cloud.google.com/go/compute v1.10.0 // indirect
 	github.com/Azure/azure-sdk-for-go v67.0.0+incompatible // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.5.0-beta.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0-beta.4 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
@@ -45,7 +45,7 @@ require (
 	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v0.8.1 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 // indirect
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
 	github.com/aws/aws-sdk-go-v2 v1.17.5 // indirect
 	github.com/aws/aws-sdk-go-v2/config v1.18.15 // indirect
@@ -89,7 +89,7 @@ require (
 	github.com/go-openapi/jsonreference v0.20.0 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.4.3 // indirect
 	github.com/golang/glog v1.0.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
@@ -171,3 +171,5 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
+
+replace github.com/fluxcd/pkg/oci => github.com/weisdd/fluxcd-pkg/oci v0.0.0-20230216100018-1186e0dede79

go.sum

@@ -37,10 +37,10 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 h1:EKPd1
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0=
 github.com/Azure/azure-sdk-for-go v67.0.0+incompatible h1:SVBwznSETB0Sipd0uyGJr7khLhJOFRUEUb+0JgkCvDo=
 github.com/Azure/azure-sdk-for-go v67.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 h1:rTnT/Jrcm+figWlYz4Ixzt0SJVR2cMC8lvZcimipiEY=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.1 h1:T8quHYlUGyb/oqtSTwqlCr1ilJHrDv+ZtpSfo+hm1BU=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.1/go.mod h1:gLa1CL2RNE4s7M3yopJ/p0iq5DdY6Yv5ZUt9MTRZOQM=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.5.0-beta.1 h1:yLM4ZIC+NRvzwFGpXjUbf5FhPBVxJgmYXkjePgNAx64=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.5.0-beta.1/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0-beta.4 h1:jpSh2461XzXBEw1MJwvVRJwZS0CAgqS0h6jBdoIFtLk=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0-beta.4/go.mod h1:oWa/ZXP08smIi12UyWVbVikBxoZHZCyxijZamTK1i8Q=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
@@ -67,8 +67,8 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z
 github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
 github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
-github.com/AzureAD/microsoft-authentication-library-for-go v0.8.1 h1:oPdPEZFSbl7oSPEAIPMPBMUmiL+mqgzBJwM/9qYcwNg=
-github.com/AzureAD/microsoft-authentication-library-for-go v0.8.1/go.mod h1:4qFor3D/HDsvBME35Xy9rwW9DecL+M2sNw1ybjPtwA0=
+github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 h1:UE9n9rkJF62ArLb1F3DEjRt8O3jLwMWdSoypKV4f3MU=
+github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
@@ -210,8 +210,6 @@ github.com/fluxcd/pkg/apis/event v0.4.1 h1:63wP8NM/uA4680F4Ft8q8/0rJivX90i7FmMkR
 github.com/fluxcd/pkg/apis/event v0.4.1/go.mod h1:LHT1ZsbMrcHwCHQCaFtQviQBZwhMOAbTUPK6+KgBkFo=
 github.com/fluxcd/pkg/apis/meta v0.19.1 h1:fCI5CnTXpAqr67UlaI9q0H+OztMKB5kDTr6xV6vlAo0=
 github.com/fluxcd/pkg/apis/meta v0.19.1/go.mod h1:ZPPMYrPnWwPQYNEGM/Uc0N4SurUPS3xNI3IIpCQEfuM=
-github.com/fluxcd/pkg/oci v0.21.1 h1:9kn19wkabE2xB77NRlOtMJlSYhZmUjdloZCzlHdAS6s=
-github.com/fluxcd/pkg/oci v0.21.1/go.mod h1:9E2DBlQII7YmeWt2ieTh38wwkiBqx3yg5NEJ51uefaA=
 github.com/fluxcd/pkg/runtime v0.31.0 h1:addyXaANHl/A68bEjCbiR4HzcFKgfXv1eaG7B7ZHxOo=
 github.com/fluxcd/pkg/runtime v0.31.0/go.mod h1:toGOOubMo4ZC1aWhB8C3drdTglr1/A1dETeNwjiIv0g=
 github.com/fluxcd/pkg/version v0.2.1 h1:RRH7+6qiWHdTvRNwpoBmilnubJ2C4FZYGgy5wTDVKVc=
@@ -256,8 +254,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
-github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs=
-github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
+github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU=
+github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
@@ -525,6 +523,8 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT
 github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME=
 github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI=
+github.com/weisdd/fluxcd-pkg/oci v0.0.0-20230216100018-1186e0dede79 h1:4BWUOxxtn2LhWKGBPrm+yJEL3EofGv6QAb4Z+FO3Mco=
+github.com/weisdd/fluxcd-pkg/oci v0.0.0-20230216100018-1186e0dede79/go.mod h1:GXQ3mmh3DX7RsEt2btj1x+XEu2s/OC0HHlQ/OnVre2U=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=

tests/integration/aws_test.go

@@ -69,3 +69,9 @@ func pushFluxTestImagesECR(ctx context.Context, localImgs map[string]string, out
 	remoteImage := repo + ":test"
 	return tftestenv.PushTestAppImagesECR(ctx, localImgs, remoteImage)
 }
+
+// getKustomizePatchesAWS return the patches that should be added to the kustomization.yaml
+// before deploying Flux. It returns an empty array since no patches are needed
+func getKustomizePatchesAWS(output map[string]*tfjson.StateOutput) []string {
+	return nil
+}

tests/integration/azure_test.go

@@ -43,6 +43,11 @@ func registryLoginACR(ctx context.Context, output map[string]*tfjson.StateOutput
 	testRepos := map[string]string{}
 
 	registryURL := output["acr_registry_url"].Value.(string)
+	fluxRegistryURL := output["flux_acr_registry_url"].Value.(string)
+	if err := tftestenv.RegistryLoginACR(ctx, fluxRegistryURL); err != nil {
+		return nil, err
+	}
+
 	if err := tftestenv.RegistryLoginACR(ctx, registryURL); err != nil {
 		return nil, err
 	}
@@ -56,6 +61,40 @@ func registryLoginACR(ctx context.Context, output map[string]*tfjson.StateOutput
 // logged in and is capable of pushing the test images.
 func pushFluxTestImagesACR(ctx context.Context, localImgs map[string]string, output map[string]*tfjson.StateOutput) (map[string]string, error) {
 	// Get the registry name and construct the image names accordingly.
-	registryURL := output["acr_registry_url"].Value.(string)
+	registryURL := output["flux_acr_registry_url"].Value.(string)
 	return tftestenv.PushTestAppImagesACR(ctx, localImgs, registryURL)
 }
+
+// getKustomizePatchesAzure return the patches that should be added to the kustomization.yaml
+// before deploying Flux. It returns two patches, one to annotate the image-reflector-controller
+// service account and the other for the image-reflector-controller deployment. These are needed
+// for workload identity to work properly on Azure
+func getKustomizePatchesAzure(output map[string]*tfjson.StateOutput) []string {
+	appClientId := output["spn_id"].Value.(string)
+	saAnnotation := `
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: image-reflector-controller
+  namespace: flux-system
+  annotations:
+    azure.workload.identity/client-id: "%s"
+  labels:
+    azure.workload.identity/use: "true"
+`
+	saPatch := fmt.Sprintf(saAnnotation, appClientId)
+	deployPatch := `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: image-reflector-controller
+  namespace: flux-system
+  labels:
+    azure.workload.identity/use: "true"
+spec:
+  template:
+    metadata:
+      labels:
+        azure.workload.identity/use: "true"
+	`
+	return []string{deployPatch, saPatch}
+}

tests/integration/gcp_test.go

@@ -69,3 +69,9 @@ func pushFluxTestImagesGCR(ctx context.Context, localImgs map[string]string, out
 	repositoryID := output["gcp_artifact_repository"].Value.(string)
 	return tftestenv.PushTestAppImagesGCR(ctx, localImgs, project, region, repositoryID)
 }
+
+// getKustomizePatchesGCP return the patches that should be added to the kustomization.yaml
+// before deploying Flux. It returns an empty array since no patches are needed
+func getKustomizePatchesGCP(output map[string]*tfjson.StateOutput) []string {
+	return nil
+}

tests/integration/suite_test.go

@@ -98,6 +98,12 @@ type registryLoginFunc func(ctx context.Context, output map[string]*tfjson.State
 // pushed to a corresponding registry repository for the image.
 type pushTestImages func(ctx context.Context, localImgs map[string]string, output map[string]*tfjson.StateOutput) (map[string]string, error)
 
+// getKustomizePatches is used to return provider specific kustomize patches
+// that would be added to the kustomization.yaml before Flux is deployed. It takes
+// in a terraform state output as some value might be needed for
+// creating the patch,
+type getKustomizePatches func(output map[string]*tfjson.StateOutput) []string
+
 // ProviderConfig is the test configuration of a supported cloud provider to run
 // the tests against.
 type ProviderConfig struct {
@@ -110,6 +116,8 @@ type ProviderConfig struct {
 	createKubeconfig tftestenv.CreateKubeconfig
 	// pushFluxTestImages is used to push flux test images to a remote registry.
 	pushFluxTestImages pushTestImages
+	// getKustomizePatches is used to get provider specific kustomize patches
+	getKustomizePatches getKustomizePatches
 }
 
 func init() {
@@ -198,8 +206,10 @@ func TestMain(m *testing.M) {
 		panic(fmt.Sprintf("Failed to create and push images: %v", err))
 	}
 
+	patches := providerCfg.getKustomizePatches(output)
+
 	// Update flux install manifests with the pushed test images.
-	if err := updateAndBuildFluxInstallManifests(ctx, pushedImages); err != nil {
+	if err := updateAndBuildFluxInstallManifests(ctx, pushedImages, patches); err != nil {
 		panic(fmt.Sprintf("Failed to update and build flux install manifests: %v", err))
 	}
 
@@ -224,24 +234,27 @@ func getProviderConfig(provider string) *ProviderConfig {
 	switch provider {
 	case "aws":
 		return &ProviderConfig{
-			terraformPath:      terraformPathAWS,
-			registryLogin:      registryLoginECR,
-			pushFluxTestImages: pushFluxTestImagesECR,
-			createKubeconfig:   createKubeconfigEKS,
+			terraformPath:       terraformPathAWS,
+			registryLogin:       registryLoginECR,
+			pushFluxTestImages:  pushFluxTestImagesECR,
+			createKubeconfig:    createKubeconfigEKS,
+			getKustomizePatches: getKustomizePatchesAWS,
 		}
 	case "azure":
 		return &ProviderConfig{
-			terraformPath:      terraformPathAzure,
-			registryLogin:      registryLoginACR,
-			pushFluxTestImages: pushFluxTestImagesACR,
-			createKubeconfig:   createKubeConfigAKS,
+			terraformPath:       terraformPathAzure,
+			registryLogin:       registryLoginACR,
+			pushFluxTestImages:  pushFluxTestImagesACR,
+			createKubeconfig:    createKubeConfigAKS,
+			getKustomizePatches: getKustomizePatchesAzure,
 		}
 	case "gcp":
 		return &ProviderConfig{
-			terraformPath:      terraformPathGCP,
-			registryLogin:      registryLoginGCR,
-			pushFluxTestImages: pushFluxTestImagesGCR,
-			createKubeconfig:   createKubeconfigGKE,
+			terraformPath:       terraformPathGCP,
+			registryLogin:       registryLoginGCR,
+			pushFluxTestImages:  pushFluxTestImagesGCR,
+			createKubeconfig:    createKubeconfigGKE,
+			getKustomizePatches: getKustomizePatchesGCP,
 		}
 	}
 	return nil

tests/integration/terraform/azure/main.tf

@@ -13,19 +13,73 @@ locals {
 }
 
 module "aks" {
-  source = "git::https://github.com/fluxcd/test-infra.git//tf-modules/azure/aks"
+  source = "git::https://github.com/somtochiama/test-infra.git//tf-modules/azure/aks?ref=az-workload"
 
   name     = local.name
   location = var.azure_location
 }
 
 module "acr" {
-  source = "git::https://github.com/fluxcd/test-infra.git//tf-modules/azure/acr"
+  source = "git::https://github.com/somtochiama/test-infra.git//tf-modules/azure/acr?ref=az-workload"
 
   name             = local.name
   location         = var.azure_location
-  aks_principal_id = module.aks.principal_id
   resource_group   = module.aks.resource_group
+}
+
+module "acr_flux" {
+  source = "git::https://github.com/somtochiama/test-infra.git//tf-modules/azure/acr?ref=az-workload"
+
+  name             = "manager${random_pet.suffix.id}"
+  location         = var.azure_location
+  resource_group   = module.aks.resource_group
+  aks_principal_id = [module.aks.principal_id]
 
   depends_on = [module.aks]
 }
+
+resource "azuread_application" "flux" {
+  display_name = "acr-sp"
+
+  required_resource_access {
+    resource_app_id = "00000003-0000-0000-c000-000000000000"
+
+    resource_access {
+      id   = "df021288-bdef-4463-88db-98f22de89214"
+      type = "Role"
+    }
+  }
+
+  required_resource_access {
+    resource_app_id = "00000002-0000-0000-c000-000000000000"
+
+    resource_access {
+      id   = "1cda74f2-2616-4834-b122-5cb1b07f8a59"
+      type = "Role"
+    }
+    resource_access {
+      id   = "78c8a3c8-a07e-4b9e-af1b-b5ccab50a175"
+      type = "Role"
+    }
+  }
+}
+
+resource "azuread_service_principal" "flux" {
+  application_id = azuread_application.flux.application_id
+}
+
+resource "azurerm_role_assignment" "acr" {
+  scope                =  module.acr.registry_id
+  role_definition_name = "AcrPull"
+  principal_id         = azuread_service_principal.flux.object_id
+}
+
+
+resource "azuread_application_federated_identity_credential" "example" {
+  application_object_id = azuread_application.flux.object_id
+  display_name          = "image-reflector-sa"
+  description           = "Kubernetes service account federated credential"
+  audiences             = ["api://AzureADTokenExchange"]
+  issuer                = module.aks.cluster_oidc_url
+  subject               = "system:serviceaccount:flux-system:image-reflector-controller"
+}

tests/integration/terraform/azure/outputs.tf

@@ -10,3 +10,15 @@ output "acr_registry_url" {
 output "acr_registry_id" {
   value = module.acr.registry_id
 }
+
+output "flux_acr_registry_url" {
+  value = module.acr_flux.registry_url
+}
+
+output "flux_acr_registry_id" {
+  value = module.acr_flux.registry_id
+}
+
+output "spn_id" {
+  value = azuread_application.flux.application_id
+}

tests/integration/utils.go

@@ -19,18 +19,16 @@ package integration
 import (
 	"context"
 	"fmt"
-	"log"
+	"github.com/fluxcd/test-infra/tftestenv"
 	"path"
 	"strings"
-
-	"github.com/fluxcd/test-infra/tftestenv"
 )
 
 // updateAndBuildFluxInstallManifests assumes that ./build/flux/ already exists
 // with downloaded install.yaml and copied kustomization.yaml. It updates the
 // kustomization.yaml with new test images and builds a new install manifest
 // at ./build/flux.yaml.
-func updateAndBuildFluxInstallManifests(ctx context.Context, images map[string]string) error {
+func updateAndBuildFluxInstallManifests(ctx context.Context, images map[string]string, patches []string) error {
 	// Construct kustomize set image arguments.
 	setImgArgs := []string{}
 	for name, img := range images {
@@ -40,7 +38,17 @@ func updateAndBuildFluxInstallManifests(ctx context.Context, images map[string]s
 		arg := fmt.Sprintf("%s=%s", imageName, img)
 		setImgArgs = append(setImgArgs, arg)
 	}
-	log.Println("setting images:", setImgArgs)
+
+	for _, patch := range patches {
+		// add patches to kustomization.
+		err := tftestenv.RunCommand(ctx, "./build/flux",
+			fmt.Sprintf("kustomize edit add patch --patch '%s'", patch),
+			tftestenv.RunCommandOptions{},
+		)
+		if err != nil {
+			return err
+		}
+	}
 
 	// Update all the images in kustomization.
 	err := tftestenv.RunCommand(ctx, "./build/flux",