From 0688b36e0cefb3b75b92e152bc98c5d55b2ba46d Mon Sep 17 00:00:00 2001 From: Jonathan West Date: Fri, 20 Sep 2024 10:48:30 -0400 Subject: [PATCH 1/9] Allow GitOps Service to run on standalone Argo CD Signed-off-by: Jonathan West --- Makefile | 5 +- argocd.sh | 37 +++++- .../utils/argocd_login_credentials.go | 13 ++- manifests/k8s-argo-cd/kustomization.yaml | 29 +++++ tests-e2e/.gitignore | 2 +- tests-e2e/Makefile | 2 +- .../argocd_gitopsengine_instance_test.go | 2 + tests-e2e/argocd/argocd_instance_test.go | 3 + .../core/gitopsdeployment_status_test.go | 12 -- .../core/gitopsdeployment_syncrun_test.go | 15 ++- tests-e2e/core/gitopsdeployment_test.go | 12 -- .../core/managed_environment_status_test.go | 6 +- tests-e2e/core/managed_environment_test.go | 69 +---------- tests-e2e/fixture/fixture.go | 107 +++++++++++++++++- tests-e2e/go.mod | 2 +- 15 files changed, 209 insertions(+), 107 deletions(-) create mode 100644 manifests/k8s-argo-cd/kustomization.yaml diff --git a/Makefile b/Makefile index d09ff5de4c..b555bf397e 100644 --- a/Makefile +++ b/Makefile @@ -141,7 +141,8 @@ install-argocd-openshift: kustomize ## Using OpenShift GitOps, install Argo CD t PATH=$(MAKEFILE_ROOT)/bin:$(PATH) $(MAKEFILE_ROOT)/manifests/scripts/openshift-argo-deploy/deploy.sh install-argocd-k8s: ## (Non-OpenShift): Install Argo CD to the gitops-service-argocd namespace - ARGO_CD_VERSION=$(ARGO_CD_VERSION) manifests/scripts/k8s-argo-deploy/deploy.sh + ARGO_CD_VERSION=$(ARGO_CD_VERSION) ./argocd.sh install + # manifests/scripts/k8s-argo-deploy/deploy.sh uninstall-argocd: ## Uninstall Argo CD from gitops-service-argocd namespace (from either OpenShift or K8s) kubectl delete namespace "$(ARGO_CD_NAMESPACE)" || true @@ -197,6 +198,8 @@ test: test-backend test-backend-shared test-cluster-agent test-appstudio-control setup-e2e-openshift: install-argocd-openshift devenv-k8s-e2e ## Setup steps for E2E tests to run with Openshift CI +setup-e2e-local-k8s: install-argocd-k8s devenv-docker reset-db ## Setup steps for E2E tests to run with Local Openshift Cluster + setup-e2e-local: install-argocd-openshift devenv-docker reset-db ## Setup steps for E2E tests to run with Local Openshift Cluster start-e2e: ## Start the managed gitops processes for E2E tests. diff --git a/argocd.sh b/argocd.sh index b317a40f77..81baba8acd 100755 --- a/argocd.sh +++ b/argocd.sh @@ -4,6 +4,15 @@ ARGO_CD_VERSION="${ARGO_CD_VERSION:-stable}" ARGO_CD_NAMESPACE="${ARGO_CD_NAMESPACE:-gitops-service-argocd}" ARGO_CD_PORT="${ARGO_CD_PORT:-4000}" + +SCRIPT_PATH="$( + cd -- "$(dirname "$0")" >/dev/null 2>&1 || exit + pwd -P +)" + +export ROOT_PATH=$SCRIPT_PATH + + # This script installs ArgoCD vanilla according to https://argo-cd.readthedocs.io/en/stable/getting_started/ # Checks if a binary is present on the local system @@ -16,19 +25,36 @@ exit_if_binary_not_installed() { done } +# Install Argo CD to target namespace, using kustomize. +# - Kustomize ensures that the ClusterRoleBindings are pointing to the ServiceAccount in the proper namespace. +build_argo_cd_manifests() { + KUSTOMIZE_TMP_DIR=$(mktemp -d) + cp -r $ROOT_PATH/manifests/k8s-argo-cd/* $KUSTOMIZE_TMP_DIR + + mv $KUSTOMIZE_TMP_DIR/kustomization.yaml $KUSTOMIZE_TMP_DIR/kustomization.yaml.old + + cat $KUSTOMIZE_TMP_DIR/kustomization.yaml.old | ARGO_CD_NAMESPACE=$ARGO_CD_NAMESPACE ARGO_CD_VERSION=$ARGO_CD_VERSION envsubst > $KUSTOMIZE_TMP_DIR/kustomization.yaml + +} + # Install 'ArgoCD Web UI' in your Kubernetes cluster if [ "$1" = "install" ]; then exit_if_binary_not_installed "kubectl" + exit_if_binary_not_installed "kustomize" if kubectl -n "$ARGO_CD_NAMESPACE" get pods | grep argocd-server | grep '1/1' | grep 'Running' &>/dev/null; then echo "ArgoCD is already running..." echo "Skipping ArgoCD setup." - exit 1 + exit 0 fi # Apply the argo-cd-route manifest kubectl create namespace "$ARGO_CD_NAMESPACE" || true - kubectl apply -n "$ARGO_CD_NAMESPACE" -f https://raw.githubusercontent.com/argoproj/argo-cd/$ARGO_CD_VERSION/manifests/install.yaml + + # Build and apply the Argo CD manifests + build_argo_cd_manifests + + kustomize build $KUSTOMIZE_TMP_DIR | kubectl apply -f - # Get the secret counter=0 @@ -104,5 +130,10 @@ fi # Remove 'ArgoCD Web UI' from your Kubernetes cluster if [ "$1" = "remove" ]; then exit_if_binary_not_installed "kubectl" - kubectl delete -n "$ARGO_CD_NAMESPACE" -f https://raw.githubusercontent.com/argoproj/argo-cd/$ARGO_CD_VERSION/manifests/install.yaml + exit_if_binary_not_installed "kustomize" + + build_argo_cd_manifests + + kustomize build $KUSTOMIZE_TMP_DIR | kubectl delete -f - + kubectl delete namespace "$ARGO_CD_NAMESPACE" || true fi diff --git a/cluster-agent/utils/argocd_login_credentials.go b/cluster-agent/utils/argocd_login_credentials.go index cc349bcb4e..297016181b 100644 --- a/cluster-agent/utils/argocd_login_credentials.go +++ b/cluster-agent/utils/argocd_login_credentials.go @@ -4,16 +4,17 @@ import ( "context" "errors" "fmt" + "os" "strings" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" argocdclient "github.com/argoproj/argo-cd/v2/pkg/apiclient" "github.com/argoproj/argo-cd/v2/util/grpc" "github.com/go-logr/logr" "github.com/golang/protobuf/ptypes/empty" routev1 "github.com/openshift/api/route/v1" - "k8s.io/apimachinery/pkg/util/intstr" sharedutil "github.com/redhat-appstudio/managed-gitops/backend-shared/util" logutil "github.com/redhat-appstudio/managed-gitops/backend-shared/util/log" @@ -221,9 +222,14 @@ func (cs *CredentialService) getCredentialsFromNamespace(req credentialRequest, return nil, nil, errors.New("no Argo CD admin passwords found in " + req.namespaceName) } + var serverHostName string + // Retrieve the Argo CD host name from the Route - serverHostName := "" - { + + argoCDServerAddrEnvVar := strings.TrimSpace(os.Getenv("ARGO_CD_SERVER_ADDR")) + if len(argoCDServerAddrEnvVar) > 0 { + serverHostName = argoCDServerAddrEnvVar + } else { routeList := &routev1.RouteList{} @@ -249,6 +255,7 @@ func (cs *CredentialService) getCredentialsFromNamespace(req credentialRequest, } } + if serverHostName == "" { return nil, nil, errors.New("Unable to locate Route in " + req.namespaceName) } diff --git a/manifests/k8s-argo-cd/kustomization.yaml b/manifests/k8s-argo-cd/kustomization.yaml new file mode 100644 index 0000000000..736e043992 --- /dev/null +++ b/manifests/k8s-argo-cd/kustomization.yaml @@ -0,0 +1,29 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- https://raw.githubusercontent.com/argoproj/argo-cd/$ARGO_CD_VERSION/manifests/install.yaml + +namespace: $ARGO_CD_NAMESPACE + +patchesStrategicMerge: +- |- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: argocd-application-controller + subjects: + - kind: ServiceAccount + name: argocd-application-controller + namespace: $ARGO_CD_NAMESPACE + +- |- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: argocd-server + subjects: + - kind: ServiceAccount + name: argocd-server + namespace: $ARGO_CD_NAMESPACE + diff --git a/tests-e2e/.gitignore b/tests-e2e/.gitignore index 6029f8a4b8..8c8626ab9f 100644 --- a/tests-e2e/.gitignore +++ b/tests-e2e/.gitignore @@ -3,5 +3,5 @@ cover.out vendor bin coverage.out - +ginkgo.report diff --git a/tests-e2e/Makefile b/tests-e2e/Makefile index 3d3b2b512b..4d87781fe9 100644 --- a/tests-e2e/Makefile +++ b/tests-e2e/Makefile @@ -33,7 +33,7 @@ help: ## Display this help. .PHONY: test test: ## Run E2E tests. - ENABLE_APPPROJECT_ISOLATION="true" DEV_ONLY_ALLOW_NON_TLS_CONNECTION_TO_POSTGRESQL=true go test -v -p=1 -timeout=100m -race -count=1 -coverprofile=coverage.out ./... + ENABLE_APPPROJECT_ISOLATION="true" DEV_ONLY_ALLOW_NON_TLS_CONNECTION_TO_POSTGRESQL=true go test -v -p=1 -timeout=100m -race -count=1 -coverprofile=coverage.out ./core/ ./argocd/ # go-get-tool will 'go install' any package $2 and install it to $1. PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) diff --git a/tests-e2e/argocd/argocd_gitopsengine_instance_test.go b/tests-e2e/argocd/argocd_gitopsengine_instance_test.go index 57cb213a33..1a1cc99889 100644 --- a/tests-e2e/argocd/argocd_gitopsengine_instance_test.go +++ b/tests-e2e/argocd/argocd_gitopsengine_instance_test.go @@ -31,6 +31,8 @@ var _ = Describe("ArgoCD instance via GitOpsEngineInstance Operations Test", fun It("ensures that a standalone ArgoCD gets created successfully when an operation CR of resource-type GitOpsEngineInstance is created", func() { + fixture.SkipIfArgoCDOperandRequired() + dbq, err := db.NewUnsafePostgresDBQueries(true, true) Expect(err).ToNot(HaveOccurred()) defer dbq.CloseDatabase() diff --git a/tests-e2e/argocd/argocd_instance_test.go b/tests-e2e/argocd/argocd_instance_test.go index 1f1150d1e9..c5cdf2599e 100644 --- a/tests-e2e/argocd/argocd_instance_test.go +++ b/tests-e2e/argocd/argocd_instance_test.go @@ -44,6 +44,9 @@ var _ = Describe("Standalone ArgoCD instance E2E tests", func() { }) testStandaloneArgoCD := func(testUpdate bool) { + + fixture.SkipIfArgoCDOperandRequired() + By("creating ArgoCD resource") ctx := context.Background() log := log.FromContext(ctx) diff --git a/tests-e2e/core/gitopsdeployment_status_test.go b/tests-e2e/core/gitopsdeployment_status_test.go index 028e1344a1..631a11976b 100644 --- a/tests-e2e/core/gitopsdeployment_status_test.go +++ b/tests-e2e/core/gitopsdeployment_status_test.go @@ -46,18 +46,6 @@ var _ = Describe("GitOpsDeployment Status Tests", func() { Status: managedgitopsv1alpha1.HeathStatusCodeHealthy, }, }, - { - Group: "route.openshift.io", - Version: "v1", - Kind: "Route", - Namespace: fixture.GitOpsServiceE2ENamespace, - Name: name, - Status: managedgitopsv1alpha1.SyncStatusCodeSynced, - Health: &managedgitopsv1alpha1.HealthStatus{ - Status: managedgitopsv1alpha1.HeathStatusCodeHealthy, - Message: "Route is healthy", - }, - }, { Group: "", Version: "v1", diff --git a/tests-e2e/core/gitopsdeployment_syncrun_test.go b/tests-e2e/core/gitopsdeployment_syncrun_test.go index a34b1baaf1..4fb67bd076 100644 --- a/tests-e2e/core/gitopsdeployment_syncrun_test.go +++ b/tests-e2e/core/gitopsdeployment_syncrun_test.go @@ -22,6 +22,7 @@ import ( ) var _ = Describe("GitOpsDeploymentSyncRun E2E tests", func() { + Context("Create, update and delete GitOpsDeploymentSyncRun", func() { var ( @@ -65,10 +66,14 @@ var _ = Describe("GitOpsDeploymentSyncRun E2E tests", func() { }) AfterEach(func() { - err := k8sClient.Get(ctx, client.ObjectKeyFromObject(argocdCR), argocdCR) - Expect(err == nil || apierr.IsNotFound(err)).To(BeTrue()) - removeCustomHealthCheckForDeployment(ctx, k8sClient, argocdCR) + if fixture.IsArgoCDOperandAvailable() { + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(argocdCR), argocdCR) + Expect(err == nil || apierr.IsNotFound(err)).To(BeTrue()) + + removeCustomHealthCheckForDeployment(ctx, k8sClient, argocdCR) + } + }) It("creating a new GitOpsDeploymentSyncRun should sync an Argo CD Application", func() { @@ -197,6 +202,8 @@ var _ = Describe("GitOpsDeploymentSyncRun E2E tests", func() { }) It("deleting the GitOpsDeploymentSyncRun should terminate a running Sync operation", func() { + fixture.SkipIfArgoCDOperandRequired() + gitOpsDeploymentResource = gitopsDeplFixture.BuildGitOpsDeploymentResource("test-deply-with-presync", "https://github.com/managed-gitops-test-data/deployment-presync-hook", "guestbook", managedgitopsv1alpha1.GitOpsDeploymentSpecType_Manual) @@ -314,6 +321,8 @@ var _ = Describe("GitOpsDeploymentSyncRun E2E tests", func() { It("should sync if the previous sync operation is terminated", func() { + fixture.SkipIfArgoCDOperandRequired() + gitOpsDeploymentResource = gitopsDeplFixture.BuildGitOpsDeploymentResource("test-deply-with-presync", "https://github.com/managed-gitops-test-data/deployment-presync-hook", "guestbook", managedgitopsv1alpha1.GitOpsDeploymentSpecType_Manual) diff --git a/tests-e2e/core/gitopsdeployment_test.go b/tests-e2e/core/gitopsdeployment_test.go index 677892a800..c41dbc090f 100644 --- a/tests-e2e/core/gitopsdeployment_test.go +++ b/tests-e2e/core/gitopsdeployment_test.go @@ -76,18 +76,6 @@ var _ = Describe("GitOpsDeployment E2E tests", func() { Status: managedgitopsv1alpha1.HeathStatusCodeHealthy, }, }, - { - Group: "route.openshift.io", - Version: "v1", - Kind: "Route", - Namespace: fixture.GitOpsServiceE2ENamespace, - Name: name, - Status: managedgitopsv1alpha1.SyncStatusCodeSynced, - Health: &managedgitopsv1alpha1.HealthStatus{ - Status: managedgitopsv1alpha1.HeathStatusCodeHealthy, - Message: "Route is healthy", - }, - }, { Group: "", Version: "v1", diff --git a/tests-e2e/core/managed_environment_status_test.go b/tests-e2e/core/managed_environment_status_test.go index 1497cf9d18..9a40323264 100644 --- a/tests-e2e/core/managed_environment_status_test.go +++ b/tests-e2e/core/managed_environment_status_test.go @@ -43,8 +43,12 @@ var _ = Describe("Managed Environment Status E2E tests", func() { Expect(managedEnv.Status.Conditions).To(HaveLen(1)) condition := managedEnv.Status.Conditions[0] Expect(condition.Status).To(Equal(metav1.ConditionFalse)) - Expect(condition.Reason).To(Equal(string(managedgitopsv1alpha1.ConditionReasonUnableToCreateClient))) + Expect(condition.Reason).To(Or( + Equal(string(managedgitopsv1alpha1.ConditionReasonUnableToCreateClient)), + Equal(string(managedgitopsv1alpha1.ConditionReasonUnableToInstallServiceAccount)), + )) Expect(condition.Message).To(ContainSubstring("no such host")) + }) It("should have a connection status condition of False when the credentials are missing a required field", func() { diff --git a/tests-e2e/core/managed_environment_test.go b/tests-e2e/core/managed_environment_test.go index a54f1d8486..4333c309ee 100644 --- a/tests-e2e/core/managed_environment_test.go +++ b/tests-e2e/core/managed_environment_test.go @@ -2,8 +2,6 @@ package core import ( "context" - "fmt" - "os" "time" "github.com/argoproj/argo-cd/v2/common" @@ -14,7 +12,6 @@ import ( "github.com/redhat-appstudio/managed-gitops/backend-shared/util" "github.com/redhat-appstudio/managed-gitops/tests-e2e/fixture" "github.com/redhat-appstudio/managed-gitops/tests-e2e/fixture/k8s" - "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/controller-runtime/pkg/client" appv1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" @@ -183,7 +180,7 @@ var _ = Describe("GitOpsDeployment Managed Environment E2E tests", func() { By("creating the GitOpsDeploymentManagedEnvironment and Secret") - _, apiServerURL, err := extractKubeConfigValues() + _, apiServerURL, err := fixture.ExtractKubeConfigValues() Expect(err).ToNot(HaveOccurred()) kubeConfigContents := k8s.GenerateKubeConfig(apiServerURL, fixture.GitOpsServiceE2ENamespace, tokenSecret) @@ -319,7 +316,7 @@ var _ = Describe("GitOpsDeployment Managed Environment E2E tests", func() { By("creating the GitOpsDeploymentManagedEnvironment") - _, apiServerURL, err := extractKubeConfigValues() + _, apiServerURL, err := fixture.ExtractKubeConfigValues() Expect(err).ToNot(HaveOccurred()) // Set the tokenSecret to be "" to intentionally fail @@ -407,7 +404,7 @@ var _ = Describe("GitOpsDeployment Managed Environment E2E tests", func() { By("creating the GitOpsDeploymentManagedEnvironment") - _, apiServerURL, err := extractKubeConfigValues() + _, apiServerURL, err := fixture.ExtractKubeConfigValues() Expect(err).ToNot(HaveOccurred()) kubeConfigContents := k8s.GenerateKubeConfig(apiServerURL, fixture.GitOpsServiceE2ENamespace, tokenSecret) @@ -524,7 +521,7 @@ var _ = Describe("GitOpsDeployment Managed Environment E2E tests", func() { By("creating the GitOpsDeploymentManagedEnvironment and its Secret, using that service account token") - _, apiServerURL, err := extractKubeConfigValues() + _, apiServerURL, err := fixture.ExtractKubeConfigValues() Expect(err).ToNot(HaveOccurred()) kubeConfigContents := k8s.GenerateKubeConfig(apiServerURL, newNamespace.Name, tokenSecret) @@ -981,61 +978,3 @@ var _ = Describe("GitOpsDeployment Managed Environment E2E tests", func() { }) }) - -// extractKubeConfigValues returns contents of k8s config from $KUBE_CONFIG, plus server api url (and error) -func extractKubeConfigValues() (string, string, error) { - - loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() - - config, err := loadingRules.Load() - if err != nil { - return "", "", err - } - - context, ok := config.Contexts[config.CurrentContext] - if !ok || context == nil { - return "", "", fmt.Errorf("no context") - } - - cluster, ok := config.Clusters[context.Cluster] - if !ok || cluster == nil { - return "", "", fmt.Errorf("no cluster") - } - - var kubeConfigDefault string - - paths := loadingRules.Precedence - { - - for _, path := range paths { - - GinkgoWriter.Println("Attempting to read kube config from", path) - - // homeDir, err := os.UserHomeDir() - // if err != nil { - // return "", "", err - // } - - _, err = os.Stat(path) - if err != nil { - GinkgoWriter.Println("Unable to resolve path", path, err) - } else { - // Success - kubeConfigDefault = path - break - } - - } - - if kubeConfigDefault == "" { - return "", "", fmt.Errorf("unable to retrieve kube config path") - } - } - - kubeConfigContents, err := os.ReadFile(kubeConfigDefault) - if err != nil { - return "", "", err - } - - return string(kubeConfigContents), cluster.Server, nil -} diff --git a/tests-e2e/fixture/fixture.go b/tests-e2e/fixture/fixture.go index 5a357a7038..8c041e175f 100644 --- a/tests-e2e/fixture/fixture.go +++ b/tests-e2e/fixture/fixture.go @@ -24,6 +24,7 @@ import ( apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + apiexts "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierr "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -60,7 +61,7 @@ const ( // EnableNamespaceBackedArgoCD: When enabled, this will ensure that certain tests/fixtures add the managed-by label to Namespace. This allows the gitops-service-argocd instance to manage them. // // When Argo CD is running in cluster-scoped mode (i.e. due to concerns with performance issues when Argo CD is running in namespace-scoped mode), this field should be set to false. - EnableNamespaceBackedArgoCD = true + EnableNamespaceBackedArgoCD = false ) // EnsureCleanSlateNonKCPVirtualWorkspace should be called before every E2E tests: @@ -459,6 +460,49 @@ func removeAllFinalizers(k8sClient client.Client, obj client.Object) error { return nil } +func isArgoCDClusterScoped(argoCDNamespaceParam string, clientConfig *rest.Config) (bool, error) { + + k8sClient, err := GetKubeClient(clientConfig) + if err != nil { + return false, err + } + + saList := &corev1.ServiceAccountList{} + + if err := k8sClient.List(context.Background(), saList, &client.ListOptions{Namespace: argoCDNamespaceParam}); err != nil { + return false, err + } + + crbList := &rbacv1.ClusterRoleBindingList{} + if err := k8sClient.List(context.Background(), crbList); err != nil { + return false, err + } + + // For each ClusterRoleBinding + for _, crbItem := range crbList.Items { + + // For each Subject of the Binding + for _, subject := range crbItem.Subjects { + + // Return true if there exists a subject pointing to one of the ServiceAccounts + // in the Argo CD namepace. + if subject.Namespace == argoCDNamespaceParam { + + for _, sa := range saList.Items { + + if sa.Name == subject.Name { + return true, nil + } + } + + } + } + } + + return false, nil + +} + func EnsureDestinationNamespaceExists(namespaceParam string, argoCDNamespaceParam string, clientConfig *rest.Config) error { kubeClientSet, err := kubernetes.NewForConfig(clientConfig) @@ -490,7 +534,7 @@ func EnsureDestinationNamespaceExists(namespaceParam string, argoCDNamespacePara // We used to wait here, prior to moving to Cluster Scoped Argo CD // eg. Call: waitForArgoCDToProcessNamespace(namespaceParam, argoCDNamespaceParam, kubeClientSet) if EnableNamespaceBackedArgoCD { - if err := waitForArgoCDToProcessNamespace(namespaceParam, argoCDNamespaceParam, kubeClientSet); err != nil { + if err := waitForArgoCDToProcessNamespace(namespaceParam, argoCDNamespaceParam, kubeClientSet, clientConfig); err != nil { return fmt.Errorf("unable to wait for Argo CD to process namespace: %w", err) } } @@ -501,7 +545,11 @@ func EnsureDestinationNamespaceExists(namespaceParam string, argoCDNamespacePara // Wait for Argo CD to process the namespace, before we exit: // - This helps us avoid a race condition where the namespace is created, but Argo CD has not yet // set up proper roles for it. -func waitForArgoCDToProcessNamespace(namespaceParam string, argoCDNamespaceParam string, kubeClientSet *kubernetes.Clientset) error { +func waitForArgoCDToProcessNamespace(namespaceParam string, argoCDNamespaceParam string, kubeClientSet *kubernetes.Clientset, clientConfig *rest.Config) error { + + if isClusterScoped, err := isArgoCDClusterScoped(argoCDNamespaceParam, clientConfig); isClusterScoped || err != nil { + return err + } // Wait for Argo CD to process the namespace, before we exit: // - This helps us avoid a race condition where the namespace is created, but Argo CD has not yet @@ -822,6 +870,10 @@ func GetKubeClient(config *rest.Config) (client.Client, error) { return nil, err } + if err := apiexts.AddToScheme(scheme); err != nil { + return nil, err + } + err = codereadytoolchainv1alpha1.AddToScheme(scheme) if err != nil { return nil, err @@ -953,5 +1005,52 @@ func ExtractKubeConfigValues() (string, string, error) { return "", "", err } - return string(kubeConfigContents), cluster.Server, nil + serverValRes := cluster.Server + kubeConfigContentsRes := string(kubeConfigContents) + + // Allow overriding of the API Server URL via env var, to allow testing in quirky network configurations + overrideAPIServiceAddrSrc := strings.TrimSpace(os.Getenv("OVERRIDE_APISERVER_ADDR_SOURCE")) + if len(overrideAPIServiceAddrSrc) > 0 { + overrideAPIServiceAddrDest := strings.TrimSpace(os.Getenv("OVERRIDE_APISERVER_ADDR_DEST")) + + serverValRes = strings.ReplaceAll(serverValRes, overrideAPIServiceAddrSrc, overrideAPIServiceAddrDest) + kubeConfigContentsRes = strings.ReplaceAll(kubeConfigContentsRes, overrideAPIServiceAddrSrc, overrideAPIServiceAddrDest) + } + + return kubeConfigContentsRes, serverValRes, nil +} + +// SkipIfArgoCDOperandRequired will cause a test to be skipped if the ArgoCD +// CustomResourceDefinition is not installed (because OpenShift GitOps Operator is not installed.) +func SkipIfArgoCDOperandRequired() { + + available := IsArgoCDOperandAvailable() + + if !available { + Skip("Skipping as ArgoCD CustomResourceDefinition is not installed, which is required for test.") + } + +} + +func IsArgoCDOperandAvailable() bool { + + k8sConfig, err := GetServiceProviderWorkspaceKubeConfig() + Expect(err).To(BeNil()) + + k8sClient, err := GetKubeClient(k8sConfig) + Expect(err).To(BeNil()) + + crdList := &apiexts.CustomResourceDefinitionList{} + err = k8sClient.List(context.Background(), crdList) + Expect(err).To(BeNil()) + + argoCDCRFound := false + for _, crd := range crdList.Items { + if crd.Spec.Names.Kind == "ArgoCD" { + argoCDCRFound = true + } + } + + return argoCDCRFound + } diff --git a/tests-e2e/go.mod b/tests-e2e/go.mod index ed4127546a..162edf4c7d 100644 --- a/tests-e2e/go.mod +++ b/tests-e2e/go.mod @@ -16,6 +16,7 @@ require ( github.com/redhat-appstudio/managed-gitops/backend v0.0.0-00010101000000-000000000000 github.com/redhat-appstudio/managed-gitops/backend-shared v0.0.0 github.com/redhat-appstudio/managed-gitops/cluster-agent v0.0.0 + k8s.io/apiextensions-apiserver v0.31.2 k8s.io/apimachinery v0.31.0 k8s.io/client-go v12.0.0+incompatible sigs.k8s.io/controller-runtime v0.19.0 @@ -183,7 +184,6 @@ require ( gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/api v0.31.0 - k8s.io/apiextensions-apiserver v0.31.2 // indirect k8s.io/component-base v0.31.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect From f334eaaff9a25b12474054113a5b5a92be140048 Mon Sep 17 00:00:00 2001 From: Jonathan West Date: Mon, 23 Sep 2024 13:02:25 -0400 Subject: [PATCH 2/9] Add initial k3d-based GitHub E2E test runner Signed-off-by: Jonathan West --- .github/workflows/ci.yml | 72 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..720f476761 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,72 @@ +name: Build and test the operator +on: + push: + branches: + - 'main' + + pull_request: + branches: + - 'main' + +jobs: + + test-e2e: + name: Run end-to-end tests + runs-on: ubuntu-latest + strategy: + matrix: + k3s-version: [ v1.27.1 ] + steps: + - name: Install k3d + run: | + set -x + curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash + sudo mkdir -p $HOME/.kube && sudo chown -R runner $HOME/.kube + k3d cluster create --servers 3 --image rancher/k3s:${{ matrix.k3s-version }}-k3s1 + kubectl version + k3d version + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Golang + uses: actions/setup-go@v5 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version-file: tests-e2e/go.mod + + - name: GH actions workaround - Kill XSP4 process + run: | + sudo pkill mono || true + + - name: Restore go build cache + uses: actions/cache@v4 + with: + path: ~/.cache/go-build + key: ${{ runner.os }}-go-build-v1-${{ github.run_id }} + + - name: Add /usr/local/bin to PATH + run: | + echo "/usr/local/bin" >> $GITHUB_PATH + + - name: Download Go dependencies + run: | + go mod download + + - name: Run the tests + run: | + set -o pipefail + make setup-e2e-local-k8s + + kubectl port-forward svc/argocd-server -n gitops-service-argocd 51212:443 & + + ARGO_CD_SERVER_ADDR=127.0.0.1:51212 make start-e2e & + + OVERRIDE_APISERVER_ADDR_SOURCE="0.0.0.0" OVERRIDE_APISERVER_ADDR_DEST="192.168.8.152" make test-e2e + + make e2e-reset + pkill go + pkill main + From e9f367f4aeb7ab0a20b421399e5f2a90c6343bef Mon Sep 17 00:00:00 2001 From: Jonathan West Date: Fri, 31 Jan 2025 07:14:09 -0500 Subject: [PATCH 3/9] Update --- .github/workflows/ci.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 720f476761..c17bac4a58 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,9 +31,6 @@ jobs: - name: Setup Golang uses: actions/setup-go@v5 - - - name: Set up Go - uses: actions/setup-go@v3 with: go-version-file: tests-e2e/go.mod @@ -51,10 +48,6 @@ jobs: run: | echo "/usr/local/bin" >> $GITHUB_PATH - - name: Download Go dependencies - run: | - go mod download - - name: Run the tests run: | set -o pipefail From 3791ba21c9b9fbfe9f6f79c07f18f2f08e567511 Mon Sep 17 00:00:00 2001 From: Jonathan West Date: Fri, 31 Jan 2025 07:26:21 -0500 Subject: [PATCH 4/9] Update --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c17bac4a58..0c64461242 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,6 +51,8 @@ jobs: - name: Run the tests run: | set -o pipefail + make download-deps + make setup-e2e-local-k8s kubectl port-forward svc/argocd-server -n gitops-service-argocd 51212:443 & From bcaa69fe08fffc2ac1e5a14bf7952e405d6a486d Mon Sep 17 00:00:00 2001 From: Jonathan West Date: Fri, 31 Jan 2025 08:44:56 -0500 Subject: [PATCH 5/9] Update --- .github/workflows/ci.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c64461242..d5abf280cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,7 @@ on: jobs: + test-e2e: name: Run end-to-end tests runs-on: ubuntu-latest @@ -17,6 +18,15 @@ jobs: matrix: k3s-version: [ v1.27.1 ] steps: + - name: Public IP + id: ip + uses: haythem/public-ip@v1.3 + + # - name: See IP Addresses + # run: | + # echo ${{ steps.ip.outputs.ipv4 }} + # echo ${{ steps.ip.outputs.ipv6 }} + - name: Install k3d run: | set -x @@ -59,7 +69,7 @@ jobs: ARGO_CD_SERVER_ADDR=127.0.0.1:51212 make start-e2e & - OVERRIDE_APISERVER_ADDR_SOURCE="0.0.0.0" OVERRIDE_APISERVER_ADDR_DEST="192.168.8.152" make test-e2e + OVERRIDE_APISERVER_ADDR_SOURCE="0.0.0.0" OVERRIDE_APISERVER_ADDR_DEST="${{ steps.ip.outputs.ipv4 }}" make test-e2e make e2e-reset pkill go From 94c806a40d9a3e3351cd54b60d8e936532b9bda1 Mon Sep 17 00:00:00 2001 From: Jonathan West Date: Fri, 31 Jan 2025 08:59:35 -0500 Subject: [PATCH 6/9] Update --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5abf280cc..f493b36921 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,8 +19,8 @@ jobs: k3s-version: [ v1.27.1 ] steps: - name: Public IP - id: ip - uses: haythem/public-ip@v1.3 + id: ip + uses: haythem/public-ip@v1.3 # - name: See IP Addresses # run: | From 7b5aac98880b82f73a6ad7a642e9151f78c2283c Mon Sep 17 00:00:00 2001 From: Jonathan West Date: Fri, 31 Jan 2025 12:20:07 -0500 Subject: [PATCH 7/9] Update --- .github/workflows/ci.yml | 8 ++++---- tests-e2e/fixture/fixture.go | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f493b36921..3c013865eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,9 +18,9 @@ jobs: matrix: k3s-version: [ v1.27.1 ] steps: - - name: Public IP - id: ip - uses: haythem/public-ip@v1.3 + # - name: Public IP + # id: ip + # uses: haythem/public-ip@v1.3 # - name: See IP Addresses # run: | @@ -69,7 +69,7 @@ jobs: ARGO_CD_SERVER_ADDR=127.0.0.1:51212 make start-e2e & - OVERRIDE_APISERVER_ADDR_SOURCE="0.0.0.0" OVERRIDE_APISERVER_ADDR_DEST="${{ steps.ip.outputs.ipv4 }}" make test-e2e + OVERRIDE_APISERVER_ADDR_SOURCE="0.0.0.0" OVERRIDE_APISERVER_ADDR_DEST="kubernetes.default.svc" make test-e2e make e2e-reset pkill go diff --git a/tests-e2e/fixture/fixture.go b/tests-e2e/fixture/fixture.go index 8c041e175f..40d8c5042e 100644 --- a/tests-e2e/fixture/fixture.go +++ b/tests-e2e/fixture/fixture.go @@ -1017,6 +1017,8 @@ func ExtractKubeConfigValues() (string, string, error) { kubeConfigContentsRes = strings.ReplaceAll(kubeConfigContentsRes, overrideAPIServiceAddrSrc, overrideAPIServiceAddrDest) } + fmt.Println("JGW", kubeConfigContentsRes) + return kubeConfigContentsRes, serverValRes, nil } From 7990ac07b5bfbab75cc666856b6d7933bf9d2b0d Mon Sep 17 00:00:00 2001 From: Jonathan West Date: Wed, 9 Apr 2025 06:07:23 -0400 Subject: [PATCH 8/9] Update --- .github/workflows/ci.yml | 2 +- tests-e2e/argocd/queryscoped_cluster_secrets_test.go | 5 +++++ tests-e2e/fixture/fixture.go | 4 ++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c013865eb..5f6d277106 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,7 +69,7 @@ jobs: ARGO_CD_SERVER_ADDR=127.0.0.1:51212 make start-e2e & - OVERRIDE_APISERVER_ADDR_SOURCE="0.0.0.0" OVERRIDE_APISERVER_ADDR_DEST="kubernetes.default.svc" make test-e2e + GITHUB_K3D=true OVERRIDE_APISERVER_ADDR_SOURCE="0.0.0.0" OVERRIDE_APISERVER_ADDR_DEST="kubernetes.default.svc" make test-e2e make e2e-reset pkill go diff --git a/tests-e2e/argocd/queryscoped_cluster_secrets_test.go b/tests-e2e/argocd/queryscoped_cluster_secrets_test.go index 3c8ab30c01..2365277a3d 100644 --- a/tests-e2e/argocd/queryscoped_cluster_secrets_test.go +++ b/tests-e2e/argocd/queryscoped_cluster_secrets_test.go @@ -138,6 +138,11 @@ var _ = Describe("Argo CD Application tests", func() { func(clusterScoped bool) { + if fixture.IsGitHubK3D() { + Skip("when running on GitHub CI via K3d, we don't have an API endpoint IP we can use, so skip") + return + } + createServiceAccountsAndSecretsForTest(clusterScoped) for _, userName := range users { diff --git a/tests-e2e/fixture/fixture.go b/tests-e2e/fixture/fixture.go index 40d8c5042e..7bd5b8adf7 100644 --- a/tests-e2e/fixture/fixture.go +++ b/tests-e2e/fixture/fixture.go @@ -930,6 +930,10 @@ func GetE2ETestUserWorkspaceKubeClient() (client.Client, error) { return k8sClient, nil } +func IsGitHubK3D() bool { + return strings.TrimSpace(strings.ToLower(os.Getenv("GITHUB_K3D"))) == "true" +} + // IsRunningInStonesoupEnvironment returns true if the tests are running in a Stonesoup environment, false otherwise func IsRunningInStonesoupEnvironment() bool { // Stonesoup environment is bootstrapped by an AppOfApps Gitops Deployment. From d3178c84b24556233bf8c12af59b8ab44df5db17 Mon Sep 17 00:00:00 2001 From: Jonathan West Date: Wed, 9 Apr 2025 07:22:25 -0400 Subject: [PATCH 9/9] Update --- tests-e2e/argocd/queryscoped_cluster_secrets_test.go | 5 +++++ tests-e2e/fixture/fixture.go | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests-e2e/argocd/queryscoped_cluster_secrets_test.go b/tests-e2e/argocd/queryscoped_cluster_secrets_test.go index 2365277a3d..b6195972ca 100644 --- a/tests-e2e/argocd/queryscoped_cluster_secrets_test.go +++ b/tests-e2e/argocd/queryscoped_cluster_secrets_test.go @@ -208,6 +208,11 @@ var _ = Describe("Argo CD Application tests", func() { DescribeTable("ensure users cannot deploy to another user's namespace, when using different Argo CD cluster secrets for each user, each with a query parameter", func(clusterScoped bool) { + if fixture.IsGitHubK3D() { + Skip("when running on GitHub CI via K3d, we don't have an API endpoint IP we can use, so skip") + return + } + createServiceAccountsAndSecretsForTest(clusterScoped) Expect(users).To(HaveLen(2)) diff --git a/tests-e2e/fixture/fixture.go b/tests-e2e/fixture/fixture.go index 7bd5b8adf7..5af726bb7b 100644 --- a/tests-e2e/fixture/fixture.go +++ b/tests-e2e/fixture/fixture.go @@ -1041,14 +1041,14 @@ func SkipIfArgoCDOperandRequired() { func IsArgoCDOperandAvailable() bool { k8sConfig, err := GetServiceProviderWorkspaceKubeConfig() - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) k8sClient, err := GetKubeClient(k8sConfig) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) crdList := &apiexts.CustomResourceDefinitionList{} err = k8sClient.List(context.Background(), crdList) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) argoCDCRFound := false for _, crd := range crdList.Items {