Merge branch 'main' into test-acm-policies

Author: claudiol

.ansible-lint

@@ -1,7 +1,6 @@
 # Vim filetype=yaml
 ---
 offline: false
-#requirements: ansible/execution_environment/requirements.yml
 
 exclude_paths:
   - .cache/

.github/workflows/ansible-lint.yml

@@ -11,7 +11,4 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Lint Ansible Playbook
-        uses: ansible/ansible-lint-action@v6
-        # Let's point it to the path
-        with:
-          path: "ansible/"
+        uses: ansible/ansible-lint@06f616d6e86e9ce4c74393318d1cbb2d016af413

.github/workflows/jsonschema.yaml

@@ -1,41 +1,19 @@
 ---
 name: Verify json schema
 
-#
-# Documentation:
-# https://help.github.com/en/articles/workflow-syntax-for-github-actions
-#
-
-#############################
-# Start the job on all push #
-#############################
 on: [push, pull_request]
 
-###############
-# Set the Job #
-###############
 jobs:
   jsonschema_tests:
-    # Name the Job
     name: Json Schema tests
     strategy:
       matrix:
         python-version: [3.11]
-    # Set the agent to run on
     runs-on: ubuntu-latest
 
-    ##################
-    # Load all steps #
-    ##################
     steps:
-      ##########################
-      # Checkout the code base #
-      ##########################
       - name: Checkout Code
         uses: actions/checkout@v4
-        with:
-          # Full git history is needed to get a proper list of changed files within `super-linter`
-          fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
@@ -54,19 +32,20 @@ jobs:
 
       - name: Verify secrets json schema against templates
         run: |
-          # check-jsonschema needs .yaml as an extension
           cp ./values-secret.yaml.template ./values-secret.yaml
-          check-jsonschema --schemafile https://raw.githubusercontent.com/validatedpatterns/rhvp.cluster_utils/refs/heads/main/roles/vault_utils/values-secrets.v2.schema.json values-secret.yaml
+          check-jsonschema --fill-defaults --schemafile https://raw.githubusercontent.com/validatedpatterns/rhvp.cluster_utils/refs/heads/main/roles/vault_utils/values-secrets.v2.schema.json values-secret.yaml
           rm -f ./values-secret.yaml
 
       - name: Verify ClusterGroup values.schema.json against values-*yaml files
         run: |
-          set -e; for i in values-hub.yaml values-group-one.yaml; do
+          set -e
+          find . -maxdepth 1 -type f -name "values-*.yaml" ! -name "values-global.yaml" -print0 | while IFS= read -r -d '' i;
+          do
             echo "$i"
             # disable shellcheck of single quotes in yq
             # shellcheck disable=2016
             yq eval-all '. as $item ireduce ({}; . * $item )' values-global.yaml "$i" > tmp.yaml
-            check-jsonschema --schemafile https://raw.githubusercontent.com/validatedpatterns/clustergroup-chart/refs/heads/main/values.schema.json tmp.yaml
+            check-jsonschema --fill-defaults --schemafile https://raw.githubusercontent.com/validatedpatterns/clustergroup-chart/refs/heads/main/values.schema.json tmp.yaml
             rm -f tmp.yaml
           done
 

.github/workflows/sync-rhdp-branch.yml

@@ -0,0 +1,29 @@
+# This is a job to be run in the validatedpatterns organization only.
+# It tries to keep the rhdp-deploy branch uptodate by creating a PR from main
+---
+name: Sync main to rhdp-deploy
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  sync-branches:
+    if: |
+      github.repository_owner == 'validatedpatterns'
+    runs-on: ubuntu-latest
+    name: Git Sync branch
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - name: Opening pull request
+        id: pull
+        uses: mbaldessari/git-sync-branch@v0.2.0
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          FROM_BRANCH: "main"
+          TO_BRANCH: "rhdp-deploy"

.github/workflows/update-metadata.yml

@@ -0,0 +1,24 @@
+# This job requires a secret called DOCS_TOKEN which should be a PAT token
+# that has the permissions described in:
+# validatedpatterns/docs/.github/workflows/metadata-docs.yml@main
+---
+name: Update docs pattern metadata
+on:
+  push:
+    paths:
+      - "pattern-metadata.yaml"
+      - ".github/workflows/update-metadata.yml"
+    branches:
+      - main
+
+jobs:
+  update-metadata:
+    uses: validatedpatterns/docs/.github/workflows/metadata-docs.yml@main
+    permissions: # Workflow-level permissions
+      contents: read  # Required for "read-all"
+      packages: write # Allows writing to packages
+      id-token: write # Allows creating OpenID Connect (OIDC) tokens
+    secrets: inherit
+    # For testing you can point to a different branch in the docs repository
+    # with:
+    #  DOCS_BRANCH: "main"

common/.github/workflows/pattern-sh-ci.yml

@@ -0,0 +1,48 @@
+name: Run Bash Script on Multiple Distributions
+
+on:
+  push:
+    paths:
+      - "scripts/**"
+      - "Makefile"
+    branches:
+      - main
+  pull_request:
+    paths:
+      - "scripts/**"
+      - "Makefile"
+
+jobs:
+  run-script:
+    name: Run Bash Script
+    strategy:
+      matrix:
+        # Fedora is not an option yet
+        os: [ubuntu-latest, ubuntu-22.04]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Install Podman on Ubuntu
+        if: contains(matrix.os, 'ubuntu')
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y podman
+
+      # Currently we do not do MacOSX as it is not free, maybe in the future
+      # - name: Install Podman on macOS
+      #   if: contains(matrix.os, 'macos')
+      #   run: |
+      #     brew install podman
+      #     podman machine init
+      #     podman machine start
+
+      - name: Verify Podman Installation
+        run: podman --version
+
+      - name: Run pattern.sh script
+        run: |
+          export TARGET_BRANCH=main
+          ./scripts/pattern-util.sh make validate-origin

common/Makefile

@@ -4,21 +4,36 @@ ifneq ($(origin TARGET_SITE), undefined)
   TARGET_SITE_OPT=--set main.clusterGroupName=$(TARGET_SITE)
 endif
 
+# Set this to true if you want to skip any origin validation
+DISABLE_VALIDATE_ORIGIN ?= false
+ifeq ($(DISABLE_VALIDATE_ORIGIN),true)
+  VALIDATE_ORIGIN := 
+else
+  VALIDATE_ORIGIN := validate-origin
+endif
+
 # This variable can be set in order to pass additional helm arguments from the
 # the command line. I.e. we can set things without having to tweak values files
 EXTRA_HELM_OPTS ?=
 
+# This variable can be set in order to pass additional ansible-playbook arguments from the
+# the command line. I.e. we can set -vvv for more verbose logging
+EXTRA_PLAYBOOK_OPTS ?=
+
 # INDEX_IMAGES=registry-proxy.engineering.redhat.com/rh-osbs/iib:394248
 # or
 # INDEX_IMAGES=registry-proxy.engineering.redhat.com/rh-osbs/iib:394248,registry-proxy.engineering.redhat.com/rh-osbs/iib:394249
 INDEX_IMAGES ?=
 
-TARGET_ORIGIN ?= origin
+# git branch --show-current is also available as of git 2.22, but we will use this for compatibility
+TARGET_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
+
+#default to the branch remote
+TARGET_ORIGIN ?= $(shell git config branch.$(TARGET_BRANCH).remote)
+
 # This is to ensure that whether we start with a git@ or https:// URL, we end up with an https:// URL
 # This is because we expect to use tokens for repo authentication as opposed to SSH keys
 TARGET_REPO=$(shell git ls-remote --get-url --symref $(TARGET_ORIGIN) | sed -e 's/.*URL:[[:space:]]*//' -e 's%^git@%%' -e 's%^https://%%' -e 's%:%/%' -e 's%^%https://%')
-# git branch --show-current is also available as of git 2.22, but we will use this for compatibility
-TARGET_BRANCH=$(shell git rev-parse --abbrev-ref HEAD)
 
 UUID_FILE ?= ~/.config/validated-patterns/pattern-uuid
 UUID_HELM_OPTS ?=
@@ -67,12 +82,8 @@ preview-%:
 	@common/scripts/preview.sh $(CLUSTERGROUP) $* $(TARGET_REPO) $(TARGET_BRANCH)
 
 .PHONY: operator-deploy
-operator-deploy operator-upgrade: validate-prereq validate-origin validate-cluster ## runs helm install
-	@set -e -o pipefail
-	# Retry five times because the CRD might not be fully installed yet
-	for i in {1..5}; do \
-		helm template --include-crds --name-template $(NAME) $(PATTERN_INSTALL_CHART) $(HELM_OPTS) | oc apply -f- && break || sleep 10; \
-	done
+operator-deploy operator-upgrade: validate-prereq $(VALIDATE_ORIGIN) validate-cluster ## runs helm install
+	@common/scripts/deploy-pattern.sh $(NAME) $(PATTERN_INSTALL_CHART) $(HELM_OPTS)
 
 .PHONY: uninstall
 uninstall: ## runs helm uninstall
@@ -115,7 +126,7 @@ secrets-backend-none: ## Edits values files to remove secrets manager + ESO
 .PHONY: load-iib
 load-iib: ## CI target to install Index Image Bundles
 	@set -e; if [ x$(INDEX_IMAGES) != x ]; then \
-		ansible-playbook rhvp.cluster_utils.iib-ci; \
+		ansible-playbook $(EXTRA_PLAYBOOK_OPTS) rhvp.cluster_utils.iib_ci; \
 	else \
 		echo "No INDEX_IMAGES defined. Bailing out"; \
 		exit 1; \
@@ -129,12 +140,22 @@ token-kubeconfig: ## Create a local ~/.kube/config with password (not usually ne
 
 # We only check the remote ssh git branch's existance if we're not running inside a container
 # as getting ssh auth working inside a container seems a bit brittle
+# If the main repoUpstreamURL field is set, then we need to check against
+# that and not target_repo
 .PHONY: validate-origin
 validate-origin: ## verify the git origin is available
 	@echo "Checking repository:"
-	@echo -n "  $(TARGET_REPO) - branch '$(TARGET_BRANCH)': "
-	@git ls-remote --exit-code --heads $(TARGET_REPO) $(TARGET_BRANCH) >/dev/null &&\
-		echo "OK" || (echo "NOT FOUND"; exit 1)
+	$(eval UPSTREAMURL := $(shell yq -r '.main.git.repoUpstreamURL // (.main.git.repoUpstreamURL = "")' values-global.yaml))
+	@if [ -z "$(UPSTREAMURL)" ]; then\
+		echo -n "  $(TARGET_REPO) - branch '$(TARGET_BRANCH)': ";\
+		git ls-remote --exit-code --heads $(TARGET_REPO) $(TARGET_BRANCH) >/dev/null &&\
+			echo "OK" || (echo "NOT FOUND"; exit 1);\
+	else\
+		echo "Upstream URL set to: $(UPSTREAMURL)";\
+		echo -n "  $(UPSTREAMURL) - branch '$(TARGET_BRANCH)': ";\
+		git ls-remote --exit-code --heads $(UPSTREAMURL) $(TARGET_BRANCH) >/dev/null &&\
+			echo "OK" || (echo "NOT FOUND"; exit 1);\
+	fi
 
 .PHONY: validate-cluster
 validate-cluster: ## Do some cluster validations before installing
@@ -143,7 +164,7 @@ validate-cluster: ## Do some cluster validations before installing
 	@oc cluster-info >/dev/null && echo "OK" || (echo "Error"; exit 1)
 	@echo -n "  storageclass: "
 	@if [ `oc get storageclass -o go-template='{{printf "%d\n" (len .items)}}'` -eq 0 ]; then\
-		echo "None Found"; exit 1;\
+		echo "WARNING: No storageclass found";\
 	else\
 		echo "OK";\
 	fi
@@ -153,15 +174,20 @@ validate-cluster: ## Do some cluster validations before installing
 validate-schema: ## validates values files against schema in common/clustergroup
 	$(eval VAL_PARAMS := $(shell for i in ./values-*.yaml; do echo -n "$${i} "; done))
 	@echo -n "Validating clustergroup schema of: "
-	@set -e; for i in $(VAL_PARAMS); do echo -n " $$i"; helm template common/clustergroup $(HELM_OPTS) -f "$${i}" >/dev/null; done
+	@set -e; for i in $(VAL_PARAMS); do echo -n " $$i"; helm template oci://quay.io/hybridcloudpatterns/clustergroup $(HELM_OPTS) -f "$${i}" >/dev/null; done
 	@echo
 
 .PHONY: validate-prereq
 validate-prereq: ## verify pre-requisites
+	$(eval GLOBAL_PATTERN := $(shell yq -r .global.pattern values-global.yaml))
+	@if [ $(NAME) != $(GLOBAL_PATTERN) ]; then\
+		echo "";\
+		echo "WARNING: folder directory is \"$(NAME)\" and global.pattern is set to \"$(GLOBAL_PATTERN)\"";\
+		echo "this can create problems. Please make sure they are the same!";\
+		echo "";\
+	fi
 	@if [ ! -f /run/.containerenv ]; then\
 	  echo "Checking prerequisites:";\
-	  for t in $(EXECUTABLES); do if ! which $$t > /dev/null 2>&1; then echo "No $$t in PATH"; exit 1; fi; done;\
-	  echo "  Check for '$(EXECUTABLES)': OK";\
 	  echo -n "  Check for python-kubernetes: ";\
 	  if ! ansible -m ansible.builtin.command -a "{{ ansible_python_interpreter }} -c 'import kubernetes'" localhost > /dev/null 2>&1; then echo "Not found"; exit 1; fi;\
 	  echo "OK";\
@@ -182,16 +208,16 @@ validate-prereq: ## verify pre-requisites
 .PHONY: argo-healthcheck
 argo-healthcheck: ## Checks if all argo applications are synced
 	@echo "Checking argo applications"
-	$(eval APPS := $(shell oc get applications -A -o jsonpath='{range .items[*]}{@.metadata.namespace}{","}{@.metadata.name}{"\n"}{end}'))
+	$(eval APPS := $(shell oc get applications.argoproj.io -A -o jsonpath='{range .items[*]}{@.metadata.namespace}{","}{@.metadata.name}{"\n"}{end}'))
 	@NOTOK=0; \
 	for i in $(APPS); do\
 		n=`echo "$${i}" | cut -f1 -d,`;\
 		a=`echo "$${i}" | cut -f2 -d,`;\
-		STATUS=`oc get -n "$${n}" application/"$${a}" -o jsonpath='{.status.sync.status}'`;\
+		STATUS=`oc get -n "$${n}" applications.argoproj.io/"$${a}" -o jsonpath='{.status.sync.status}'`;\
 		if [[ $$STATUS != "Synced" ]]; then\
 			NOTOK=$$(( $${NOTOK} + 1));\
 		fi;\
-		HEALTH=`oc get -n "$${n}" application/"$${a}" -o jsonpath='{.status.health.status}'`;\
+		HEALTH=`oc get -n "$${n}" applications.argoproj.io/"$${a}" -o jsonpath='{.status.health.status}'`;\
 		if [[ $$HEALTH != "Healthy" ]]; then\
 			NOTOK=$$(( $${NOTOK} + 1));\
 		fi;\
@@ -208,7 +234,7 @@ argo-healthcheck: ## Checks if all argo applications are synced
 .PHONY: qe-tests
 qe-tests: ## Runs the tests that QE runs
 	@set -e; if [ -f ./tests/interop/run_tests.sh ]; then \
-		./tests/interop/run_tests.sh; \
+		pushd ./tests/interop; ./run_tests.sh; popd; \
 	else \
 		echo "No ./tests/interop/run_tests.sh found skipping"; \
 	fi

common/README.md

@@ -34,16 +34,16 @@ main:
 ## Start Here
 
 This repository is never used as standalone. It is usually imported in each pattern as a subtree.
-In order to import the common/ the very first time you can use
-`https://github.com/validatedpatterns/multicloud-gitops/blob/main/common/scripts/make_common_subtree.sh`
+In order to import the common subtree the very first time you can use the script
+[make_common_subtree.sh](scripts/make-common-subtree.sh).
 
 In order to update your common subtree inside your pattern repository you can either use
 `https://github.com/validatedpatterns/utilities/blob/main/scripts/update-common-everywhere.sh` or
-do it manually by doing the following:
+do it manually with the following commands:
 
 ```sh
-git remote add -f upstream-common https://github.com/validatedpatterns/common.git
-git merge -s subtree -Xtheirs -Xsubtree=common upstream-common/main
+git remote add -f common-upstream https://github.com/validatedpatterns/common.git
+git merge -s subtree -Xtheirs -Xsubtree=common common-upstream/main
 ```
 
 ## Secrets

common/scripts/argocd-login.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+## Login to validated patterns argocd instances
+
+# Detect Argo CD namespaces
+ARGOCD_NAMESPACES=$(oc get argoCD -A -o jsonpath='{.items[*].metadata.namespace}')
+if [ -z "$ARGOCD_NAMESPACES" ]; then
+    echo "Error: No Argo CD instances found in the cluster."
+    exit 1
+fi
+
+# Split the namespaces into an array
+NAMESPACES=($ARGOCD_NAMESPACES)
+
+# Check if there are at least two Argo CD instances
+if [ ${#NAMESPACES[@]} -lt 2 ]; then
+    echo "Error: Less than two Argo CD instances found.  Found instances in namespaces: $ARGOCD_NAMESPACES"
+    exit 1
+fi
+
+
+for NAMESPACE in ${NAMESPACES[@]}; do
+    # get the instance name
+    ARGOCD_INSTANCE=$(oc get argocd -n "$NAMESPACE" -o jsonpath='{.items[0].metadata.name}') # assume only one per NS
+    SERVER_URL=$(oc get route "$ARGOCD_INSTANCE"-server -n "$NAMESPACE" -o jsonpath='{.status.ingress[0].host}')
+    PASSWORD=$(oc get secret "$ARGOCD_INSTANCE"-cluster -n "$NAMESPACE" -o jsonpath='{.data.admin\.password}' | base64 -d)
+    echo $PASSWORD
+    argocd login --skip-test-tls --insecure --grpc-web "$SERVER_URL" --username "admin" --password "$PASSWORD"
+    if [ "$?" -ne 0 ]; then
+      echo "Login to Argo CD ${SERVER_URL} failed. Exiting."
+      exit 1
+    fi
+    
+done