test real deployment in ci/cd

Author: vprivat-ads

.github/common/resources/configure-minikube-deployment.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+# Copyright 2025 CS Group
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+APPS=rs-server-deployment/apps
+
+# Lower the CPU/memory requests
+# Minimum memory for postgresql must be > shared_buffers, which is 1/4 of the total ram
+sed -i -e 's!instances: 3!instances: 1!g' -e 's!cpu: "1"!cpu: "0.1"!g' -e 's!memory: "2G"!memory: "256M"!g' -e 's!memory: "1024M"!memory: "100M"!g' "${APPS}/01-eo-cnpgstac/values.yaml"
+sed -i -e 's!cpu: "100m"!cpu: "1m"!g' -e 's!ram: "256Mi"!ram: "10Mi"!g'\
+  "${APPS}/mockup-prip-s1a/values.yaml"\
+  "${APPS}/mockup-prip-s2b/values.yaml"\
+  "${APPS}/mockup-processor-dpr/values.yaml"\
+  "${APPS}/mockup-station-adgs/values.yaml"\
+  "${APPS}/mockup-station-cadip-cadip/values.yaml"\
+  "${APPS}/mockup-station-cadip-mti/values.yaml"\
+  "${APPS}/mockup-station-cadip-sgs/values.yaml"\
+  "${APPS}/mockup-station-lta/values.yaml"
+sed -i -e 's!cpu: "100m"!cpu: "10m"!g' -e 's!ram: "256Mi"!ram: "32Mi"!g'\
+  "${APPS}/rs-dpr-service/values.yaml"\
+  "${APPS}/rs-server-adgs/values.yaml"\
+  "${APPS}/rs-server-cadip/values.yaml"\
+  "${APPS}/rs-server-catalog/values.yaml"\
+  "${APPS}/rs-server-frontend/values.yaml"\
+  "${APPS}/rs-server-prip/values.yaml"\
+  "${APPS}/rs-server-staging/values.yaml"

.github/workflows/test-deployment.yml

@@ -29,56 +29,139 @@ jobs:
   test-deploy:
     if: github.actor != 'dependabot[bot]' # ignore pull requests by github bot
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: "Deployment with monitoring"
+            deploy_monitoring: true
+# TODO enable this in RSPY-836
+#          - name: "Deployment without monitoring"
+#            deploy_monitoring: false
+    name: ${{ matrix.name }}
     steps:
-      - name: Checkout repository
+      - name: Get current branch name
+        id: vars
+        run: echo "branch_name=${{ github.head_ref }}" >> $GITHUB_OUTPUT
+      - name: Check if branch exists in rs-infra-core
+        id: check_branch
+        run: |
+          BRANCH=${{ steps.vars.outputs.branch_name }}
+          if git ls-remote --heads https://github.com/RS-PYTHON/rs-infra-core.git $BRANCH | grep -q $BRANCH; then
+            echo "branch_to_use=$BRANCH" >> $GITHUB_OUTPUT
+          else
+            echo "branch_to_use=" >> $GITHUB_OUTPUT  # Leave empty to use default
+          fi
+        shell: bash
+      - name: Checkout infra-core repository
         uses: actions/checkout@v4
         with:
           repository: RS-PYTHON/rs-infra-core
-          submodules: recursive
+          ref: ${{ steps.check_branch.outputs.branch_to_use }}
+      - name: Checkout infra-monitoring repository
+        uses: actions/checkout@v4
+        if: ${{ matrix.deploy_monitoring }}
+        with:
+          path: rs-infra-monitoring
+          repository: RS-PYTHON/rs-infra-monitoring
+      - name: Checkout workflow-env repository
+        uses: actions/checkout@v4
+        with:
+          path: rs-workflow-env
+          repository: RS-PYTHON/rs-workflow-env
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
           path: rs-server-deployment
           ref: ${{ github.ref }}
-      - name: Install requirements
+      - name: Cache Miniforge and Conda env
+        id: cache-conda
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/miniforge3
+            /usr/share/miniconda
+          key: conda-${{ runner.os }}-${{ hashFiles('.github/common/resources/install-requirements.sh') }}
+          restore-keys: |
+            conda-${{ runner.os }}-
+      - name: Aggressive cleanup
         run: |
-          # Install miniforge
-          mkdir -p ~/miniforge3
-          wget -q "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" -O ~/miniforge3/miniforge.sh
-          bash ~/miniforge3/miniforge.sh -b -u -p ~/miniforge3
-          rm -f ~/miniforge3/miniforge.sh
-
-          # Init conda
-          ~/miniforge3/bin/conda init bash
-
-          # Create conda env with python
-          conda create -y -n rspy python=3.11
-
-          # Install Ansible, Terraform, Openstackclient
-          conda run -n rspy conda install -y -c conda-forge ansible terraform python-openstackclient passlib boto3 kubernetes-helm kubernetes-client python-kubernetes
-          
-          conda run -n rspy ansible-galaxy collection install openstack.cloud amazon.aws kubernetes.core
-          ln -s /usr/share/miniconda/envs/rspy/bin/kubectl /usr/local/bin/kubectl
+          # Cleanup disk space (level between 0 and 14, more cleaning takes more time)
+          .github/common/resources/aggressive-cleanup.sh 2
+        shell: bash
+      - name: Install requirements (if cache missed)
+        if: steps.cache-conda.outputs.cache-hit != 'true'
+        run: |
+          .github/common/resources/install-requirements.sh
         shell: bash
       - name: Start minikube
         uses: medyagh/setup-minikube@latest
         with:
-          start-args: '--profile cluster.local'
+          addons: 'metallb,metrics-server'
+          cpus: 4
           memory: 8000m
+          start-args: '--disk-size=32g'
+      - name: "Configure cluster labels, IP and DNS (host + CoreDNS)"
+        run: |
+          if [ "${{ matrix.deploy_monitoring }}" = "true" ]; then
+            .github/common/resources/configure-cluster.sh "node-role.kubernetes.io/infra= node-role.kubernetes.io/rs_env= node-role.kubernetes.io/rs_server= node-role.kubernetes.io/access_csc=" "apikeymanager iam kube monitoring oauth2-proxy processing stac-browser-auxip stac-browser-cadip stac-browser-catalog stac-browser-prip"
+          else
+            .github/common/resources/configure-cluster.sh "node-role.kubernetes.io/infra= node-role.kubernetes.io/rs_env= node-role.kubernetes.io/rs_server= node-role.kubernetes.io/access_csc=" "apikeymanager iam kube oauth2-proxy processing stac-browser-auxip stac-browser-cadip stac-browser-catalog stac-browser-prip"
+          fi
+        shell: bash
+      - name: Deploy minio
+        run: |
+          # Minio for cloudnative pg - https://github.com/minio/minio/tree/master/helm/minio#installing-the-chart-toy-setup
+          helm repo add minio https://charts.min.io/
+          helm repo update minio
+          if [ "${{ matrix.deploy_monitoring }}" = "true" ]; then
+            helm install minio minio/minio --namespace minio --set mode=standalone --set replicas=1 --set persistence.enabled=false --set resources.requests.memory=128Mi --set rootUser=s3_access_key,rootPassword=s3_secret_key --set buckets[0].name=rs-cluster-psql,buckets[1].name=rs-cluster-velero,buckets[2].name=rs-cluster-loki-chunks,buckets[3].name=rs-cluster-loki-ruler,buckets[4].name=rs-cluster-tempo --create-namespace --wait
+          else
+            helm install minio minio/minio --namespace minio --set mode=standalone --set replicas=1 --set persistence.enabled=false --set resources.requests.memory=128Mi --set rootUser=s3_access_key,rootPassword=s3_secret_key --set buckets[0].name=rs-cluster-psql,buckets[1].name=rs-cluster-velero --create-namespace --wait
+          fi
       - name: Generate inventory
         run: |
-          cp -rfp inventory/sample inventory/mycluster
-          mv inventory/mycluster/.env.template inventory/mycluster/.env
-          mv inventory/mycluster/openrc.sh.template inventory/mycluster/openrc.sh
-          sed -i 's!<changeme_with_full_path>/miniforge3/envs/rspy!/usr/share/miniconda/envs/rspy!g' inventory/mycluster/hosts.yaml
+          .github/common/resources/configure-inventory.sh
+          # --- Generate inventory
           conda run -n rspy --no-capture-output env PYTHONUNBUFFERED=1 ANSIBLE_FORCE_COLOR=1 ansible-playbook registry.yaml -i inventory/mycluster/hosts.yaml -e ci_mode=true
           conda run -n rspy --no-capture-output env PYTHONUNBUFFERED=1 ANSIBLE_FORCE_COLOR=1 ansible-playbook generate_inventory.yaml -i inventory/mycluster/hosts.yaml
         shell: bash
-      - name: Deploy the core apps (check)
+      - name: Deploy the core apps (for real)
+        run: |
+          # --- Use local cluster issuer instead of Let's Encrypt
+          if [ "${{ matrix.deploy_monitoring }}" = "true" ]; then
+            .github/common/resources/configure-local-ca.sh apikeymanager:processing iam:iam kube:default monitoring:monitoring processing:processing oauth2-proxy:iam stac-browser-auxip:processing stac-browser-cadip:processing stac-browser-catalog:processing stac-browser-prip:processing
+          else
+            .github/common/resources/configure-local-ca.sh apikeymanager:processing iam:iam kube:default processing:processing oauth2-proxy:iam stac-browser-auxip:processing stac-browser-cadip:processing stac-browser-catalog:processing stac-browser-prip:processing
+          fi
+          conda run -n rspy --no-capture-output env PYTHONUNBUFFERED=1 ANSIBLE_FORCE_COLOR=1 ansible-playbook apps.yaml -i inventory/mycluster/hosts.yaml
+        shell: bash
+      - name: Deploy the monitoring apps (for real)
+        if: ${{ matrix.deploy_monitoring }}
         run: |
-          conda run -n rspy --no-capture-output env PYTHONUNBUFFERED=1 ANSIBLE_FORCE_COLOR=1 ansible-playbook --check apps.yaml -i inventory/mycluster/hosts.yaml -e private_registry=true
+          ./rs-infra-monitoring/.github/common/resources/configure-minikube-deployment.sh
+          conda run -n rspy --no-capture-output env PYTHONUNBUFFERED=1 ANSIBLE_FORCE_COLOR=1 ansible-playbook apps.yaml -i inventory/mycluster/hosts.yaml -e '{"package_paths": ["./rs-infra-monitoring/apps/"]}'
+        shell: bash
+      - name: Deploy the required workflow-env apps (for real)
+        run: |
+          # rs-server requires apikeymanager, dask
+          for app in apikeymanager-db apikeymanager dask-gateway ; do
+            echo Installing $app...
+            conda run -n rspy --no-capture-output env PYTHONUNBUFFERED=1 ANSIBLE_FORCE_COLOR=1 ansible-playbook apps.yaml -i inventory/mycluster/hosts.yaml  -e '{"package_paths": ["./rs-workflow-env/apps/"], "app": "'$app'"}'
+          done
         shell: bash
       - name: Deploy the rs-server apps (check)
         run: |
           conda run -n rspy --no-capture-output env PYTHONUNBUFFERED=1 ANSIBLE_FORCE_COLOR=1 ansible-playbook --check apps.yaml -i inventory/mycluster/hosts.yaml -e '{"package_paths": ["./rs-server-deployment/apps/"]}' -e private_registry=true
         shell: bash
+      - name: Deploy the rs-server apps (for real)
+        run: |
+          sed -i 's!debug: false!debug: true!g' roles/app-installer/defaults/main.yaml
+          ./rs-server-deployment/.github/common/resources/configure-minikube-deployment.sh
+          conda run -n rspy --no-capture-output env PYTHONUNBUFFERED=1 ANSIBLE_FORCE_COLOR=1 ansible-playbook apps.yaml -i inventory/mycluster/hosts.yaml -e '{"package_paths": ["./rs-server-deployment/apps/"]}'
+        shell: bash
+      - name: 🩺 Kubernetes failure diagnostics
+        if: failure()
+        run: |
+          .github/common/resources/minikube-diagnostics.sh
+        shell: bash

.pre-commit-config.yaml

@@ -2,7 +2,7 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
@@ -12,7 +12,7 @@ repos:
     -   id: check-added-large-files
 
 -   repo: https://github.com/gruntwork-io/pre-commit
-    rev: v0.1.29 # Get the latest from: https://github.com/gruntwork-io/pre-commit/releases
+    rev: v0.1.30 # Get the latest from: https://github.com/gruntwork-io/pre-commit/releases
     hooks:
     -   id: terraform-fmt
       # -   id: terraform-validate

apps/01-dask-cluster-staging/configmap.yaml

@@ -203,4 +203,3 @@ data:
     while True:
         print(f"[heartbeat] workers={current_worker_count(client)} target={TARGET_WORKERS}")
         time.sleep(30)
-

apps/01-dask-cluster-staging/deployment.yaml

@@ -46,4 +46,3 @@ spec:
       - configMap:
           name: dask-cluster-script
         name: script-volume
-

apps/01-dask-cluster-staging/kustomization.yaml

@@ -11,13 +11,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-commonLabels:
-  app.kubernetes.io/instance: "{{ app_name }}"
-
-namespace: dask-gateway
-
-resources:
-  - configmap.yaml
-  - deployment.yaml
-  - secret.yaml
+namespace: dask-gateway
+resources:
+- configmap.yaml
+- deployment.yaml
+- secret.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+labels:
+- includeSelectors: true
+  pairs:
+    app.kubernetes.io/instance: '{{ app_name }}'

apps/01-eo-cnpgstac/kustomization.yaml

@@ -12,19 +12,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-commonLabels:
-  app.kubernetes.io/instance: "{{ app_name }}"
 namespace: database
 
 helmCharts:
 - name: eo-cnpgstac
+  releaseName: '{{ app_name }}'
   repo: oci://643vlk6z.gra7.container-registry.ovh.net/public/
-  releaseName: "{{ app_name }}"
-  version: 1.0.0-alpha
   valuesFile: values.yaml
+  version: 1.0.0-alpha
 
 resources:
-  - grafanadatasource.yaml
-  - podmonitor.yaml
-  - scheduledbackup.yaml
-  - secret.yaml
+- grafanadatasource.yaml
+- podmonitor.yaml
+- scheduledbackup.yaml
+- secret.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+labels:
+- includeSelectors: true
+  pairs:
+    app.kubernetes.io/instance: '{{ app_name }}'

apps/01-eo-cnpgstac/values.yaml

@@ -12,9 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-commonLabels:
-  wait-for-deployment: Ready
-
 ## @section CloudNative PostGIS parameters
 cnpgstac:
   ## @param cnpgstac.enabled Enabled CloudNative PostGIS
@@ -23,6 +20,9 @@ cnpgstac:
   ## @param cnpgstac.clusterName Name of the cnpgstac cluster
   clusterName: cnpgstac
 
+  clusterLabels:
+    wait-for-deployment: Ready
+
   ## Define postgres user password
   ## ref: https://cloudnative-pg.io/documentation/current/security/#postgresql
   superUserAccess:
@@ -88,7 +88,7 @@ cnpgstac:
 
     ## Enable readonly mode in pgstac
     ## If true, usage of titiler-pgstac is limited
-    ## ref: https://github.com/stac-utils/pgstac/pull/215 and https://github.com/stac-utils/pgstac/issues/296 
+    ## ref: https://github.com/stac-utils/pgstac/pull/215 and https://github.com/stac-utils/pgstac/issues/296
     enableReadOnly: true
 
   ## CloudNative PostGIS resource requests and limits
@@ -138,4 +138,4 @@ cnpgstac:
   tolerations:
     - effect: NoSchedule
       key: role
-      value: rs_server
+      value: rs_server

apps/01-pi-db/kustomization.yaml

@@ -12,11 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-commonLabels:
-  app.kubernetes.io/instance: "{{ app_name }}"
 
 namespace: database
 
 resources:
-  - database.yaml
-  - grafanadatasource.yaml
+- database.yaml
+- grafanadatasource.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+labels:
+- includeSelectors: true
+  pairs:
+    app.kubernetes.io/instance: '{{ app_name }}'

apps/01-pygeoapi-db/kustomization.yaml

@@ -12,11 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-commonLabels:
-  app.kubernetes.io/instance: "{{ app_name }}"
 
 namespace: database
 
 resources:
-  - database.yaml
-  - grafanadatasource.yaml
+- database.yaml
+- grafanadatasource.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+labels:
+- includeSelectors: true
+  pairs:
+    app.kubernetes.io/instance: '{{ app_name }}'