Multiple prefect instances (#33)

* feat: update vars

* fix: CI

* fix: CI wrong ref for branch on PR

* fix: use updated inventory variables

* fix: use updated variable name in prefect database

* fix: missing parenthesis

Author: nleconte-csgroup

.github/workflows/test-deployment.yml

@@ -30,10 +30,26 @@ jobs:
     if: github.actor != 'dependabot[bot]' # ignore pull requests by github bot
     runs-on: ubuntu-latest
     steps:
+      - name: Get current branch name
+        id: vars
+        run: echo "branch_name=${{ github.head_ref }}" >> $GITHUB_OUTPUT
+
+      - name: Check if branch exists in rs-infra-core
+        id: check_branch
+        run: |
+          BRANCH=${{ steps.vars.outputs.branch_name }}
+          if git ls-remote --heads https://github.com/RS-PYTHON/rs-infra-core.git $BRANCH | grep -q $BRANCH; then
+            echo "branch_to_use=$BRANCH" >> $GITHUB_OUTPUT
+          else
+            echo "branch_to_use=" >> $GITHUB_OUTPUT  # Leave empty to use default
+          fi
+        shell: bash
+
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
           repository: RS-PYTHON/rs-infra-core
+          ref: ${{ steps.check_branch.outputs.branch_to_use }}
           submodules: recursive
       - name: Checkout repository
         uses: actions/checkout@v4

apps/01-prefect3-db/database.yaml

@@ -15,30 +15,30 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: create-prefect3-db
+  name: "create-{{ prefect3server.ops.name }}-db"
 spec:
   ttlSecondsAfterFinished: 360
   template:
     spec:
       containers:
-      - name: create-prefect3-db
+      - name: "create-{{ prefect3server.ops.name }}-db"
         image: ghcr.io/cloudnative-pg/postgresql:16.1
         command: ["/scripts/create-database.sh"]
         volumeMounts:
           - name: create-database
             mountPath: /scripts/
         env:
         - name: DATABASE
-          value: {{ prefect3.database.name }}
+          value: "{{ prefect3server.ops.name }}"
         - name: PGNEWUSER
           valueFrom:
             secretKeyRef:
-              name: {{ prefect3.database.secret }}
+              name: "{{ prefect3server.ops.name }}-database-password"
               key: username
         - name: PGNEWPASS
           valueFrom:
             secretKeyRef:
-              name: {{ prefect3.database.secret }}
+              name: "{{ prefect3server.ops.name }}-database-password"
               key: password
         - name: PGADMIN
           valueFrom:
@@ -82,10 +82,10 @@ spec:
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ prefect3.database.secret }}
+  name: "{{ prefect3server.ops.name }}-database-password"
   labels: 
     app: prefect3
 type: Opaque
 stringData:
-  username: {{ prefect3.database.username }}
-  password: {{ prefect3.database.password }}
+  username: "{{ prefect3server.ops.database.username }}"
+  password: "{{ prefect3server.ops.database.password }}"

apps/prefect3-server/values.yaml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 # -- fully override common.names.fullname
-fullnameOverride: "prefect-server"
+fullnameOverride: "{{ prefect3server.ops.name }}-server"
 
 global:
   prefect:
@@ -30,12 +30,12 @@ server:
     # -- set PREFECT_UI_ENABLED; enable the UI on the server
     enabled: true
     # -- sets PREFECT_UI_API_URL
-    #prefectUiApiUrl: "https://processing.{{ platform_domain_name }}/api"
+    #prefectUiApiUrl: "https://{{ prefect3server.ops.subDomain }}.{{ platform_domain_name }}/api"
     # -- sets PREFECT_UI_URL
-    #prefectUiUrl: "https://processing.{{ platform_domain_name }}"
+    #prefectUiUrl: "https://{{ prefect3server.ops.subDomain }}.{{ platform_domain_name }}"
     # -- sets PREFECT_UI_STATIC_DIRECTORY
     #prefectUiStaticDirectory: "/ui_build"
-    prefectUiApiUrl: https://processing.{{ platform_domain_name }}/api
+    prefectUiApiUrl: "https://{{ prefect3server.ops.subDomain }}.{{ platform_domain_name }}/api"
 
   env:
     - name: PREFECT_LOGGING_EXTRA_LOGGERS
@@ -95,16 +95,16 @@ ingress:
   className: "nginx"
   #path must be /
   host:
-    hostname: processing.{{ platform_domain_name }}
+    hostname: "{{ prefect3server.ops.subDomain }}.{{ platform_domain_name }}"
     path: /
     pathType: Prefix
 
   annotations: 
     cert-manager.io/cluster-issuer: letsencrypt-prod
     nginx.ingress.kubernetes.io/backend-protocol: HTTP
     nginx.ingress.kubernetes.io/proxy-buffer-size: 32k
-    nginx.ingress.kubernetes.io/auth-signin: https://oauth2-proxy.{{ platform_domain_name }}/oauth2/start?rd=https://$host
-    nginx.ingress.kubernetes.io/auth-url: https://oauth2-proxy.{{ platform_domain_name }}/oauth2/auth
+    nginx.ingress.kubernetes.io/auth-signin: "https://oauth2-proxy.{{ platform_domain_name }}/oauth2/start?rd=https://$host"
+    nginx.ingress.kubernetes.io/auth-url: "https://oauth2-proxy.{{ platform_domain_name }}/oauth2/auth?allowed_groups={{ prefect3server.ops.allowedRoles | default('role:RS-ADMIN', true) }}"
     nginx.ingress.kubernetes.io/auth-response-headers: Authorization
 
   tls: true
@@ -113,8 +113,8 @@ ingress:
 
   extraTls:
     - hosts:
-        - processing.{{ platform_domain_name }}
-      secretName: processing-{{ platform_domain_name | replace(".", "-") }}
+        - "{{ prefect3server.ops.subDomain }}.{{ platform_domain_name }}"
+      secretName: "{{ prefect3server.ops.subDomain }}-{{ platform_domain_name | replace(".", "-") }}"
 
 # Disable bundled postgresql
 postgresql:
@@ -123,9 +123,9 @@ postgresql:
 # Postgresql credentials
 secret:
   create: true
-  name: {{ prefect3.database.secret }}
-  username: {{ prefect3.database.username }}
-  password: {{ prefect3.database.password }}
-  host: {{ prefect3.database.host }}
+  name: "{{ prefect3server.ops.name }}-database-password"
+  username: "{{ prefect3server.ops.database.username }}"
+  password: "{{ prefect3server.ops.database.password }}"
+  host: "{{ prefect3server.ops.database.host }}"
   port: 5432
-  database: {{ prefect3.database.name }}
+  database: "{{ prefect3server.ops.name }}"

apps/prefect3-worker-eopf/values.yaml

@@ -24,10 +24,10 @@ worker:
     pullPolicy: IfNotPresent
   apiConfig: selfHostedServer
   config:
-    workPool: {{ prefect3worker.eopf.name }}
+    workPool: "{{ prefect3worker.eopf.name }}"
     limit: 200
     baseJobTemplate:
-      name: {{ prefect3worker.eopf.name }}-base-job-template
+      name: "{{ prefect3worker.eopf.name }}-base-job-template"
       configuration: |
         {
           "variables": {
@@ -291,7 +291,7 @@ worker:
       effect: NoSchedule
 
   selfHostedServerApiConfig:
-    apiUrl: http://prefect-server.processing.svc.cluster.local:4200/api
+    apiUrl: "http://{{ prefect3server.ops.name }}-server.processing.svc.cluster.local:4200/api"
 
   # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
   livenessProbe:

apps/prefect3-worker-general/values.yaml

@@ -24,10 +24,10 @@ worker:
     pullPolicy: IfNotPresent
   apiConfig: selfHostedServer
   config:
-    workPool: {{ prefect3worker.general.name }}
+    workPool: "{{ prefect3worker.general.name }}"
     limit: 200
     baseJobTemplate:
-      name: {{ prefect3worker.general.name }}-base-job-template
+      name: "{{ prefect3worker.general.name }}-base-job-template"
       configuration: |
         {
           "variables": {
@@ -291,7 +291,7 @@ worker:
       effect: NoSchedule
 
   selfHostedServerApiConfig:
-    apiUrl: http://prefect-server.processing.svc.cluster.local:4200/api
+    apiUrl: "http://{{ prefect3server.ops.name }}-server.processing.svc.cluster.local:4200/api"
 
   # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
   livenessProbe:

apps/prefect3-worker-staging/values.yaml

@@ -24,10 +24,10 @@ worker:
     pullPolicy: IfNotPresent
   apiConfig: selfHostedServer
   config:
-    workPool: {{ prefect3worker.staging.name }}
+    workPool: "{{ prefect3worker.staging.name }}"
     limit: 200
     baseJobTemplate:
-      name: {{ prefect3worker.staging.name }}-base-job-template
+      name: "{{ prefect3worker.staging.name }}-base-job-template"
       configuration: |
         {
           "variables": {
@@ -291,7 +291,7 @@ worker:
       effect: NoSchedule
 
   selfHostedServerApiConfig:
-    apiUrl: http://prefect-server.processing.svc.cluster.local:4200/api
+    apiUrl: "http://{{ prefect3server.ops.name }}-server.processing.svc.cluster.local:4200/api"
 
   # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
   livenessProbe: