feat(service): horizontal scaling (#3178)

Author: olevski

chartpress.yaml

@@ -15,3 +15,4 @@ charts:
         buildArgs:
           CLEAN_INSTALL: "1"
           BUILD_CORE_SERVICE: "1"
+

helm-chart/renku-core/templates/configmap.yaml

@@ -1,54 +1,9 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ include "renku-core.fullname" . }}-nginx
+  name: {{ include "renku-core.fullname" . }}-metadata-versions
 data:
-  nginx-server-blocks.conf: |
-    {{- range $version := .Values.versions }}
-    upstream {{ $version.name }} {
-      server {{ include "renku-core.fullname" $ }}-{{ $version.name }};
-      keepalive 32;
-      keepalive_timeout 60s;
-    }
-    {{ end }}
-
-    server {
-      listen 8080;
-
-      server_name {{ include "renku-core.fullname" . }};
-
-      sendfile on;
-      tcp_nopush on;
-      client_max_body_size 0; # Required for uploading large files
-
-      location /renku/version {
-        root /;
-        add_header Content-Type application/json;
-        try_files /usr/share/nginx/html/version.json =404;
-      }
-
-      {{- range $version := .Values.versions }}
-      location /renku/{{ $version.prefix }} {
-        rewrite /renku/{{ $version.prefix }}/(.*) /renku/$1  break;
-        proxy_set_header Host $host;
-        proxy_pass http://{{ $version.name }};
-        proxy_send_timeout {{ $.Values.requestTimeout }}s;
-        proxy_read_timeout {{ $.Values.requestTimeout }}s;
-        proxy_http_version 1.1;
-        proxy_set_header "Connection" "";
-      }
-      {{- end }}
-
-      location /renku {
-        proxy_set_header Host $host;
-        proxy_pass http://{{ .Values.versions.latest.name }};
-        proxy_send_timeout {{ $.Values.requestTimeout }}s;
-        proxy_read_timeout {{ $.Values.requestTimeout }}s;
-        proxy_http_version 1.1;
-        proxy_set_header "Connection" "";
-      }
-    }
-  version.json: |
+  metadata-versions.json: |
     {
       "name": "renku-core",
       "versions": [

helm-chart/renku-core/templates/deployment-nginx.yaml

@@ -17,7 +17,10 @@ metadata:
 spec:
   replicas: {{ $.Values.replicaCount }}
   strategy:
-    type: Recreate
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 0
+      maxSurge: 1
   selector:
     matchLabels:
       app.kubernetes.io/name: {{ include "renku-core.name" $ }}
@@ -44,6 +47,9 @@ spec:
       volumes:
         - name: shared-volume
           emptyDir: {}
+        - name: metadata-versions
+          configMap:
+            name: {{ include "renku-core.fullname" $ }}-metadata-versions
         {{- include "certificates.volumes" $ | nindent 8 }}
       initContainers:
         {{- include "certificates.initContainer" $ | nindent 8 }}
@@ -139,10 +145,14 @@ spec:
               value: {{ $.Values.global.renku.domain }}
             - name: RENKU_PROJECT_DEFAULT_CLI_VERSION
               value: {{ $.Values.global.renku.cli_version | default "" | quote }}
+            - name: METADATA_VERSIONS_LIST
+              value: /svc/config/metadata-versions/metadata-versions.json
             {{- include "certificates.env.python" $ | nindent 12 }}
           volumeMounts:
             - name: shared-volume
               mountPath: {{ $.Values.cacheDirectory }}
+            - name: metadata-versions
+              mountPath: /svc/config/metadata-versions
             {{- include "certificates.volumeMounts.system" $ | nindent 12 }}
           ports:
             - name: http
@@ -157,7 +167,7 @@ spec:
               path: /health
               port: http
           resources:
-            {{- toYaml $.Values.resources | nindent 12 }}
+            {{- toYaml $.Values.resources.core | nindent 12 }}
         - name: {{ $.Chart.Name }}-datasets-workers
           image: "{{ $version.image.repository }}:{{ $version.image.tag }}"
           imagePullPolicy: {{ $version.image.pullPolicy }}
@@ -213,6 +223,8 @@ spec:
             - name: shared-volume
               mountPath: {{ $.Values.cacheDirectory }}
             {{- include "certificates.volumeMounts.system" $ | nindent 12 }}
+          resources:
+            {{- toYaml $.Values.resources.datasetsWorkers | nindent 12 }}
 
         - name: {{ $.Chart.Name }}-management-workers
           image: "{{ $version.image.repository }}:{{ $version.image.tag }}"
@@ -267,6 +279,8 @@ spec:
             - name: shared-volume
               mountPath: {{ $.Values.cacheDirectory }}
             {{- include "certificates.volumeMounts.system" $ | nindent 12 }}
+          resources:
+            {{- toYaml $.Values.resources.managementWorkers | nindent 12 }}
 
         - name: {{ $.Chart.Name }}-scheduler
           image: "{{ $version.image.repository }}:{{ $version.image.tag }}"
@@ -307,6 +321,8 @@ spec:
             {{- include "certificates.env.python" $ | nindent 12 }}
           volumeMounts:
             {{- include "certificates.volumeMounts.system" $ | nindent 12 }}
+          resources:
+            {{- toYaml $.Values.resources.scheduler | nindent 12 }}
       {{- with $.Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

helm-chart/renku-core/templates/deployment.yaml

@@ -0,0 +1,29 @@
+{{- range $version := .Values.versions }}
+---
+{{- if $.Capabilities.APIVersions.Has "autoscaling/v2" }}
+apiVersion: autoscaling/v2
+{{- else if $.Capabilities.APIVersions.Has "autoscaling/v2beta2" }}
+apiVersion: autoscaling/v2beta2
+{{- else if $.Capabilities.APIVersions.Has "autoscaling/v2beta1" }}
+apiVersion: autoscaling/v2beta1
+{{- else }}
+  {{- fail "ERROR: You must have at least autoscaling/v2beta1 to use HorizontalPodAutoscaler" }}
+{{- end }}
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "renku-core.fullname" $ }}-{{ $version.name }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "renku-core.fullname" $ }}-{{ $version.name }}
+  minReplicas: {{ $.Values.horizontalPodAutoscaling.minReplicas }}
+  maxReplicas: {{ $.Values.horizontalPodAutoscaling.maxReplicas }}
+  metrics:
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: {{ $.Values.horizontalPodAutoscaling.averageMemoryUtilization }}
+{{ end }}

helm-chart/renku-core/templates/hpa.yaml

@@ -0,0 +1,12 @@
+{{- range $version := .Values.versions }}
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ include "renku-core.fullname" $ }}-{{ $version.name }}
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/deploymentVersion: {{ $version.name }}
+{{ end }}

helm-chart/renku-core/templates/pdb.yaml

@@ -101,16 +101,6 @@
           },
           "type": "object"
       },
-      "ingress": {
-        "description": "Kubernetes ingress definition",
-        "properties": {
-            "enabled": {
-                "description": "whether to enable the ingress or not",
-                "type": "boolean"
-            }
-        },
-        "type": "object"
-      },
       "metrics": {
           "description": "Definition of Redis Queue metrics",
           "properties": {
@@ -141,8 +131,30 @@
           "type": "object"
       },
       "nodeSelector": {
-          "description": "Node selector for deployment",
-          "type": "object"
+        "description": "Node selector for deployment",
+        "type": "object"
+    },
+      "horizontalPodAutoscaling": {
+          "description": "Setup for scaling the core service",
+          "type": "object",
+          "properties": {
+            "minReplicas": {
+                "description": "Flag to turn on/off Sentry",
+                "type": "integer",
+                "minimum": 1
+            },
+            "maxReplicas": {
+                "description": "URI of the sentry Data Source Name",
+                "type": "integer",
+                "minimum": 1
+            },
+            "averageMemoryUtilization": {
+                "description": "Name of the sentry environment to post to",
+                "type": "integer",
+                "exclusiveMinimum": 0,
+                "exclusiveMaximum": 100
+            }
+        }
       },
       "tolerations": {
           "description": "Tolerations for deployment",
@@ -237,7 +249,6 @@
         "gitLFSSkipSmudge",
         "jwtTokenSecret",
         "service",
-        "ingress",
         "metrics",
         "sentry",
         "versions"

helm-chart/renku-core/templates/service-nginx.yaml

@@ -1,7 +1,7 @@
 # Default values for renku-core.
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
-replicaCount: 1
+replicaCount: 2
 global:
   ## Specify a secret that containes the certificate
   ## if you would like to use a custom CA. The key for the secret
@@ -56,25 +56,18 @@ fullnameOverride: ""
 service:
   type: ClusterIP
   port: 80
-ingress:
-  enabled: false
 metrics:
   enabled: false
   image:
     repository: renku/rqmetrics
     tag: 0.0.2
     pullPolicy: IfNotPresent
-resources: {}
-# We usually recommend not to specify default resources and to leave this as a conscious
-# choice for the user. This also increases chances charts run on environments with little
-# resources, such as Minikube. If you do want to specify resources, uncomment the following
-# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
-# limits:
-#   cpu: 100m
-#   memory: 128Mi
-# requests:
-#   cpu: 100m
-#   memory: 128Mi
+resources:
+  core: {}
+  rqmetrics: {}
+  datasetsWorkers: {}
+  managementWorkers: {}
+  scheduler: {}
 
 # nodeSelector: {}
 
@@ -114,3 +107,8 @@ podSecurityContext:
   fsGroup: 100
 securityContext:
   allowPrivilegeEscalation: false
+
+horizontalPodAutoscaling:
+  minReplicas: 2
+  maxReplicas: 10
+  averageMemoryUtilization: 50

helm-chart/renku-core/values.schema.json

@@ -1,5 +1,5 @@
 #
-# Copyright 2020 - Swiss Data Science Center (SDSC)
+# Copyright 2022 - Swiss Data Science Center (SDSC)
 # A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
 # Eidgenössische Technische Hochschule Zürich (ETHZ).
 #
@@ -78,3 +78,6 @@
 # Sentry configuration
 SENTRY_ENABLED = os.getenv("SENTRY_ENABLED", "false").lower() == "true"
 SENTRY_SAMPLERATE = float(os.getenv("SENTRY_SAMPLE_RATE", 0.2))
+
+# List of all available metadata versions
+METADATA_VERSIONS_LIST = os.getenv("METADATA_VERSIONS_LIST", "/svc/config/metadata-versions/metadata-versions.json")