build: commons celery configuration

Author: arcstur

deployment/celery-editgroups-commons.yaml

@@ -0,0 +1,35 @@
+# Run Celery on kubernetes
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: editgroups-commons.celery.sh
+  namespace: tool-editgroups-commons
+  labels:
+    name: editgroups-commons.celery.sh
+    toolforge: tool
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: editgroups-commons.celery.sh
+      toolforge: tool
+  template:
+    metadata:
+      labels:
+        name: editgroups-commons.celery.sh
+        toolforge: tool
+    spec:
+      containers:
+        - name: celery
+          image: docker-registry.tools.wmflabs.org/toolforge-python311-sssd-base:latest
+          command: [ "/data/project/editgroups-commons/www/python/src/tasks-commons.sh" ]
+          workingDir: /data/project/editgroups-commons/www/python/src
+          env:
+            - name: HOME
+              value: /data/project/editgroups-commons
+          imagePullPolicy: Always
+          resources:
+            requests:
+              memory: "512Mi"
+            limits:
+              memory: "1024Mi"

restart_celery.sh

@@ -1,6 +1,15 @@
 #!/bin/bash
 export GOMAXPROCS=1
-kubectl delete deployment editgroups.celery.sh ;
+TOOLNAME=$(whoami | cut -d "." -f 2)
+echo "Running as ${TOOLNAME}"
+
+if [[ "$TOOLNAME" == "editgroups" ]]; then
+  CELERY_FN=celery
+else
+  CELERY_FN="celery-${TOOLNAME}"
+fi
+
+kubectl delete deployment "${TOOLNAME}.celery.sh";
 echo "Waiting for Celery to stop";
-sleep 45 ;
-kubectl create -f /data/project/editgroups/www/python/src/deployment/celery.yaml && kubectl get pods
+sleep 45;
+kubectl create -f "/data/project/${TOOLNAME}/www/python/src/deployment/${CELERY_FN}.yaml"

tasks-commons.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+
+VENV_DIR=/data/project/editgroups-commons/www/python/venv
+
+if [[ -f ${VENV_DIR}/bin/activate ]]; then
+      source ${VENV_DIR}/bin/activate
+else
+      echo "Creating virtualenv"
+      rm -rf ${VENV_DIR}
+      pyvenv ${VENV_DIR}
+      source ${VENV_DIR}/bin/activate
+      echo "Installing requirements"
+      pip install -r requirements.txt
+fi;
+echo "Starting celery"
+export C_FORCE_ROOT=True
+/data/project/editgroups-commons/www/python/venv/bin/python3 /data/project/editgroups-commons/www/python/venv/bin/celery --app=editgroups-commons.celery:app worker -l INFO -B  --concurrency=3 --max-memory-per-child=50000
+echo $?
+echo "Celery done"
+