Allow splitting image to multiple containers for Kubernetes deployment

.github/workflows/release-latest.yml

@@ -3,14 +3,15 @@ name: Build the Docker images and push them to the container registry
 on:
   push:
     branches: [ "master" ]
+  workflow_dispatch: # manual trigger
 
 jobs:
   build:
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        target: [misp-core, misp-modules, misp-core-slim, misp-modules-slim]
+        target: [misp-core, misp-modules, misp-core-slim, misp-modules-slim, misp-web]
 
     permissions:
       contents: read
@@ -32,7 +33,8 @@ jobs:
         run: |
           sed -e '/^[[:space:]]*$/d' -e '/[#@]/d' -e 's/\"//g' -e 's/\(^[^=]*\)=\(.*\)/\1="\2"/' template.env > env.hcl
           echo "COMMIT_HASH=`echo '${{ github.sha }}' | cut -c 1-7`" >> "$GITHUB_ENV"
-          echo "NAMESPACE=ghcr.io/misp/misp-docker" >> "$GITHUB_ENV"
+          OWNER=$(echo "$GITHUB_REPOSITORY_OWNER" | tr '[:upper:]' '[:lower:]')
+          echo "NAMESPACE=ghcr.io/${OWNER}/misp-docker" >> "$GITHUB_ENV"
 
       - name: Log in to the container registry
         uses: docker/login-action@v3

.github/workflows/test-build-latest.yml

@@ -10,7 +10,7 @@ jobs:
 
     strategy:
       matrix:
-        target: [misp-core, misp-modules, misp-core-slim, misp-modules-slim]
+        target: [misp-core, misp-modules, misp-core-slim, misp-modules-slim, misp-web]
 
     steps:
       - name: Checkout repository

core/Dockerfile-web

@@ -0,0 +1,27 @@
+# Use a minimal and secure official NGINX image
+FROM nginxinc/nginx-unprivileged:stable-alpine
+
+# Set working directory
+WORKDIR /app
+
+# Copy necessary from the local 'files' directory into the container
+COPY --chown=101:0 files/etc/nginx/ /etc/nginx/
+COPY --chown=101:0 files/var/www/html/ /var/www/html/
+COPY --chown=101:0 files/entrypoint_nginx.sh /app/.
+COPY --chown=101:0 files/entrypoint_k8s_nginx.sh /app/.
+
+#DEBUG
+RUN echo "Listing /app:" && ls -la /app && \
+    echo "Listing /etc/nginx:" && ls -la /etc/nginx && \
+    echo "Listing /var/www/html:" && ls -la /var/www/html
+
+# Ensure appropriate permissions
+RUN chmod -R 755 /etc/nginx /var/www/html /app
+# Ensure the entrypoint script is executable
+RUN chmod +x /app/entrypoint_k8s_nginx.sh
+
+# Set the entrypoint
+ENTRYPOINT ["/app/entrypoint_k8s_nginx.sh"]
+
+# Use a non-root user (already set in nginx-unprivileged image)
+USER 101

core/files/entrypoint.sh

@@ -76,5 +76,12 @@ export NGINX_X_FORWARDED_FOR=${NGINX_X_FORWARDED_FOR:-false}
 export NGINX_SET_REAL_IP_FROM=${NGINX_SET_REAL_IP_FROM}
 export NGINX_CLIENT_MAX_BODY_SIZE=${NGINX_CLIENT_MAX_BODY_SIZE:-50M}
 
-# start supervisord using the main configuration file so we have a socket interface
-/usr/local/bin/supervisord -c /etc/supervisor/supervisord.conf
+if [ "$MISP_PHP_ONLY" ]; then
+    # Not ideal, but let supervisord manage the workers still
+    mv /etc/supervisor/conf.d/10-supervisor.conf{.k8s,}
+    /usr/local/bin/supervisord -c /etc/supervisor/supervisord.conf &
+    exec /entrypoint_k8s_fpm.sh
+else
+    # start supervisord using the main configuration file so we have a socket interface
+    /usr/local/bin/supervisord -c /etc/supervisor/supervisord.conf
+fi

core/files/entrypoint_fpm.sh

@@ -32,6 +32,7 @@ change_php_vars() {
         sed -i "s/session.use_strict_mode = .*/session.use_strict_mode = 1/" "$FILE"
         echo "Configure PHP | Setting 'date.timezone = ${PHP_TIMEZONE}'"
         sed -i "s/;?date.timezone = .*/date.timezone = ${PHP_TIMEZONE}/" "$FILE"
+        sed -i "s|session.cookie_domain = .*|session.cookie_domain = ${BASE_URL}|" "$FILE"
     done
 
     for FILE in /etc/php/*/fpm/pool.d/www.conf
@@ -61,9 +62,21 @@ change_php_vars() {
             echo "Configure PHP | Disabling 'pm.status_listen'"
             sed -i -E "s/^pm.status_listen =/;pm.status_listen =/" "$FILE"
         fi
+        if [ "$MISP_PHP_ONLY" ] && [ -n "$PHP_HOST" ]; then
+            echo "Configure PHP | Setting 'listen' to 0.0.0.0:${PHP_FPM_PORT:-9000}"
+            sed -i "/^listen =/s@=.*@= 0.0.0.0:${PHP_FPM_PORT:-9000}@" "$FILE"
+        elif [[ -n "$PHP_FPM_SOCK_FILE" ]]; then
+            echo "Configure PHP | Setting 'listen' to ${PHP_FPM_SOCK_FILE}"
+            sed -i "/^listen =/s@=.*@= ${PHP_FPM_SOCK_FILE}@" "$FILE"
+        fi
     done
 }
 
+# Return to skip running below commands if not sourced
+if [ -n "${BASH_SOURCE[0]}" ]; then
+    return
+fi
+
 echo "Configure PHP | Change PHP values ..." && change_php_vars
 
 echo "Configure PHP | Starting PHP FPM"

core/files/entrypoint_k8s_fpm.sh

@@ -0,0 +1,26 @@
+#!/bin/bash -e
+
+source /entrypoint_nginx.sh
+source /entrypoint_fpm.sh
+
+# Initialize MySQL
+echo "INIT | Initialize MySQL ..." && init_mysql
+
+# Initialize MISP
+echo "INIT | Initialize MISP files and configurations ..." && init_misp_data_files
+echo "INIT | Update MISP app/files directory ..." && update_misp_data_files
+echo "INIT | Enforce MISP permissions ..." && enforce_misp_data_permissions
+
+# Run configure MISP script
+echo "INIT | Configure MISP installation ..."
+/configure_misp.sh
+
+if [[ -x /custom/files/customize_misp.sh ]]; then
+    echo "INIT | Customize MISP installation ..."
+    /custom/files/customize_misp.sh
+fi
+
+echo "Configure PHP | Change PHP values ..." && change_php_vars
+
+echo "Configure PHP | Starting PHP FPM"
+exec /usr/sbin/php-fpm8.2 -R -F

core/files/entrypoint_k8s_nginx.sh

@@ -0,0 +1,18 @@
+#!/bin/bash -e
+
+source ./entrypoint_nginx.sh
+
+# Initialize nginx
+echo "INIT | Initialize NGINX ..." && init_nginx
+
+# Configure NGINX to connec to PHP-FPM over TCP if a host is provided
+if [[ -n "$PHP_FPM_HOST" ]]; then
+    echo "... setting 'fastcgi_pass' to $PHP_FPM_HOST:${PHP_FPM_PORT:-9000}"
+    sed -i "s@fastcgi_pass .*;@fastcgi_pass $PHP_FPM_HOST:${PHP_FPM_PORT:-9000};@" /etc/nginx/includes/misp
+fi
+
+echo "INIT | Flip NGINX live ..." && flip_nginx true true
+
+# launch nginx as current shell process in container
+exec nginx -g 'daemon off;'
+

core/files/entrypoint_nginx.sh

@@ -255,13 +255,19 @@ flip_nginx() {
     echo "... nginx docroot set to ${NGINX_DOC_ROOT}"
     sed -i "s|root.*var/www.*|root ${NGINX_DOC_ROOT};|" /etc/nginx/includes/misp
 
-    if [[ "$reload" = "true" ]]; then
+    if [[ "$reload" = "true" ]] && [[ -z "$KUBERNETES_SERVICE_HOST" ]]; then
         echo "... nginx reloaded"
         nginx -s reload
     fi
 }
 
 init_nginx() {
+    # Optional location of PHP-FPM sock file
+    if [[ -n "$PHP_FPM_SOCK_FILE" ]]; then
+        echo "... setting 'fastcgi_pass' to unix:${PHP_FPM_SOCK_FILE}"
+        sed -i "s@fastcgi_pass .*;@fastcgi_pass unix:${PHP_FPM_SOCK_FILE};@" /etc/nginx/includes/misp
+    fi
+
     # Adjust timeouts
     echo "... adjusting 'fastcgi_read_timeout' to ${FASTCGI_READ_TIMEOUT}"
     sed -i "s/fastcgi_read_timeout .*;/fastcgi_read_timeout ${FASTCGI_READ_TIMEOUT};/" /etc/nginx/includes/misp
@@ -401,6 +407,10 @@ init_nginx() {
     flip_nginx false false
 }
 
+# Return to skip running below commands if not sourced
+if [ -n "${BASH_SOURCE[0]}" ]; then
+    return
+fi
 
 # Initialize MySQL
 echo "INIT | Initialize MySQL ..." && init_mysql

core/files/etc/supervisor/conf.d/10-supervisor.conf.k8s

@@ -0,0 +1,12 @@
+[supervisord]
+nodaemon=true
+user=root
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[inet_http_server]
+port=127.0.0.1:9001
+username=supervisor
+password=supervisor

docker-bake.hcl

@@ -84,6 +84,7 @@ group "default" {
     "misp-modules-slim",
     "misp-core",
     "misp-core-slim",
+    "misp-web",
   ]
 }
 
@@ -160,3 +161,10 @@ target "misp-core-slim" {
   }
   platforms = "${PLATFORMS}"
 }
+
+target "misp-web" {
+  context = "core/."
+  dockerfile = "Dockerfile-web"
+  tags = flatten(["${NAMESPACE}/misp-web:latest", "${NAMESPACE}/misp-web:${COMMIT_HASH}" ])
+  platforms = "${PLATFORMS}"
+}

kubernetes/k8s_cronjobs/base/base-api-cronjob.yaml

@@ -0,0 +1,27 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: misp-curljob
+spec:
+  schedule: "* * * * *" # Placeholder; Overridden in overlays
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: OnFailure
+          containers:
+            - name: curl-job
+              image: curlimages/curl:latest
+              command: ["/bin/sh", "-c"]
+              args: ["echo 'Placeholder Command'"] # Placeholder; Overridden in overlays
+              env:
+                - name: BASE_URL
+                  valueFrom:
+                    secretKeyRef:
+                      name: misp-cron-secret
+                      key: url
+                - name: MISP_API_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: misp-cron-secret
+                      key: api_key

kubernetes/k8s_cronjobs/base/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - base-api-cronjob.yaml

kubernetes/k8s_cronjobs/k8s-misp-cron-secret.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: misp-cron-secret
+type: Opaque
+stringData:
+  url: "https://misp.example.com" # Replace with your MISP instance URL
+data:
+  api_key: "" # Base64 encoded value of your MISP API key goes here

kubernetes/k8s_cronjobs/kustomization.yaml

@@ -0,0 +1,9 @@
+resources:
+  - overlays/cacheFeed
+  - overlays/fetchFeed
+  - overlays/pullAll
+  - overlays/pushAll
+  - overlays/updateGalaxies
+  - overlays/updateNoticeLists
+  - overlays/updateTaxonomies
+  - overlays/updateWarningLists

kubernetes/k8s_cronjobs/overlays/cacheFeed/kustomization.yaml

@@ -0,0 +1,7 @@
+resources:
+  - ../../base/
+
+patches:
+  - path: patch.yaml
+
+nameSuffix: -feed-cache

kubernetes/k8s_cronjobs/overlays/cacheFeed/patch.yaml

@@ -0,0 +1,18 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: misp-curljob
+spec:
+  schedule: "20 2 * * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: curl-job
+            args:
+              - >
+                curl -sS -X POST "$BASE_URL/feeds/cacheFeeds/all"
+                -H "Accept: application/json"
+                -H "Content-Type: application/json"
+                -H "Authorization: $MISP_API_KEY"

kubernetes/k8s_cronjobs/overlays/fetchFeed/kustomization.yaml

@@ -0,0 +1,7 @@
+resources:
+  - ../../base/
+
+patches:
+  - path: patch.yaml
+
+nameSuffix: -feed-fetch

kubernetes/k8s_cronjobs/overlays/fetchFeed/patch.yaml

@@ -0,0 +1,18 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: misp-curljob
+spec:
+  schedule: "30 2 * * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: curl-job
+            args:
+              - >
+                curl -sS -X POST "$BASE_URL/feeds/fetchFromAllFeeds"
+                -H "Accept: application/json"
+                -H "Content-Type: application/json"
+                -H "Authorization: $MISP_API_KEY"

kubernetes/k8s_cronjobs/overlays/pullAll/kustomization.yaml

@@ -0,0 +1,7 @@
+resources:
+  - ../../base/
+
+patches:
+  - path: patch.yaml
+
+nameSuffix: -pullall

kubernetes/k8s_cronjobs/overlays/pullAll/patch.yaml

@@ -0,0 +1,25 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: misp-curljob
+spec:
+  schedule: "0 1 * * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: curl-job
+            args:
+              - >
+                curl -sS -X GET "$BASE_URL/servers"
+                -H "Accept: application/json"
+                -H "Content-Type: application/json"
+                -H "Authorization: $MISP_API_KEY"
+                | awk '/"Server":/,/}/'
+                | grep -o  '"id": "[0-9]*"' 
+                | grep -o '[0-9]\+'
+                | xargs -I {} curl -sS -X POST $BASE_URL/servers/pull/{}
+                -H "Accept: application/json"
+                -H "Content-Type: application/json"
+                -H "Authorization: $MISP_API_KEY"

kubernetes/k8s_cronjobs/overlays/pushAll/kustomization.yaml

@@ -0,0 +1,7 @@
+resources:
+  - ../../base/
+
+patches:
+  - path: patch.yaml
+
+nameSuffix: -pushall