Introduce beforeExit and switch to watchman

beforeExit allows to run a series of commands before exit occurs.  The function
is async.

- setExitHandler is an unpublished escape hatch not intended for public use
- exitHandler is not intended for public use but is necessary in start-server.js
  which has been given its own file

nodemon has been removed

watchman has its own set of issues.  It is not clear when they appear exactly
but the issues suggest our current approach may work in our current setting.
Because we have not seen any extra errors during testing we'll accept this as
being as good as it gets.  We switched because nodemon has some open issues and
we could not get it to reliably forward signals necessary to clean up state
during development.

Author: erikap

Dockerfile

@@ -3,6 +3,7 @@ FROM node:20-bookworm
 LABEL maintainer="[email protected]"
 
 RUN apt-get update && apt-get -y upgrade && apt-get -y install git openssh-client rsync jq
+RUN cd /tmp/ && wget https://github.com/watchexec/watchexec/releases/download/v2.3.2/watchexec-2.3.2-x86_64-unknown-linux-gnu.deb && dpkg -i watchexec-2.3.2-x86_64-unknown-linux-gnu.deb
 
 ENV MU_SPARQL_ENDPOINT='http://database:8890/sparql'
 ENV MU_APPLICATION_GRAPH='http://mu.semte.ch/application'

boot.sh

@@ -4,12 +4,17 @@ then
     # Run live-reload development
     echo "" > /tmp/service-status.lock
     echo "" > /tmp/service-restart
-    exec /usr/src/app/node_modules/.bin/nodemon \
-         --watch /app \
-         --watch /config \
-         --watch /tmp/service-restart \
-         --ext js,coffee,ts,mjs,cjs,json \
-         --exec /usr/src/app/run-development.sh
+
+    exec watchexec \
+         --project-origin="/" \
+         --watch="/app" \
+         --watch="/config" \
+         --exts="js,coffee,ts,mjs,cjs,json" \
+         --stop-timeout="60s" \
+         --stop-signal="SIGUSR2" \
+         --shell=none --no-process-group \
+         --restart \
+         /usr/src/app/run-development.sh
 elif [ "$NODE_ENV" == "production" ]
 then
     exec /usr/src/app/run-production.sh

helpers/mu/index.js

@@ -1,4 +1,4 @@
-import { app, errorHandler, setExitHandler, getExitHandler } from './server.js';
+import { app, errorHandler, beforeExit, exitHandler, setExitHandler } from './server.js';
 import sparql from './sparql.js';
 import { v1 as uuidV1 } from 'uuid';
 
@@ -22,7 +22,8 @@ const mu = {
   sparqlEscapeBool: sparql.sparqlEscapeBool,
   uuid,
   errorHandler,
-  getExitHandler,
+  beforeExit,
+  exitHandler,
   setExitHandler
 };
 
@@ -56,8 +57,9 @@ export {
   sparqlEscapeBool,
   uuid,
   errorHandler,
-  setExitHandler,
-  getExitHandler
+  beforeExit,
+  exitHandler,
+  setExitHandler
 };
 
 export default mu;

helpers/mu/server.js

@@ -34,18 +34,34 @@ const errorHandler = function(err, req, res, next) {
   });
 };
 
+/** @type { (() => Promise<void>)[] } */
+const beforeExitCallbacks = [];
+
+/**
+ * Define an async callback function to run on server shutdown
+ * @param { () => Promise<void> } callback Callback function to execute
+ */
+function beforeExit(callback) {
+  beforeExitCallbacks.push(callback);
+}
+
 // managing server cleanup
-let exitHandler = function(server) {
+let exitHandler = async function(server) {
   console.debug("Shutting down server");
-  server.close( () => {
-    console.debug("Shut down complete");
+  if (beforeExitCallbacks.length) {
+    console.debug("Executing before exit callbacks");
+    for (let callback of beforeExitCallbacks) {
+      await callback(server);
+    }
+  }
+  await new Promise((acc) => {
+    server.close( () => {
+      console.debug("Shut down complete");
+      acc();
+    });
   });
 };
 
-const getExitHandler = function() {
-  return exitHandler;
-}
-
 /**
  * Sets a new handler for shutting down the server.
  *
@@ -61,6 +77,7 @@ export default app;
 export {
   app,
   errorHandler,
+  beforeExit,
   setExitHandler,
-  getExitHandler
+  exitHandler
 }

package.json

@@ -23,7 +23,6 @@
     "env-var": "^7.0.0",
     "express": "^4.17.1",
     "express-http-context": "~1.2.4",
-    "nodemon": "^3.1.10",
     "sparql-client-2": "https://github.com/erikap/node-sparql-client.git",
     "typescript": "^4.6.2",
     "uuid": "^9.0.0"

run-development.sh

@@ -1,13 +1,16 @@
 #!/bin/bash
-sleep 0.1
 SERVICE_STATUS=$(cat /tmp/service-status.lock)
 if [ "$SERVICE_STATUS" == "exit_after_compile" ]
 then
-  echo "More changed files detected during compilation. Continuing compilation and scheduling another compile run."
-  exit 0
+    echo "More changed files detected during compilation. Continuing compilation and scheduling another compile run."
+    exit 0
 fi
 
-trap '{ SERVICE_STATUS=$(cat /tmp/service-status.lock); if [ $SERVICE_STATUS == "compiling" ]; then echo "exit_after_compile" > /tmp/service-status.lock; elif [ $SERVICE_STATUS == "running" ]; then exit 0; fi }' SIGUSR2 > /dev/null 2>&1
+# watchexec may send SIGUSR2 during compilation when more files have changed.
+# This requires to finish the compilation so no files are lingering about, but
+# stop the script before starting node.  watchexec will then start the script
+# again.
+trap '{ SERVICE_STATUS=$(cat /tmp/service-status.lock); if [ $SERVICE_STATUS == "compiling" ]; then echo "exit_after_compile" > /tmp/service-status.lock; fi }' SIGUSR2 > /dev/null 2>&1
 
 echo "compiling" > /tmp/service-status.lock
 
@@ -25,7 +28,6 @@ source ./helpers.sh
 cd /usr/src/app/
 
 
-
 ######################
 # Install dependencies
 ######################
@@ -43,9 +45,9 @@ fi
 cmp -s /app/package.json /tmp/last-build-service-package.json
 if [ "$?" == "1" ] && [ $IS_FIRST_RUN == false ]
 then
-  PACKAGE_JSON_CHANGED=true
+    PACKAGE_JSON_CHANGED=true
 else
-  PACKAGE_JSON_CHANGED=false
+    PACKAGE_JSON_CHANGED=false
 fi
 
 if [ -f /app/package.json ]
@@ -78,23 +80,24 @@ fi
 # Determine npm command and install dependencies
 if [ -f /app/package.json ]
 then
-  if $IS_FIRST_RUN
-  then
-    if $HAS_PACKAGE_LOCK
+    if $IS_FIRST_RUN
+    then
+        if $HAS_PACKAGE_LOCK
+        then
+            npm_install_command=ci
+        else
+            npm_install_command=install
+        fi
+    elif $PACKAGE_JSON_CHANGED
     then
-      npm_install_command=ci
-    else
-      npm_install_command=install
+        npm_install_command=install
     fi
-  elif $PACKAGE_JSON_CHANGED
-  then
-    npm_install_command=install
-  fi
 fi
 ./npm-install-dependencies.sh development $npm_install_command
 ./validate-package-json.sh
 touch /tmp/dependencies-installed-once-for-dev
 
+
 ###############
 # Transpilation
 ###############
@@ -103,19 +106,30 @@ touch /tmp/dependencies-installed-once-for-dev
 
 cp /usr/src/app/helpers/mu/package.json /usr/src/dist/node_modules/mu/
 
+
 ##############
 # Start server
 ##############
 SERVICE_STATUS=$(cat /tmp/service-status.lock)
 if [ "$SERVICE_STATUS" == "exit_after_compile" ]
 then
-  echo "" > /tmp/service-status.lock
-  touch /tmp/service-restart
-  exit 0
+    echo "" > /tmp/service-status.lock
+    exit 0
 else
-  echo "running" > /tmp/service-status.lock
-  cd /usr/src/dist/
-  node \
-    --inspect="0.0.0.0:9229" \
-    ./start-server.js
+    echo "running" > /tmp/service-status.lock
+    cd /usr/src/dist/
+    node \
+        --inspect="0.0.0.0:9229" \
+        ./start-server.js &
+    NODE_PID=$!
+    trap 'kill -s SIGUSR2 $NODE_PID' SIGUSR2 # SIGINT and SIGTERM are not necessary here now
+
+    # TODO: Is this ps + while verification step is still necessary?  The
+    # process appeared not to fully exit after `wait`.
+    while ps -p $NODE_PID > /dev/null
+    do
+        wait $NODE_PID
+    done
+    echo "" > /tmp/service-status.lock
+    exit 0
 fi

start-server.js

@@ -1,4 +1,4 @@
-import { app, getExitHandler } from 'mu';
+import { app, exitHandler } from 'mu';
 import './app.js'; // load the user's app
 
 var port = process.env.PORT || '80';
@@ -10,9 +10,15 @@ const server = app.listen(port, hostname, function () {
 });
 
 // faster stopping
-process.on('SIGTERM', () => {
-  getExitHandler()(server)
+process.on('SIGTERM', async () => {
+  await exitHandler(server);
+  process.exit(0);
 });
-process.on('SIGINT', () => {
-  getExitHandler()(server);
+process.on('SIGINT', async () => {
+  await exitHandler(server);
+  process.exit(0);
+});
+process.on('SIGUSR2', async () => {
+  await exitHandler(server);
+  process.exit(0);
 });