Add health monitoring script and enhance NGINX configuration for upstream checks

Author: Robbie1977

Dockerfile

@@ -5,9 +5,11 @@ LABEL description="NGINX caching proxy for Owlery"
 
 ARG NGINX_CONF=nginx.conf.template
 COPY $NGINX_CONF /etc/nginx/nginx.conf.template
+COPY health-monitor.sh /usr/local/bin/health-monitor.sh
 
-RUN mkdir -p /var/cache/nginx && chown -R nginx:nginx /var/cache/nginx
+RUN mkdir -p /var/cache/nginx && chown -R nginx:nginx /var/cache/nginx && \
+    chmod +x /usr/local/bin/health-monitor.sh
 
 EXPOSE 80
 
-CMD ["/bin/sh", "-c", "UPSTREAM_SERVER=${UPSTREAM_SERVER:-owl.virtualflybrain.org:80} CACHE_MAX_SIZE=${CACHE_MAX_SIZE:-20g} envsubst '${UPSTREAM_SERVER} ${CACHE_MAX_SIZE}' < /etc/nginx/nginx.conf.template > /etc/nginx/nginx.conf && nginx -g 'daemon off;'"]
+CMD ["/bin/sh", "-c", "UPSTREAM_SERVER=${UPSTREAM_SERVER:-owl.virtualflybrain.org:80} CACHE_MAX_SIZE=${CACHE_MAX_SIZE:-20g} DNS_RESOLVER=${DNS_RESOLVER:-8.8.8.8 1.1.1.1} envsubst '${UPSTREAM_SERVER} ${CACHE_MAX_SIZE} ${DNS_RESOLVER}' < /etc/nginx/nginx.conf.template > /etc/nginx/nginx.conf && /usr/local/bin/health-monitor.sh & nginx -g 'daemon off;'"]

README.md

@@ -30,22 +30,29 @@ services:
       - "80:80"
     environment:
       - UPSTREAM_SERVER=owl:8080  # For production with owl service
-      - CACHE_MAX_SIZE=1000g      # 1TB cache size for high-traffic deployments
+      - CACHE_MAX_SIZE=1t         # 1TB cache size for high-traffic deployments
+      - DNS_RESOLVER=169.254.169.250  # Rancher internal DNS (check /etc/resolv.conf)
 ```
 
 ### Health Check
 
 ```bash
 curl http://localhost/health
-# Returns: OK
+# Returns: upstream response or "UPSTREAM_UNAVAILABLE" (503) if upstream is down
+# Includes X-Upstream-Status header showing actual upstream response code
 ```
 
+The health endpoint now proxies to the upstream server to verify connectivity. If the upstream is unavailable, it returns 503 with "UPSTREAM_UNAVAILABLE".
+
+**Health Monitoring**: A background process logs warnings every 5 minutes if the upstream server becomes unreachable, but the container continues running to serve cached content.
+
 ## Configuration
 
 ### Environment Variables
 
 - `UPSTREAM_SERVER`: Backend server URL (default: `owl.virtualflybrain.org:80`)
-- `CACHE_MAX_SIZE`: Maximum cache size on disk (default: `20g`, accepts NGINX size units: `k`/`K` kilobytes, `m`/`M` megabytes, `g`/`G` gigabytes)
+- `CACHE_MAX_SIZE`: Maximum cache size on disk (default: `20g`, accepts NGINX size units like `1t` for 1TB)
+- `DNS_RESOLVER`: DNS resolver servers (default: `8.8.8.8 1.1.1.1`, space-separated list). Check `cat /etc/resolv.conf` in your container to find the correct value for your environment.
 
 ### Cache Headers
 
@@ -69,7 +76,8 @@ The proxy adds helpful headers to responses:
 - **Base image**: nginx:1.26-alpine
 - **Cache storage**: `/var/cache/nginx/owlery` with 1:2 directory levels
 - **Cache zone**: 100MB in-memory metadata zone
-- **Max cache size**: 20GB on disk (configurable via `CACHE_MAX_SIZE` environment variable, supports k/K, m/M, g/G units)
+- **Max cache size**: 20GB on disk (configurable via `CACHE_MAX_SIZE` environment variable)
+- **Health monitoring**: Background process checks upstream connectivity every 5 minutes and logs warnings
 
 ### Caching Behavior
 
@@ -83,9 +91,10 @@ The proxy adds helpful headers to responses:
 ### Networking
 
 - **Listen port**: 80
+- **DNS resolver**: Configurable via `DNS_RESOLVER` (default: Google Public DNS with 30s TTL for fast upstream IP updates). Check `cat /etc/resolv.conf` in your container for the correct value.
 - **Host-agnostic**: Ignores Host header for routing
 - **Connection pooling**: 16 keep-alive connections to backend
-- **Timeouts**: 90s connect/read/send
+- **Timeouts**: 90s connect/read/send, 3s for health checks
 
 ## Build and Deployment
 

health-monitor.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# Health monitoring script for upstream server
+# Logs warnings but doesn't exit container
+
+UPSTREAM_HOST=$(echo $UPSTREAM_SERVER | cut -d: -f1)
+UPSTREAM_PORT=$(echo $UPSTREAM_SERVER | cut -d: -f2)
+
+echo "Monitoring upstream server: $UPSTREAM_HOST:$UPSTREAM_PORT"
+
+while true; do
+    if nc -z -w3 $UPSTREAM_HOST $UPSTREAM_PORT 2>/dev/null; then
+        echo "$(date): Upstream server is healthy"
+    else
+        echo "$(date): WARNING - Upstream server $UPSTREAM_HOST:$UPSTREAM_PORT is unreachable"
+    fi
+
+    sleep 300  # Check every 5 minutes
+done

nginx.conf.template

@@ -5,6 +5,9 @@ events {
 }
 
 http {
+    resolver ${DNS_RESOLVER} valid=30s;
+    resolver_timeout 5s;
+
     upstream owlery {
         server ${UPSTREAM_SERVER};
         keepalive 16;
@@ -38,8 +41,13 @@ http {
 
         location /health {
             access_log off;
-            return 200 "OK\n";
+            proxy_pass http://owlery/;
+            proxy_connect_timeout 3s;
+            proxy_read_timeout 3s;
+            proxy_intercept_errors on;
+            error_page 500 502 503 504 =503 "UPSTREAM_UNAVAILABLE\n";
             add_header Content-Type text/plain;
+            add_header X-Upstream-Status $upstream_status;
         }
 
         location / {