add mitmproxy/nginx-debug inspection capabilities

rpardini · rpardini · commit 8ff06e3dec8a · 2018-11-04T11:23:52.000+01:00
- avoid some caching for non-blob urls
diff --git a/Dockerfile b/Dockerfile
@@ -1,10 +1,18 @@
 # We start from my nginx fork which includes the proxy-connect module from tEngine
 # Source is available at https://github.com/rpardini/nginx-proxy-connect-stable-alpine
-# Its equivalent to nginx:stable-alpine 1.14.0, with alpine 3.7
-FROM rpardini/nginx-proxy-connect-stable-alpine:latest
+FROM rpardini/nginx-proxy-connect-stable-alpine:nginx-1.14.0-alpine-3.8
 
 # Add openssl, bash and ca-certificates, then clean apk cache -- yeah complain all you want.
-RUN apk add --update openssl bash ca-certificates && rm -rf /var/cache/apk/*
+# Also added deps for mitmproxy.
+RUN apk add --update openssl bash ca-certificates su-exec git g++ libffi libffi-dev libstdc++ openssl openssl-dev python3 python3-dev
+RUN LDFLAGS=-L/lib pip3 install mitmproxy
+RUN apk del --purge git g++ libffi-dev openssl-dev python3-dev && rm -rf /var/cache/apk/* && rm -rf ~/.cache/pip
+
+# Required for mitmproxy
+ENV LANG=en_US.UTF-8
+
+# Check the installed mitmproxy version
+RUN mitmproxy --version
 
 # Create the cache directory and CA directory
 RUN mkdir -p /docker_mirror_cache /ca
@@ -27,13 +35,18 @@ RUN chmod +x /create_ca_cert.sh /entrypoint.sh
 # Clients should only use 3128, not anything else.
 EXPOSE 3128
 
+# In debug mode, 8081 exposes the mitmweb interface.
+EXPOSE 8081
+
 ## Default envs.
 # A space delimited list of registries we should proxy and cache; this is in addition to the central DockerHub.
 ENV REGISTRIES="k8s.gcr.io gcr.io quay.io"
 # A space delimited list of registry:user:password to inject authentication for
 ENV AUTH_REGISTRIES="some.authenticated.registry:oneuser:onepassword another.registry:user:password"
 # Should we verify upstream's certificates? Default to true.
 ENV VERIFY_SSL="true"
+# Enable debugging mode; this inserts mitmproxy/mitmweb between the CONNECT proxy and the caching layer
+ENV DEBUG="true"
 
 # Did you want a shell? Sorry. This only does one job; use exec /bin/bash if you wanna inspect stuff
 ENTRYPOINT ["/entrypoint.sh"]
diff --git a/create_ca_cert.sh b/create_ca_cert.sh
@@ -34,7 +34,7 @@ if [ -f "$CA_CRT_FILE" ] ; then
 else
     logInfo "No CA was found. Generating one."
     logInfo "*** Please *** make sure to mount /ca as a volume -- if not, everytime this container starts, it will regenerate the CA and nothing will work."
-    
+
     openssl genrsa -des3 -passout pass:foobar -out ${CA_KEY_FILE} 4096
 
     logInfo "generate CA cert with key and self sign it: ${CAID}"
@@ -52,7 +52,7 @@ EOF
 
     [[ ${DEBUG} -gt 0 ]] && logInfo "show the CA cert details"
     [[ ${DEBUG} -gt 0 ]] && openssl x509 -noout -text -in ${CA_CRT_FILE}
-    
+
     echo 01 > ${CA_SRL_FILE}
 
 fi
@@ -116,3 +116,6 @@ openssl x509 -req -days 365 -in web.csr -CA ia.crt -CAkey ia.key -out web.crt -p
 
 logInfo "Concatenating fullchain.pem..."
 cat web.crt ia.crt ${CA_CRT_FILE}  > fullchain.pem
+
+logInfo "Concatenating fullchain_with_key.pem"
+cat fullchain.pem web.key > fullchain_with_key.pem
diff --git a/entrypoint.sh b/entrypoint.sh
@@ -36,7 +36,7 @@ for ONEREGISTRYIN in ${AUTH_REGISTRIES}; do
 done
 
 echo "" > /etc/nginx/docker.verify.ssl.conf
-if [ "a$VERIFY_SSL" == "atrue" ]; then
+if [[ "a${VERIFY_SSL}" == "atrue" ]]; then
     cat << EOD > /etc/nginx/docker.verify.ssl.conf
     # We actually wanna be secure and avoid mitm attacks.
     # Fitting, since this whole thing is a mitm...
@@ -46,11 +46,36 @@ if [ "a$VERIFY_SSL" == "atrue" ]; then
     proxy_ssl_verify_depth 2;
 EOD
     echo "Upstream SSL certificate verification enabled."
-fi 
+fi
+
+# create default config for the caching layer to listen on 443.
+echo "        listen 443 ssl default_server;" > /etc/nginx/caching.layer.listen
+echo "error_log  /var/log/nginx/error.log warn;" > /etc/nginx/error.log.debug.warn
+
+# normally use non-debug version of nginx
+NGINX_BIN="nginx"
+
+if [[ "a${DEBUG}" == "atrue" ]]; then
+  # in debug mode, change caching layer to listen on 444, so that mitmproxy can sit in the middle.
+  echo "        listen 444 ssl default_server;" > /etc/nginx/caching.layer.listen
+  echo "error_log  /var/log/nginx/error.log debug;" > /etc/nginx/error.log.debug.warn
+
+  # use debug binary
+  NGINX_BIN="nginx-debug"
+
+  echo "Starting in DEBUG MODE."
+  echo "Run mitmproxy with reverse pointing to the same certs..."
+  mitmweb --no-web-open-browser --web-iface 0.0.0.0 --web-port 8081 \
+          --set keep_host_header=true --set ssl_insecure=true \
+          --mode reverse:https://127.0.0.1:444 --listen-host 0.0.0.0 \
+          --listen-port 443 --certs /certs/fullchain_with_key.pem \
+          -w /ca/outfile &
+  echo "Access mitmweb via http://127.0.0.1:8081/ "
+fi
 
 
 echo "Testing nginx config..."
-nginx -t
+${NGINX_BIN} -t
 
 echo "Starting nginx! Have a nice day."
-nginx -g "daemon off;"
+${NGINX_BIN} -g "daemon off;"
diff --git a/nginx.conf b/nginx.conf
@@ -1,7 +1,9 @@
 user  nginx;
 worker_processes  auto;
 
-error_log  /var/log/nginx/error.log warn;
+# error log config comes from external file created by entrypoint, to toggle debug on/off.
+include /etc/nginx/error.log.debug.warn;
+
 pid        /var/run/nginx.pid;
 
 events {
@@ -13,28 +15,28 @@ http {
     include       /etc/nginx/mime.types;
     default_type  application/octet-stream;
 
-    # Use a debug-oriented logging format. 
+    # Use a debug-oriented logging format.
     log_format  debugging  '$remote_addr - $remote_user [$time_local] "$request" '
     '$status $body_bytes_sent '
     '"HOST: $host" "UPSTREAM: $upstream_addr" '
     '"UPSTREAM-STATUS: $upstream_status" '
     '"SSL-PROTO: $ssl_protocol" '
-    '"CONNECT-HOST: $connect_host" "CONNECT-PORT: $connect_port" "CONNECT-ADDR: $connect_addr" '
+    '"CONNECT-HOST: $connect_host" "CONNECT-PORT: $connect_port" "CONNECT-ADDR: $connect_addr" "INTERCEPTED: $interceptedHost" '
     '"PROXY-HOST: $proxy_host" "UPSTREAM-REDIRECT: $upstream_http_location" "CACHE-STATUS: $upstream_cache_status" '
     '"AUTH: $http_authorization" ' ;
-    
+
     log_format  tweaked  '$upstream_cache_status [$time_local] "$uri" '
     '$status $body_bytes_sent '
     '"HOST:$host" '
     '"PROXY-HOST:$proxy_host" "UPSTREAM:$upstream_addr" ';
-    
+
     keepalive_timeout  300;
     gzip  off;
 
     # The cache directory. This can get huge. Better to use a Docker volume pointing here!
     # Set to 32gb which should be enough
     proxy_cache_path /docker_mirror_cache levels=1:2 max_size=32g inactive=60d keys_zone=cache:10m use_temp_path=off;
-    
+
     # Just in case you want to rewrite some hosts. Default maps directly.
     map $host $targetHost {
         hostnames;
@@ -48,7 +50,7 @@ http {
         include /etc/nginx/docker.auth.map;
         default "";
     }
-    
+
     # Map to decide which hosts get directed to the caching portion.
     # This is automatically generated from the list of cached registries, plus a few fixed hosts
     # By default, we don't intercept, allowing free flow of non-registry traffic
@@ -57,13 +59,13 @@ http {
         include /etc/nginx/docker.intercept.map;
         default "$connect_host:443";
     }
-    
-    map $dockerAuth $finalAuth { 
+
+    map $dockerAuth $finalAuth {
         "" "$http_authorization";    # if empty, keep the original passed-in from the client
         default "Basic $dockerAuth"; # if not empty, add the Basic preamble to the auth
     }
-    
-    
+
+
     # These maps parse the original Host and URI from a /forcecache redirect.
     map $request_uri $realHost {
         ~/forcecacheinsecure/([^:/]+)/originalwas(/.+) $1;
@@ -76,43 +78,44 @@ http {
         ~/forcecachesecure/([^:/]+)/originalwas(/.+) $2;
         default "DID_NOT_MATCH_PATH";
     }
-    
-    
+
+
     # The proxy director layer, listens on 3128
     server {
         listen 3128;
         server_name _;
-        
+
         # dont log the CONNECT proxy.
         access_log off;
-        
+
         proxy_connect;
         proxy_connect_address $interceptedHost;
         proxy_max_temp_file_size 0;
-        
+
         # We need to resolve the real names of our proxied servers.
         resolver 8.8.8.8 4.2.2.2 ipv6=off; # Avoid ipv6 addresses for now
 
         # forward proxy for non-CONNECT request
         location / {
             return 403 "The docker caching proxy is working!";
         }
-        
+
         location /ca.crt {
-             alias /ca/ca.crt; 
+             alias /ca/ca.crt;
         }
 
         # @TODO: add a dynamic root path that generates instructions for usage on docker clients
     }
-    
+
 
     # The caching layer
     server {
         # Listen on both 80 and 443, for all hostnames.
+        # actually could be 443 or 444, depending on debug. this is now generated by the entrypoint.
         listen 80 default_server;
-        listen 443 ssl default_server;
+        include /etc/nginx/caching.layer.listen;
         server_name _;
-        
+
         # Do some tweaked logging.
         access_log  /var/log/nginx/access.log  tweaked;
 
@@ -136,17 +139,17 @@ http {
         if ($request_method = DELETE) {
             return 405  "DELETE method is not allowed";
         }
-        
+
         proxy_read_timeout 900;
-        
-        # Use cache locking, with a huge timeout, so that multiple Docker clients asking for the same blob at the same time 
+
+        # Use cache locking, with a huge timeout, so that multiple Docker clients asking for the same blob at the same time
         # will wait for the first to finish instead of doing multiple upstream requests.
         proxy_cache_lock on;
         proxy_cache_lock_timeout 120s;
 
         # Cache all 200, 301, 302, and 307 (emitted by private registries) for 60 days.
-        proxy_cache_valid 200 301 302 307 60d; 
-        
+        proxy_cache_valid 200 301 302 307 60d;
+
         # Some extra settings to maximize cache hits and efficiency
         proxy_force_ranges on;
         proxy_ignore_client_abort on;
@@ -155,13 +158,13 @@ http {
         # Hide/ignore headers from caching. S3 especially likes to send Expires headers in the past in some situations.
         proxy_hide_header      Set-Cookie;
         proxy_ignore_headers   X-Accel-Expires Expires Cache-Control Set-Cookie;
-        
+
         # Add the authentication info, if the map matched the target domain.
         proxy_set_header Authorization $finalAuth;
-        
+
         # This comes from a include file generated by the entrypoint.
         include /etc/nginx/docker.verify.ssl.conf;
-        
+
         # Some debugging info
         # add_header X-Docker-Caching-Proxy-Real-Host $realHost;
         # add_header X-Docker-Caching-Proxy-Real-Path $realPath;
@@ -176,23 +179,35 @@ http {
         # don't cache mutable entity /v2/<name>/manifests/<reference> (unless the reference is a digest)
         location ~ ^/v2/[^\/]+/manifests/(?![A-Fa-f0-9_+.-]+:) {
             proxy_pass https://$targetHost;
+            add_header X-Docker-Caching-Proxy-Debug-Cache "no:manifests";
         }
 
         # don't cache mutable entity /v2/<name>/tags/list
         location ~ ^/v2/[^\/]+/tags/list {
             proxy_pass https://$targetHost;
+            proxy_cache off;
+            add_header X-Docker-Caching-Proxy-Debug-Cache "no:tagslist";
         }
 
         # don't cache mutable entity /v2/_catalog
         location ~ ^/v2/_catalog$ {
             proxy_pass https://$targetHost;
+            proxy_cache off;
+            add_header X-Docker-Caching-Proxy-Debug-Cache "no:catalog";
         }
-        
-        # force cache of the first hit which is always /v2/ - even for 401 unauthorized.
+
+        # dont cache the first hit which is always /v2/
         location = /v2/ {
             proxy_pass https://$targetHost;
-            proxy_cache cache;
-            proxy_cache_valid 200 301 302 307 401 60d;
+            proxy_cache off;
+            add_header X-Docker-Caching-Proxy-Debug-Cache "no:rootv2";
+        }
+
+        # dont cache /token (done against auth servers)
+        location = /token {
+            proxy_pass https://$targetHost;
+            proxy_cache off;
+            add_header X-Docker-Caching-Proxy-Debug-Cache "no:token";
         }
 
         # cache everything else
@@ -208,26 +223,33 @@ http {
             # We to it twice, one for http and another for https.
             proxy_redirect ~^https://([^:/]+)(/.+)$ https://docker.caching.proxy.internal/forcecachesecure/$1/originalwas$2;
             proxy_redirect ~^http://([^:/]+)(/.+)$ http://docker.caching.proxy.internal/forcecacheinsecure/$1/originalwas$2;
+
+            add_header X-Docker-Caching-Proxy-Debug-Cache "yes:everythingelse";
         }
 
         # handling for the redirect case explained above, with https.
         # The $realHost and $realPath variables come from a map defined at the top of this file.
         location /forcecachesecure {
             proxy_pass https://$realHost$realPath;
             proxy_cache cache;
-            
+
             # Change the cache key, so that we can cache signed S3 requests and such. Only host and path are considered.
             proxy_cache_key $proxy_host$uri;
+
+            add_header X-Docker-Caching-Proxy-Debug-Cache "yes:forcecachesecure";
+
         }
 
         # handling for the redirect case explained above, with http.
         # The $realHost and $realPath variables come from a map defined at the top of this file.
         location /forcecacheinsecure {
             proxy_pass http://$realHost$realPath;
             proxy_cache cache;
-            
+
             # Change the cache key, so that we can cache signed S3 requests and such. Only host and path are considered.
             proxy_cache_key $proxy_host$uri;
+
+            add_header X-Docker-Caching-Proxy-Debug-Cache "yes:forcecacheinsecure";
         }
     }
 }
