Merge branch 'debug'

Author: rpardini

Dockerfile

@@ -1,10 +1,18 @@
 # We start from my nginx fork which includes the proxy-connect module from tEngine
 # Source is available at https://github.com/rpardini/nginx-proxy-connect-stable-alpine
-# Its equivalent to nginx:stable-alpine 1.14.0, with alpine 3.7
-FROM rpardini/nginx-proxy-connect-stable-alpine:latest
+FROM rpardini/nginx-proxy-connect-stable-alpine:nginx-1.14.0-alpine-3.8
 
 # Add openssl, bash and ca-certificates, then clean apk cache -- yeah complain all you want.
-RUN apk add --update openssl bash ca-certificates && rm -rf /var/cache/apk/*
+# Also added deps for mitmproxy.
+RUN apk add --update openssl bash ca-certificates su-exec git g++ libffi libffi-dev libstdc++ openssl openssl-dev python3 python3-dev
+RUN LDFLAGS=-L/lib pip3 install mitmproxy
+RUN apk del --purge git g++ libffi-dev openssl-dev python3-dev && rm -rf /var/cache/apk/* && rm -rf ~/.cache/pip
+
+# Required for mitmproxy
+ENV LANG=en_US.UTF-8
+
+# Check the installed mitmproxy version
+RUN mitmproxy --version
 
 # Create the cache directory and CA directory
 RUN mkdir -p /docker_mirror_cache /ca
@@ -27,13 +35,20 @@ RUN chmod +x /create_ca_cert.sh /entrypoint.sh
 # Clients should only use 3128, not anything else.
 EXPOSE 3128
 
+# In debug mode, 8081 exposes the mitmweb interface.
+EXPOSE 8081
+
 ## Default envs.
 # A space delimited list of registries we should proxy and cache; this is in addition to the central DockerHub.
 ENV REGISTRIES="k8s.gcr.io gcr.io quay.io"
 # A space delimited list of registry:user:password to inject authentication for
 ENV AUTH_REGISTRIES="some.authenticated.registry:oneuser:onepassword another.registry:user:password"
 # Should we verify upstream's certificates? Default to true.
 ENV VERIFY_SSL="true"
+# Enable debugging mode; this inserts mitmproxy/mitmweb between the CONNECT proxy and the caching layer
+ENV DEBUG="true"
+# Enable nginx debugging mode; this uses nginx-debug binary and enabled debug logging, which is VERY verbose so separate setting
+ENV DEBUG_NGINX="false"
 
-# Did you want a shell? Sorry. This only does one job; use exec /bin/bash if you wanna inspect stuff
+# Did you want a shell? Sorry, the entrypoint never returns, because it runs nginx itself. Use 'docker exec' if you need to mess around internally.
 ENTRYPOINT ["/entrypoint.sh"]

create_ca_cert.sh

@@ -34,7 +34,7 @@ if [ -f "$CA_CRT_FILE" ] ; then
 else
     logInfo "No CA was found. Generating one."
     logInfo "*** Please *** make sure to mount /ca as a volume -- if not, everytime this container starts, it will regenerate the CA and nothing will work."
-    
+
     openssl genrsa -des3 -passout pass:foobar -out ${CA_KEY_FILE} 4096
 
     logInfo "generate CA cert with key and self sign it: ${CAID}"
@@ -52,7 +52,7 @@ EOF
 
     [[ ${DEBUG} -gt 0 ]] && logInfo "show the CA cert details"
     [[ ${DEBUG} -gt 0 ]] && openssl x509 -noout -text -in ${CA_CRT_FILE}
-    
+
     echo 01 > ${CA_SRL_FILE}
 
 fi
@@ -116,3 +116,6 @@ openssl x509 -req -days 365 -in web.csr -CA ia.crt -CAkey ia.key -out web.crt -p
 
 logInfo "Concatenating fullchain.pem..."
 cat web.crt ia.crt ${CA_CRT_FILE}  > fullchain.pem
+
+logInfo "Concatenating fullchain_with_key.pem"
+cat fullchain.pem web.key > fullchain_with_key.pem

entrypoint.sh

@@ -36,7 +36,7 @@ for ONEREGISTRYIN in ${AUTH_REGISTRIES}; do
 done
 
 echo "" > /etc/nginx/docker.verify.ssl.conf
-if [ "a$VERIFY_SSL" == "atrue" ]; then
+if [[ "a${VERIFY_SSL}" == "atrue" ]]; then
     cat << EOD > /etc/nginx/docker.verify.ssl.conf
     # We actually wanna be secure and avoid mitm attacks.
     # Fitting, since this whole thing is a mitm...
@@ -46,11 +46,38 @@ if [ "a$VERIFY_SSL" == "atrue" ]; then
     proxy_ssl_verify_depth 2;
 EOD
     echo "Upstream SSL certificate verification enabled."
-fi 
+fi
 
+# create default config for the caching layer to listen on 443.
+echo "        listen 443 ssl default_server;" > /etc/nginx/caching.layer.listen
+echo "error_log  /var/log/nginx/error.log warn;" > /etc/nginx/error.log.debug.warn
+
+# normally use non-debug version of nginx
+NGINX_BIN="nginx"
+
+if [[ "a${DEBUG}" == "atrue" ]]; then
+  # in debug mode, change caching layer to listen on 444, so that mitmproxy can sit in the middle.
+  echo "        listen 444 ssl default_server;" > /etc/nginx/caching.layer.listen
+
+  echo "Starting in DEBUG MODE (mitmproxy)."
+  echo "Run mitmproxy with reverse pointing to the same certs..."
+  mitmweb --no-web-open-browser --web-iface 0.0.0.0 --web-port 8081 \
+          --set keep_host_header=true --set ssl_insecure=true \
+          --mode reverse:https://127.0.0.1:444 --listen-host 0.0.0.0 \
+          --listen-port 443 --certs /certs/fullchain_with_key.pem \
+          -w /ca/outfile &
+  echo "Access mitmweb via http://127.0.0.1:8081/ "
+fi
+
+if [[ "a${DEBUG_NGINX}" == "atrue" ]]; then
+  echo "Starting in DEBUG MODE (nginx)."
+  echo "error_log  /var/log/nginx/error.log debug;" > /etc/nginx/error.log.debug.warn
+  # use debug binary
+  NGINX_BIN="nginx-debug"
+fi
 
 echo "Testing nginx config..."
-nginx -t
+${NGINX_BIN} -t
 
 echo "Starting nginx! Have a nice day."
-nginx -g "daemon off;"
+${NGINX_BIN} -g "daemon off;"

nginx.conf

@@ -1,7 +1,9 @@
 user  nginx;
 worker_processes  auto;
 
-error_log  /var/log/nginx/error.log warn;
+# error log config comes from external file created by entrypoint, to toggle debug on/off.
+include /etc/nginx/error.log.debug.warn;
+
 pid        /var/run/nginx.pid;
 
 events {
@@ -13,7 +15,7 @@ http {
     include       /etc/nginx/mime.types;
     default_type  application/octet-stream;
 
-    # Use a debug-oriented logging format. 
+    # Use a debug-oriented logging format.
     log_format  debugging  '$remote_addr - $remote_user [$time_local] "$request" '
     '$status $body_bytes_sent '
     '"HOST: $host" "UPSTREAM: $upstream_addr" '
@@ -22,19 +24,28 @@ http {
     '"CONNECT-HOST: $connect_host" "CONNECT-PORT: $connect_port" "CONNECT-ADDR: $connect_addr" '
     '"PROXY-HOST: $proxy_host" "UPSTREAM-REDIRECT: $upstream_http_location" "CACHE-STATUS: $upstream_cache_status" '
     '"AUTH: $http_authorization" ' ;
-    
+
+    log_format  debug_proxy  'CONNECTPROXY: $remote_addr - $remote_user [$time_local] "$request" '
+    '$status $body_bytes_sent '
+    '"HOST: $host" "UPSTREAM: $upstream_addr" '
+    '"UPSTREAM-STATUS: $upstream_status" '
+    '"SSL-PROTO: $ssl_protocol" '
+    '"CONNECT-HOST: $connect_host" "CONNECT-PORT: $connect_port" "CONNECT-ADDR: $connect_addr" "INTERCEPTED: $interceptedHost" '
+    '"PROXY-HOST: $proxy_host" "UPSTREAM-REDIRECT: $upstream_http_location" "CACHE-STATUS: $upstream_cache_status" '
+    '"AUTH: $http_authorization" ' ;
+
     log_format  tweaked  '$upstream_cache_status [$time_local] "$uri" '
     '$status $body_bytes_sent '
     '"HOST:$host" '
     '"PROXY-HOST:$proxy_host" "UPSTREAM:$upstream_addr" ';
-    
+
     keepalive_timeout  300;
     gzip  off;
 
     # The cache directory. This can get huge. Better to use a Docker volume pointing here!
     # Set to 32gb which should be enough
     proxy_cache_path /docker_mirror_cache levels=1:2 max_size=32g inactive=60d keys_zone=cache:10m use_temp_path=off;
-    
+
     # Just in case you want to rewrite some hosts. Default maps directly.
     map $host $targetHost {
         hostnames;
@@ -48,22 +59,27 @@ http {
         include /etc/nginx/docker.auth.map;
         default "";
     }
-    
+
+    # @TODO: actually for auth.docker.io, if we want to support multiple authentications, we'll need to decide
+    # @TODO: based not only on the hostname, but also URI (/token) and query string (?scope)
+    # @TODO: I wonder if this would help gcr.io and quay.io with authentication also....
+
+    map $dockerAuth $finalAuth {
+        "" "$http_authorization";    # if empty, keep the original passed-in from the docker client.
+        default "Basic $dockerAuth"; # if not empty, add the Basic preamble to the auth
+    }
+
+
     # Map to decide which hosts get directed to the caching portion.
     # This is automatically generated from the list of cached registries, plus a few fixed hosts
     # By default, we don't intercept, allowing free flow of non-registry traffic
     map $connect_host $interceptedHost {
         hostnames;
         include /etc/nginx/docker.intercept.map;
-        default "$connect_host:443";
-    }
-    
-    map $dockerAuth $finalAuth { 
-        "" "$http_authorization";    # if empty, keep the original passed-in from the client
-        default "Basic $dockerAuth"; # if not empty, add the Basic preamble to the auth
+        default "$connect_addr"; # $connect_addr is 'IP address and port of the remote host, e.g. "192.168.1.5:12345". IP address is resolved from host name of CONNECT request line.'
     }
-    
-    
+
+
     # These maps parse the original Host and URI from a /forcecache redirect.
     map $request_uri $realHost {
         ~/forcecacheinsecure/([^:/]+)/originalwas(/.+) $1;
@@ -76,43 +92,46 @@ http {
         ~/forcecachesecure/([^:/]+)/originalwas(/.+) $2;
         default "DID_NOT_MATCH_PATH";
     }
-    
-    
+
+
     # The proxy director layer, listens on 3128
     server {
         listen 3128;
         server_name _;
-        
+
         # dont log the CONNECT proxy.
+        #access_log /var/log/nginx/access.log debug_proxy;
         access_log off;
-        
+
         proxy_connect;
         proxy_connect_address $interceptedHost;
         proxy_max_temp_file_size 0;
-        
+
         # We need to resolve the real names of our proxied servers.
         resolver 8.8.8.8 4.2.2.2 ipv6=off; # Avoid ipv6 addresses for now
 
         # forward proxy for non-CONNECT request
         location / {
-            return 403 "The docker caching proxy is working!";
+            add_header "Content-type" "text/plain" always;
+            return 200 "docker-registry-proxy: The docker caching proxy is working!";
         }
-        
+
         location /ca.crt {
-             alias /ca/ca.crt; 
+             alias /ca/ca.crt;
         }
 
         # @TODO: add a dynamic root path that generates instructions for usage on docker clients
     }
-    
+
 
     # The caching layer
     server {
         # Listen on both 80 and 443, for all hostnames.
+        # actually could be 443 or 444, depending on debug. this is now generated by the entrypoint.
         listen 80 default_server;
-        listen 443 ssl default_server;
+        include /etc/nginx/caching.layer.listen;
         server_name _;
-        
+
         # Do some tweaked logging.
         access_log  /var/log/nginx/access.log  tweaked;
 
@@ -136,17 +155,17 @@ http {
         if ($request_method = DELETE) {
             return 405  "DELETE method is not allowed";
         }
-        
+
         proxy_read_timeout 900;
-        
-        # Use cache locking, with a huge timeout, so that multiple Docker clients asking for the same blob at the same time 
+
+        # Use cache locking, with a huge timeout, so that multiple Docker clients asking for the same blob at the same time
         # will wait for the first to finish instead of doing multiple upstream requests.
         proxy_cache_lock on;
-        proxy_cache_lock_timeout 120s;
+        proxy_cache_lock_timeout 880s;
 
         # Cache all 200, 301, 302, and 307 (emitted by private registries) for 60 days.
-        proxy_cache_valid 200 301 302 307 60d; 
-        
+        proxy_cache_valid 200 301 302 307 60d;
+
         # Some extra settings to maximize cache hits and efficiency
         proxy_force_ranges on;
         proxy_ignore_client_abort on;
@@ -155,50 +174,25 @@ http {
         # Hide/ignore headers from caching. S3 especially likes to send Expires headers in the past in some situations.
         proxy_hide_header      Set-Cookie;
         proxy_ignore_headers   X-Accel-Expires Expires Cache-Control Set-Cookie;
-        
+
         # Add the authentication info, if the map matched the target domain.
         proxy_set_header Authorization $finalAuth;
-        
+
         # This comes from a include file generated by the entrypoint.
         include /etc/nginx/docker.verify.ssl.conf;
-        
-        # Some debugging info
-        # add_header X-Docker-Caching-Proxy-Real-Host $realHost;
-        # add_header X-Docker-Caching-Proxy-Real-Path $realPath;
-        # add_header X-Docker-Caching-Proxy-Auth $finalAuth;
 
         # Block API v1. We dont know how to handle these.
         # Docker-client should start with v2 and fallback to v1 if something fails, for example, if authentication failed to a protected v2 resource.
         location /v1 {
-            return 405 "API v1 is invalid. Either the image does not exist upstream, or you need auth to get a v2 endpoint working against $host";
-        }
-
-        # don't cache mutable entity /v2/<name>/manifests/<reference> (unless the reference is a digest)
-        location ~ ^/v2/[^\/]+/manifests/(?![A-Fa-f0-9_+.-]+:) {
-            proxy_pass https://$targetHost;
-        }
-
-        # don't cache mutable entity /v2/<name>/tags/list
-        location ~ ^/v2/[^\/]+/tags/list {
-            proxy_pass https://$targetHost;
+            return 405 "docker-registry-proxy: docker is trying to use v1 API. Either the image does not exist upstream, or you need to configure docker-registry-proxy to authenticate against $host";
         }
 
-        # don't cache mutable entity /v2/_catalog
-        location ~ ^/v2/_catalog$ {
-            proxy_pass https://$targetHost;
-        }
-        
-        # force cache of the first hit which is always /v2/ - even for 401 unauthorized.
-        location = /v2/ {
-            proxy_pass https://$targetHost;
-            proxy_cache cache;
-            proxy_cache_valid 200 301 302 307 401 60d;
-        }
 
-        # cache everything else
-        location / {
+        # for the /v2/..../blobs/.... URIs, do cache, and treat redirects.
+        location ~ ^/v2/(.*)/blobs/ {
             proxy_pass https://$targetHost;
             proxy_cache cache;
+            add_header X-Docker-Caching-Proxy-Debug-Cache "yes:blobs";
 
             # Handling of redirects.
             # Many registries (eg, quay.io, or k8s.gcr.io) emit a Location redirect
@@ -210,24 +204,38 @@ http {
             proxy_redirect ~^http://([^:/]+)(/.+)$ http://docker.caching.proxy.internal/forcecacheinsecure/$1/originalwas$2;
         }
 
+
         # handling for the redirect case explained above, with https.
         # The $realHost and $realPath variables come from a map defined at the top of this file.
         location /forcecachesecure {
             proxy_pass https://$realHost$realPath;
             proxy_cache cache;
-            
+
             # Change the cache key, so that we can cache signed S3 requests and such. Only host and path are considered.
             proxy_cache_key $proxy_host$uri;
+
+            add_header X-Docker-Caching-Proxy-Debug-Cache "yes:forcecachesecure";
+
         }
 
         # handling for the redirect case explained above, with http.
         # The $realHost and $realPath variables come from a map defined at the top of this file.
         location /forcecacheinsecure {
             proxy_pass http://$realHost$realPath;
             proxy_cache cache;
-            
+
             # Change the cache key, so that we can cache signed S3 requests and such. Only host and path are considered.
             proxy_cache_key $proxy_host$uri;
+
+            add_header X-Docker-Caching-Proxy-Debug-Cache "yes:forcecacheinsecure";
         }
+
+        # by default, dont cache anything.
+        location / {
+            proxy_pass https://$targetHost;
+            proxy_cache off;
+            add_header X-Docker-Caching-Proxy-Debug-Cache "no:default";
+        }
+
     }
 }