completely reworked caching, now cache by exception (/blobs/ only essentially)

- now only /v2/.../blobs/... URIs are actually cached (together with their redirect catchers)
- /manifests/, /token, and /v2/ are not cached anymore, which should solve a lot of problems
- better messages for /v1 attempts
- fix usage of $connect_host:443 (which is hostname:port and causes errors to be logged) to $connect_addr (which returns an IP:port) in the proxy layer

Author: rpardini

Dockerfile

@@ -47,6 +47,8 @@ ENV AUTH_REGISTRIES="some.authenticated.registry:oneuser:onepassword another.reg
 ENV VERIFY_SSL="true"
 # Enable debugging mode; this inserts mitmproxy/mitmweb between the CONNECT proxy and the caching layer
 ENV DEBUG="true"
+# Enable nginx debugging mode; this uses nginx-debug binary and enabled debug logging, which is VERY verbose so separate setting
+ENV DEBUG_NGINX="false"
 
-# Did you want a shell? Sorry. This only does one job; use exec /bin/bash if you wanna inspect stuff
+# Did you want a shell? Sorry, the entrypoint never returns, because it runs nginx itself. Use 'docker exec' if you need to mess around internally.
 ENTRYPOINT ["/entrypoint.sh"]

entrypoint.sh

@@ -58,12 +58,8 @@ NGINX_BIN="nginx"
 if [[ "a${DEBUG}" == "atrue" ]]; then
   # in debug mode, change caching layer to listen on 444, so that mitmproxy can sit in the middle.
   echo "        listen 444 ssl default_server;" > /etc/nginx/caching.layer.listen
-  echo "error_log  /var/log/nginx/error.log debug;" > /etc/nginx/error.log.debug.warn
-
-  # use debug binary
-  NGINX_BIN="nginx-debug"
 
-  echo "Starting in DEBUG MODE."
+  echo "Starting in DEBUG MODE (mitmproxy)."
   echo "Run mitmproxy with reverse pointing to the same certs..."
   mitmweb --no-web-open-browser --web-iface 0.0.0.0 --web-port 8081 \
           --set keep_host_header=true --set ssl_insecure=true \
@@ -73,6 +69,12 @@ if [[ "a${DEBUG}" == "atrue" ]]; then
   echo "Access mitmweb via http://127.0.0.1:8081/ "
 fi
 
+if [[ "a${DEBUG_NGINX}" == "atrue" ]]; then
+  echo "Starting in DEBUG MODE (nginx)."
+  echo "error_log  /var/log/nginx/error.log debug;" > /etc/nginx/error.log.debug.warn
+  # use debug binary
+  NGINX_BIN="nginx-debug"
+fi
 
 echo "Testing nginx config..."
 ${NGINX_BIN} -t

nginx.conf

@@ -21,6 +21,15 @@ http {
     '"HOST: $host" "UPSTREAM: $upstream_addr" '
     '"UPSTREAM-STATUS: $upstream_status" '
     '"SSL-PROTO: $ssl_protocol" '
+    '"CONNECT-HOST: $connect_host" "CONNECT-PORT: $connect_port" "CONNECT-ADDR: $connect_addr" '
+    '"PROXY-HOST: $proxy_host" "UPSTREAM-REDIRECT: $upstream_http_location" "CACHE-STATUS: $upstream_cache_status" '
+    '"AUTH: $http_authorization" ' ;
+
+    log_format  debug_proxy  'CONNECTPROXY: $remote_addr - $remote_user [$time_local] "$request" '
+    '$status $body_bytes_sent '
+    '"HOST: $host" "UPSTREAM: $upstream_addr" '
+    '"UPSTREAM-STATUS: $upstream_status" '
+    '"SSL-PROTO: $ssl_protocol" '
     '"CONNECT-HOST: $connect_host" "CONNECT-PORT: $connect_port" "CONNECT-ADDR: $connect_addr" "INTERCEPTED: $interceptedHost" '
     '"PROXY-HOST: $proxy_host" "UPSTREAM-REDIRECT: $upstream_http_location" "CACHE-STATUS: $upstream_cache_status" '
     '"AUTH: $http_authorization" ' ;
@@ -51,18 +60,23 @@ http {
         default "";
     }
 
+    # @TODO: actually for auth.docker.io, if we want to support multiple authentications, we'll need to decide
+    # @TODO: based not only on the hostname, but also URI (/token) and query string (?scope)
+    # @TODO: I wonder if this would help gcr.io and quay.io with authentication also....
+
+    map $dockerAuth $finalAuth {
+        "" "$http_authorization";    # if empty, keep the original passed-in from the docker client.
+        default "Basic $dockerAuth"; # if not empty, add the Basic preamble to the auth
+    }
+
+
     # Map to decide which hosts get directed to the caching portion.
     # This is automatically generated from the list of cached registries, plus a few fixed hosts
     # By default, we don't intercept, allowing free flow of non-registry traffic
     map $connect_host $interceptedHost {
         hostnames;
         include /etc/nginx/docker.intercept.map;
-        default "$connect_host:443";
-    }
-
-    map $dockerAuth $finalAuth {
-        "" "$http_authorization";    # if empty, keep the original passed-in from the client
-        default "Basic $dockerAuth"; # if not empty, add the Basic preamble to the auth
+        default "$connect_addr"; # $connect_addr is 'IP address and port of the remote host, e.g. "192.168.1.5:12345". IP address is resolved from host name of CONNECT request line.'
     }
 
 
@@ -86,6 +100,7 @@ http {
         server_name _;
 
         # dont log the CONNECT proxy.
+        #access_log /var/log/nginx/access.log debug_proxy;
         access_log off;
 
         proxy_connect;
@@ -97,7 +112,8 @@ http {
 
         # forward proxy for non-CONNECT request
         location / {
-            return 403 "The docker caching proxy is working!";
+            add_header "Content-type" "text/plain" always;
+            return 200 "docker-registry-proxy: The docker caching proxy is working!";
         }
 
         location /ca.crt {
@@ -145,7 +161,7 @@ http {
         # Use cache locking, with a huge timeout, so that multiple Docker clients asking for the same blob at the same time
         # will wait for the first to finish instead of doing multiple upstream requests.
         proxy_cache_lock on;
-        proxy_cache_lock_timeout 120s;
+        proxy_cache_lock_timeout 880s;
 
         # Cache all 200, 301, 302, and 307 (emitted by private registries) for 60 days.
         proxy_cache_valid 200 301 302 307 60d;
@@ -165,55 +181,18 @@ http {
         # This comes from a include file generated by the entrypoint.
         include /etc/nginx/docker.verify.ssl.conf;
 
-        # Some debugging info
-        # add_header X-Docker-Caching-Proxy-Real-Host $realHost;
-        # add_header X-Docker-Caching-Proxy-Real-Path $realPath;
-        # add_header X-Docker-Caching-Proxy-Auth $finalAuth;
-
         # Block API v1. We dont know how to handle these.
         # Docker-client should start with v2 and fallback to v1 if something fails, for example, if authentication failed to a protected v2 resource.
         location /v1 {
-            return 405 "API v1 is invalid. Either the image does not exist upstream, or you need auth to get a v2 endpoint working against $host";
+            return 405 "docker-registry-proxy: docker is trying to use v1 API. Either the image does not exist upstream, or you need to configure docker-registry-proxy to authenticate against $host";
         }
 
-        # don't cache mutable entity /v2/<name>/manifests/<reference> (unless the reference is a digest)
-        location ~ ^/v2/[^\/]+/manifests/(?![A-Fa-f0-9_+.-]+:) {
-            proxy_pass https://$targetHost;
-            add_header X-Docker-Caching-Proxy-Debug-Cache "no:manifests";
-        }
 
-        # don't cache mutable entity /v2/<name>/tags/list
-        location ~ ^/v2/[^\/]+/tags/list {
-            proxy_pass https://$targetHost;
-            proxy_cache off;
-            add_header X-Docker-Caching-Proxy-Debug-Cache "no:tagslist";
-        }
-
-        # don't cache mutable entity /v2/_catalog
-        location ~ ^/v2/_catalog$ {
-            proxy_pass https://$targetHost;
-            proxy_cache off;
-            add_header X-Docker-Caching-Proxy-Debug-Cache "no:catalog";
-        }
-
-        # dont cache the first hit which is always /v2/
-        location = /v2/ {
-            proxy_pass https://$targetHost;
-            proxy_cache off;
-            add_header X-Docker-Caching-Proxy-Debug-Cache "no:rootv2";
-        }
-
-        # dont cache /token (done against auth servers)
-        location = /token {
-            proxy_pass https://$targetHost;
-            proxy_cache off;
-            add_header X-Docker-Caching-Proxy-Debug-Cache "no:token";
-        }
-
-        # cache everything else
-        location / {
+        # for the /v2/..../blobs/.... URIs, do cache, and treat redirects.
+        location ~ ^/v2/(.*)/blobs/ {
             proxy_pass https://$targetHost;
             proxy_cache cache;
+            add_header X-Docker-Caching-Proxy-Debug-Cache "yes:blobs";
 
             # Handling of redirects.
             # Many registries (eg, quay.io, or k8s.gcr.io) emit a Location redirect
@@ -223,10 +202,9 @@ http {
             # We to it twice, one for http and another for https.
             proxy_redirect ~^https://([^:/]+)(/.+)$ https://docker.caching.proxy.internal/forcecachesecure/$1/originalwas$2;
             proxy_redirect ~^http://([^:/]+)(/.+)$ http://docker.caching.proxy.internal/forcecacheinsecure/$1/originalwas$2;
-
-            add_header X-Docker-Caching-Proxy-Debug-Cache "yes:everythingelse";
         }
 
+
         # handling for the redirect case explained above, with https.
         # The $realHost and $realPath variables come from a map defined at the top of this file.
         location /forcecachesecure {
@@ -251,5 +229,13 @@ http {
 
             add_header X-Docker-Caching-Proxy-Debug-Cache "yes:forcecacheinsecure";
         }
+
+        # by default, dont cache anything.
+        location / {
+            proxy_pass https://$targetHost;
+            proxy_cache off;
+            add_header X-Docker-Caching-Proxy-Debug-Cache "no:default";
+        }
+
     }
 }