all: rework HTTP service plumbing

Instead of making separate endpoints for each service, multiplex them all onto
a single address configured by the new --http flag.

Convert --modproxy into a Boolean flag, and collapse the --revcache flags to a
single flag specifying the target hosts.

Reorganize the reverse proxy plumbing so that it does not require weirdly
structured URLs anymore; this makes things work better with proxy environments.

Update the help text throughout.

Author: creachadair

cmd/go-cache-plugin/commands.go

@@ -120,9 +120,9 @@ func runDirect(env *command.Env) error {
 
 var serveFlags struct {
 	Socket   string `flag:"socket,default=$GOCACHE_SOCKET,Socket path (required)"`
-	ModProxy string `flag:"modproxy,default=$GOCACHE_MODPROXY,Module proxy service address ([host]:port)"`
-	RevProxy string `flag:"revproxy,default=$GOCACHE_REVPROXY,Reverse proxy service address ([host]:port)"`
-	Targets  string `flag:"revproxy-targets,default=$GOCACHE_REVPROXY_TARGETS,Reverse proxy targets (comma-separated)"`
+	HTTP     string `flag:"http,default=$GOCACHE_HTTP,HTTP service address ([host]:port)"`
+	ModProxy bool   `flag:"modproxy,default=$GOCACHE_MODPROXY,Enable a Go module proxy (requires --http)"`
+	RevProxy string `flag:"revproxy,default=$GOCACHE_REVPROXY,Reverse proxy these hosts (comma-separated)"`
 	SumDB    string `flag:"sumdb,default=$GOCACHE_SUMDB,SumDB servers to proxy for (comma-separated)"`
 }
 
@@ -160,8 +160,30 @@ func runServe(env *command.Env) error {
 		lst.Close()
 	}))
 
+	// If an HTTP server is enabled, start it up with debug routes.
+	// If enabled, the module proxy and reverse cache attach to mux below.
+	var mux *http.ServeMux
+	if serveFlags.HTTP != "" {
+		mux = http.NewServeMux()
+		tsweb.Debugger(mux)
+		srv := &http.Server{
+			Addr:    serveFlags.HTTP,
+			Handler: mux,
+		}
+		g.Go(srv.ListenAndServe)
+		vprintf("started HTTP server at %q", serveFlags.HTTP)
+		g.Go(taskgroup.NoError(func() {
+			<-ctx.Done()
+			vprintf("signal received, stopping HTTP service")
+			srv.Shutdown(context.Background())
+		}))
+	}
+
 	// If a module proxy is enabled, start it.
-	if serveFlags.ModProxy != "" {
+	if serveFlags.ModProxy {
+		if mux == nil {
+			return env.Usagef("you must set --http to enable --modproxy")
+		}
 		modCachePath := filepath.Join(flags.CacheDir, "module")
 		if err := os.MkdirAll(modCachePath, 0700); err != nil {
 			lst.Close()
@@ -190,64 +212,35 @@ func runServe(env *command.Env) error {
 			Cacher:        cacher,
 			ProxiedSumDBs: []string{"sum.golang.org"}, // default, see below
 		}
+		vprintf("enabling Go module proxy")
 		if serveFlags.SumDB != "" {
 			proxy.ProxiedSumDBs = strings.Split(serveFlags.SumDB, ",")
 			vprintf("enabling sum DB proxy for %s", strings.Join(proxy.ProxiedSumDBs, ", "))
 		}
 		expvar.Publish("modcache", cacher.Metrics())
-
-		// Run an HTTP server exporting the proxy and debug metrics.
-		mux := http.NewServeMux()
-		mux.Handle("/", proxy)
-		tsweb.Debugger(mux)
-		srv := &http.Server{
-			Addr:    serveFlags.ModProxy,
-			Handler: mux,
-		}
-		g.Go(srv.ListenAndServe)
-		vprintf("started module proxy at %q", serveFlags.ModProxy)
-		g.Go(taskgroup.NoError(func() {
-			<-ctx.Done()
-			vprintf("signal received, stopping module proxy")
-			srv.Shutdown(context.Background())
-		}))
+		mux.Handle("/mod/", http.StripPrefix("/mod", proxy))
 	}
 
 	// If a reverse proxy is enabled, start it.
 	if serveFlags.RevProxy != "" {
-		if serveFlags.Targets == "" {
-			return env.Usagef("must provide --revproxy-targets when --revproxy is set")
+		if mux == nil {
+			return env.Usagef("you must set --http to enable --revproxy")
 		}
 		revCachePath := filepath.Join(flags.CacheDir, "revproxy")
 		if err := os.MkdirAll(revCachePath, 0700); err != nil {
 			lst.Close()
 			return fmt.Errorf("create revproxy cache: %w", err)
 		}
 		proxy := &revproxy.Server{
-			Targets:   strings.Split(serveFlags.Targets, ","),
+			Targets:   strings.Split(serveFlags.RevProxy, ","),
 			Local:     revCachePath,
 			S3Client:  s3c,
 			KeyPrefix: path.Join(flags.KeyPrefix, "revproxy"),
 			Logf:      vprintf,
 		}
 		expvar.Publish("revcache", proxy.Metrics())
-
-		mux := http.NewServeMux()
+		vprintf("enabling reverse proxy for %s", strings.Join(proxy.Targets, ", "))
 		mux.Handle("/", proxy)
-		if serveFlags.ModProxy == "" {
-			tsweb.Debugger(mux) // attach debugger if --modproxy doesn't already have it
-		}
-		srv := &http.Server{
-			Addr:    serveFlags.RevProxy,
-			Handler: mux,
-		}
-		g.Go(srv.ListenAndServe)
-		vprintf("started reverse proxy at %q for %s", serveFlags.RevProxy, strings.Join(proxy.Targets, ", "))
-		g.Go(taskgroup.NoError(func() {
-			<-ctx.Done()
-			vprintf("signal received, stopping reverse proxy")
-			srv.Shutdown(context.Background())
-		}))
 	}
 
 	for {

cmd/go-cache-plugin/go-cache-plugin.go

@@ -46,8 +46,16 @@ serving directly over stdin/stdout. The "connect" command adapts the direct
 interface to this one.
 
 By default, only the build cache is exported via the --socket path.
-If --modcache is set, the server also exports a caching module proxy at the
-specified address.`,
+
+If --http is set, the server also exports an HTTP server at that address.
+By default, this exports only /debug endpoints, including metrics.
+When --http is enabled, the following options are available:
+
+- When --modcache is true, the server also exports a caching module proxy at
+  http://<host>:<port>/mod/.
+
+- When --revproxy is set, the server also hosts a caching reverse proxy for the
+  specified hosts at http://<host>:<port>/revproxy.`,
 
 				SetFlags: command.Flags(flax.MustBind, &serveFlags),
 				Run:      command.Adapt(runServe),

cmd/go-cache-plugin/help.go

@@ -19,58 +19,8 @@ get credentials from the instnce metadata service; otherwise you will need to
 plumb AWS environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) or
 set up a configuration file.
 
-------------------------------------------------------------------------
-## To run the plugin directly as a subprocess of the toolchain
-
-     export GOCACHEPROG="go-cache-plugin --cache-dir=/tmp/gocache --bucket ..."
-     go build ...
-
-   Alternatively:
-
-     export GOCACHE_DIR=/tmp/gocache
-     export GOCACHE_S3_BUCKET=cache-bucket-name
-     export GOCACHEPROG=go-cache-plugin
-     go build ...
-
-   In this mode, you must specify the --cache-dir and --bucket settings.
-
-------------------------------------------------------------------------
-## To run the plugin as a standalone service
-
-   Use the "serve" subcommand:
-
-     go-cache-plugin serve \
-        --cache-dir=/tmp/gocache --bucket=$B \
-        --socket /var/run/cache.sock
-
-   You can then use the "connect" subcommand to wire up the toolchain:
-
-     export GOCACHEPROG="go-cache-plugin connect /var/run/cache.sock"
-
-   In this mode, the server must have credentials to access to S3, but the
-   toolchain process does not need AWS credentials.
-
-------------------------------------------------------------------------
-## Running a Go module and sum database proxy
-
-   With the --modproxy flag, the server will also export an HTTP proxy for the
-   public Go module proxy (proxy.golang.org) and sum DB (sum.golang.org) at the
-   given address:
-
-      go-cache-plugin serve ... --modproxy=localhost:5970
-
-   To use the module proxy, set the standard GOPROXY environment variable:
-
-      export GOPROXY=localhost:5970
-      export GOCACHEPROG="go-cache-plugin connect /var/run/cache.sock"
-      go build ...
-
-   To use the sum DB proxy, set the GOSUMDB environment variable:
-
-      export GOSUMDB="sum.golang.org http://localhost:5970/sumdb/sum.golang.org"
-
-   See also: https://proxy.golang.org/
-`,
+See also: "help environment".
+Related:  "direct-mode", "serve-mode", "module-proxy", "reverse-proxy".`,
 	},
 	{
 		Name: "environment",
@@ -98,9 +48,81 @@ settings can be set via environment variables as well as flags.
    Flag (serve)       Variable               Format      Default
    --------------------------------------------------------------------
     --socket          GOCACHE_SOCKET         path        (required)
-    --modproxy        GOCACHE_MODPROXY       [host]:port ""
+    --http            GOCACHE_HTTP           [host]:port ""
+    --modproxy        GOCACHE_MODPROXY       bool        false
+    --revproxy        GOCACHE_REVPROXY       host,...    ""
     --sumdb           GOCACHE_SUMDB          host,...    ""
 
-See also "help configure".`,
+See also: "help configure".`,
+	},
+	{
+		Name: "direct-mode",
+		Help: `Run the plugin directly as a subprocess of the toolchain.
+
+  export GOCACHEPROG="go-cache-plugin --cache-dir=/tmp/gocache --bucket ..."
+  go build ...
+
+Alternatively:
+
+  export GOCACHE_DIR=/tmp/gocache
+  export GOCACHE_S3_BUCKET=cache-bucket-name
+  export GOCACHEPROG=go-cache-plugin
+  go build ...
+
+In this mode, you must specify the --cache-dir and --bucket settings.`,
+	},
+	{
+		Name: "serve-mode",
+		Help: `Run the plugin as a standalone service.
+
+Use the "serve" subcommand:
+
+  go-cache-plugin serve \
+     --cache-dir=/tmp/gocache --bucket=$B \
+     --socket /var/run/cache.sock
+
+You can then use the "connect" subcommand to wire up the toolchain:
+
+  export GOCACHEPROG="go-cache-plugin connect /var/run/cache.sock"
+
+In this mode, the server must have credentials to access to S3, but the
+toolchain process does not need AWS credentials.`,
+	},
+	{
+		Name: "module-proxy",
+		Help: `Run a Go module and sum database proxy.
+
+With the --modproxy flag, the server will also export an HTTP proxy for the
+public Go module proxy (proxy.golang.org) and sum DB (sum.golang.org) at the
+given address:
+
+   go-cache-plugin serve ... --http=localhost:5970 --modproxy
+
+To use the module proxy, set the standard GOPROXY environment variable:
+
+   export GOPROXY=localhost:5970
+   export GOCACHEPROG="go-cache-plugin connect /var/run/cache.sock"
+   go build ...
+
+To use the sum DB proxy, set the GOSUMDB environment variable:
+
+   export GOSUMDB="sum.golang.org http://localhost:5970/sumdb/sum.golang.org"
+
+See also: https://proxy.golang.org/`,
+	},
+	{
+		Name: "reverse-proxy",
+		Help: `Run a caching reverse proxy.
+
+With the --revproxy flag, the server will also export a caching reverse
+proxy for the specified hosts, given as a comma-separated list:
+
+   go-cache-plugin serve ... \
+      --http=localhost:5970 \
+      --revproxy='api.example.com,*.example2.com'
+
+When this is enabled, you can configure this address as an HTTP proxy:
+
+   HTTP_PROXY=localhost:5970 curl https://api.example.com/foo`,
 	},
 }

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/creachadair/gocache v0.0.0-20240828204135-c17fe2fd53a6
 	github.com/creachadair/mds v0.17.1
 	github.com/creachadair/taskgroup v0.9.1
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da
 	github.com/goproxy/goproxy v0.17.2
 	golang.org/x/sync v0.8.0
 	honnef.co/go/tools v0.5.1
@@ -40,7 +41,6 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/go-json-experiment/json v0.0.0-20231102232822-2e55bd4e08b0 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/prometheus/client_golang v1.19.1 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect

revproxy/cache.go

@@ -91,12 +91,26 @@ func (s *Server) cacheStoreMemory(hash string, maxAge time.Duration, hdr http.He
 	s.mcacheMu.Lock()
 	defer s.mcacheMu.Unlock()
 	s.mcache.Add(hash, memCacheEntry{
-		header:  hdr,
+		header:  trimCacheHeader(hdr),
 		body:    body,
 		expires: time.Now().Add(maxAge),
 	})
 }
 
+var keepHeader = []string{
+	"Cache-Control", "Content-Type", "Date", "Etag",
+}
+
+func trimCacheHeader(h http.Header) http.Header {
+	out := make(http.Header)
+	for _, name := range keepHeader {
+		if v := h.Get(name); v != "" {
+			out.Set(name, v)
+		}
+	}
+	return out
+}
+
 // parseCacheDbject parses cached object data to extract the body and headers.
 func parseCacheObject(data []byte) ([]byte, http.Header, error) {
 	hdr, rest, ok := bytes.Cut(data, []byte("\n\n"))

revproxy/internal_test.go

@@ -1,14 +1,11 @@
 package revproxy
 
 import (
-	"net/url"
 	"testing"
 )
 
 func TestCheckTarget(t *testing.T) {
-	s := &Server{
-		Targets: []string{"foo.com", "*.bar.com"},
-	}
+	testTargets := []string{"foo.com", "*.bar.com"}
 	tests := []struct {
 		input string
 		want  bool
@@ -22,8 +19,7 @@ func TestCheckTarget(t *testing.T) {
 		{"some.other.bar.com", true},
 	}
 	for _, tc := range tests {
-		u := &url.URL{Host: "localhost", Path: tc.input}
-		if got := s.checkTarget(u); got != tc.want {
+		if got := hostMatchesTarget(tc.input, testTargets); got != tc.want {
 			t.Errorf("Check %q: got %v, want %v", tc.input, got, tc.want)
 		}
 	}