fix(gateway): Cache-Control header for UnixFS directories on /ipfs (#643)

* fix(gateway): Cache-Control header for UnixFS directories
* fix: limit static cache-control to /ipfs

just a precaution, since we still have TTL=0 for many DNSLink responses
due to #329 (comment)

---------

Co-authored-by: Joshua Noble <jnoble@filebase.com>

Author: lidel

CHANGELOG.md

@@ -20,6 +20,7 @@ The following emojis are used to highlight certain changes:
 
 - `bitswap/server` minor memory use and performance improvements
 - `bitswap` unify logger names to use uniform format bitswap/path/pkgname
+- `gateway` now always returns meaningful cache-control headers for generated HTML listings of UnixFS directories
 
 ### Removed
 

gateway/gateway_test.go

@@ -130,9 +130,9 @@ func TestHeaders(t *testing.T) {
 			path         string
 			cacheControl string
 		}{
-			{"/ipns/example.net/", "public, max-age=30"},                 // As generated directory listing
-			{"/ipns/example.com/", "public, max-age=55"},                 // As generated directory listing (different)
-			{"/ipns/unknown.com/", ""},                                   // As generated directory listing (unknown)
+			{"/ipns/example.net/", "public, max-age=30, stale-while-revalidate=2678400"}, // As generated directory listing
+			{"/ipns/example.com/", "public, max-age=55, stale-while-revalidate=2678400"}, // As generated directory listing (different)
+			{"/ipns/unknown.com/", ""},                                   // As generated directory listing (unknown TTL)
 			{"/ipns/example.net/foo/", "public, max-age=30"},             // As index.html directory listing
 			{"/ipns/example.net/foo/index.html", "public, max-age=30"},   // As deserialized UnixFS file
 			{"/ipns/example.net/?format=raw", "public, max-age=30"},      // As Raw block

gateway/handler_unixfs_dir.go

@@ -136,9 +136,14 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r *
 	dirEtag := getDirListingEtag(resolvedPath.RootCid())
 	w.Header().Set("Etag", dirEtag)
 
-	// Add TTL if known.
+	// Set Cache-Control
 	if rq.ttl > 0 {
-		w.Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d", int(rq.ttl.Seconds())))
+		// Use known TTL from IPNS Record or DNSLink TXT Record
+		w.Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d, stale-while-revalidate=2678400", int(rq.ttl.Seconds())))
+	} else if !rq.contentPath.Mutable() {
+		// Cache for 1 week, serve stale cache for up to a month
+		// (style of generated HTML may change, should not be cached forever)
+		w.Header().Set("Cache-Control", "public, max-age=604800, stale-while-revalidate=2678400")
 	}
 
 	if r.Method == http.MethodHead {