Merge pull request ipfs/kubo#9395 from ipfs/release-v0.17.0

release v0.17.0

This commit was moved from ipfs/kubo@4485d6b

Author: galargh

gateway/core/corehttp/gateway.go

@@ -77,7 +77,7 @@ func GatewayOption(writable bool, paths ...string) ServeOption {
 
 		AddAccessControlHeaders(headers)
 
-		offlineApi, err := api.WithOptions(options.Api.Offline(true))
+		offlineAPI, err := api.WithOptions(options.Api.Offline(true))
 		if err != nil {
 			return nil, err
 		}
@@ -86,7 +86,7 @@ func GatewayOption(writable bool, paths ...string) ServeOption {
 			Headers:               headers,
 			Writable:              writable,
 			FastDirIndexThreshold: int(cfg.Gateway.FastDirIndexThreshold.WithDefault(100)),
-		}, api, offlineApi)
+		}, api, offlineAPI)
 
 		gateway = otelhttp.NewHandler(gateway, "Gateway.Request")
 

gateway/core/corehttp/gateway_handler.go

@@ -39,7 +39,7 @@ const (
 )
 
 var (
-	onlyAscii = regexp.MustCompile("[[:^ascii:]]")
+	onlyASCII = regexp.MustCompile("[[:^ascii:]]")
 	noModtime = time.Unix(0, 0) // disables Last-Modified header if passed as modtime
 )
 
@@ -68,7 +68,7 @@ type redirectTemplateData struct {
 type gatewayHandler struct {
 	config     GatewayConfig
 	api        NodeAPI
-	offlineApi NodeAPI
+	offlineAPI NodeAPI
 
 	// generic metrics
 	firstContentBlockGetMetric *prometheus.HistogramVec
@@ -214,15 +214,15 @@ func newGatewayHistogramMetric(name string, help string) *prometheus.HistogramVe
 
 // NewGatewayHandler returns an http.Handler that can act as a gateway to IPFS content
 // offlineApi is a version of the API that should not make network requests for missing data
-func NewGatewayHandler(c GatewayConfig, api NodeAPI, offlineApi NodeAPI) http.Handler {
-	return newGatewayHandler(c, api, offlineApi)
+func NewGatewayHandler(c GatewayConfig, api NodeAPI, offlineAPI NodeAPI) http.Handler {
+	return newGatewayHandler(c, api, offlineAPI)
 }
 
-func newGatewayHandler(c GatewayConfig, api NodeAPI, offlineApi NodeAPI) *gatewayHandler {
+func newGatewayHandler(c GatewayConfig, api NodeAPI, offlineAPI NodeAPI) *gatewayHandler {
 	i := &gatewayHandler{
 		config:     c,
 		api:        api,
-		offlineApi: offlineApi,
+		offlineAPI: offlineAPI,
 		// Improved Metrics
 		// ----------------------------
 		// Time till the first content block (bar in /ipfs/cid/foo/bar)
@@ -430,6 +430,10 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 		carVersion := formatParams["version"]
 		i.serveCAR(r.Context(), w, r, resolvedPath, contentPath, carVersion, begin)
 		return
+	case "application/x-tar":
+		logger.Debugw("serving tar file", "path", contentPath)
+		i.serveTAR(r.Context(), w, r, resolvedPath, contentPath, begin, logger)
+		return
 	default: // catch-all for unsuported application/vnd.*
 		err := fmt.Errorf("unsupported format %q", responseFormat)
 		webError(w, "failed respond with requested content type", err, http.StatusBadRequest)
@@ -683,7 +687,7 @@ func addContentDispositionHeader(w http.ResponseWriter, r *http.Request, content
 // Set Content-Disposition to arbitrary filename and disposition
 func setContentDispositionHeader(w http.ResponseWriter, filename string, disposition string) {
 	utf8Name := url.PathEscape(filename)
-	asciiName := url.PathEscape(onlyAscii.ReplaceAllLiteralString(filename, "_"))
+	asciiName := url.PathEscape(onlyASCII.ReplaceAllLiteralString(filename, "_"))
 	w.Header().Set("Content-Disposition", fmt.Sprintf("%s; filename=\"%s\"; filename*=UTF-8''%s", disposition, asciiName, utf8Name))
 }
 
@@ -842,9 +846,10 @@ func getEtag(r *http.Request, cid cid.Cid) string {
 	responseFormat, _, err := customResponseFormat(r)
 	if err == nil && responseFormat != "" {
 		// application/vnd.ipld.foo → foo
-		f := responseFormat[strings.LastIndex(responseFormat, ".")+1:]
-		// Etag: "cid.foo" (gives us nice compression together with Content-Disposition in block (raw) and car responses)
-		suffix = `.` + f + suffix
+		// application/x-bar → x-bar
+		shortFormat := responseFormat[strings.LastIndexAny(responseFormat, "/.")+1:]
+		// Etag: "cid.shortFmt" (gives us nice compression together with Content-Disposition in block (raw) and car responses)
+		suffix = `.` + shortFormat + suffix
 	}
 	// TODO: include selector suffix when https://github.com/ipfs/kubo/issues/8769 lands
 	return prefix + cid.String() + suffix
@@ -859,14 +864,17 @@ func customResponseFormat(r *http.Request) (mediaType string, params map[string]
 			return "application/vnd.ipld.raw", nil, nil
 		case "car":
 			return "application/vnd.ipld.car", nil, nil
+		case "tar":
+			return "application/x-tar", nil, nil
 		}
 	}
 	// Browsers and other user agents will send Accept header with generic types like:
 	// Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8
-	// We only care about explciit, vendor-specific content-types.
+	// We only care about explicit, vendor-specific content-types.
 	for _, accept := range r.Header.Values("Accept") {
 		// respond to the very first ipld content type
-		if strings.HasPrefix(accept, "application/vnd.ipld") {
+		if strings.HasPrefix(accept, "application/vnd.ipld") ||
+			strings.HasPrefix(accept, "application/x-tar") {
 			mediatype, params, err := mime.ParseMediaType(accept)
 			if err != nil {
 				return "", nil, err
@@ -933,7 +941,7 @@ func (i *gatewayHandler) handlePathResolution(w http.ResponseWriter, r *http.Req
 // https://github.com/ipfs/specs/blob/main/http-gateways/PATH_GATEWAY.md#cache-control-request-header
 func (i *gatewayHandler) handleOnlyIfCached(w http.ResponseWriter, r *http.Request, contentPath ipath.Path, logger *zap.SugaredLogger) (requestHandled bool) {
 	if r.Header.Get("Cache-Control") == "only-if-cached" {
-		_, err := i.offlineApi.Block().Stat(r.Context(), contentPath)
+		_, err := i.offlineAPI.Block().Stat(r.Context(), contentPath)
 		if err != nil {
 			if r.Method == http.MethodHead {
 				w.WriteHeader(http.StatusPreconditionFailed)

gateway/core/corehttp/gateway_handler_tar.go

@@ -0,0 +1,92 @@
+package corehttp
+
+import (
+	"context"
+	"html"
+	"net/http"
+	"time"
+
+	files "github.com/ipfs/go-ipfs-files"
+	ipath "github.com/ipfs/interface-go-ipfs-core/path"
+	"github.com/ipfs/kubo/tracing"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
+	"go.uber.org/zap"
+)
+
+var unixEpochTime = time.Unix(0, 0)
+
+func (i *gatewayHandler) serveTAR(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, logger *zap.SugaredLogger) {
+	ctx, span := tracing.Span(ctx, "Gateway", "ServeTAR", trace.WithAttributes(attribute.String("path", resolvedPath.String())))
+	defer span.End()
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	// Get Unixfs file
+	file, err := i.api.Unixfs().Get(ctx, resolvedPath)
+	if err != nil {
+		webError(w, "ipfs cat "+html.EscapeString(contentPath.String()), err, http.StatusBadRequest)
+		return
+	}
+	defer file.Close()
+
+	rootCid := resolvedPath.Cid()
+
+	// Set Cache-Control and read optional Last-Modified time
+	modtime := addCacheControlHeaders(w, r, contentPath, rootCid)
+
+	// Weak Etag W/ because we can't guarantee byte-for-byte identical
+	// responses, but still want to benefit from HTTP Caching. Two TAR
+	// responses for the same CID will be logically equivalent,
+	// but when TAR is streamed, then in theory, files and directories
+	// may arrive in different order (depends on TAR lib and filesystem/inodes).
+	etag := `W/` + getEtag(r, rootCid)
+	w.Header().Set("Etag", etag)
+
+	// Finish early if Etag match
+	if r.Header.Get("If-None-Match") == etag {
+		w.WriteHeader(http.StatusNotModified)
+		return
+	}
+
+	// Set Content-Disposition
+	var name string
+	if urlFilename := r.URL.Query().Get("filename"); urlFilename != "" {
+		name = urlFilename
+	} else {
+		name = rootCid.String() + ".tar"
+	}
+	setContentDispositionHeader(w, name, "attachment")
+
+	// Construct the TAR writer
+	tarw, err := files.NewTarWriter(w)
+	if err != nil {
+		webError(w, "could not build tar writer", err, http.StatusInternalServerError)
+		return
+	}
+	defer tarw.Close()
+
+	// Sets correct Last-Modified header. This code is borrowed from the standard
+	// library (net/http/server.go) as we cannot use serveFile without throwing the entire
+	// TAR into the memory first.
+	if !(modtime.IsZero() || modtime.Equal(unixEpochTime)) {
+		w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat))
+	}
+
+	w.Header().Set("Content-Type", "application/x-tar")
+	w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^)
+
+	// The TAR has a top-level directory (or file) named by the CID.
+	if err := tarw.WriteFile(file, rootCid.String()); err != nil {
+		w.Header().Set("X-Stream-Error", err.Error())
+		// Trailer headers do not work in web browsers
+		// (see https://github.com/mdn/browser-compat-data/issues/14703)
+		// and we have limited options around error handling in browser contexts.
+		// To improve UX/DX, we finish response stream with error message, allowing client to
+		// (1) detect error by having corrupted TAR
+		// (2) be able to reason what went wrong by instecting the tail of TAR stream
+		_, _ = w.Write([]byte(err.Error()))
+		return
+	}
+}

gateway/core/corehttp/gateway_handler_unixfs_dir.go

@@ -39,10 +39,10 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
 		webError(w, "failed to parse request path", err, http.StatusInternalServerError)
 		return
 	}
-	originalUrlPath := requestURI.Path
+	originalURLPath := requestURI.Path
 
 	// Ensure directory paths end with '/'
-	if originalUrlPath[len(originalUrlPath)-1] != '/' {
+	if originalURLPath[len(originalURLPath)-1] != '/' {
 		// don't redirect to trailing slash if it's go get
 		// https://github.com/ipfs/kubo/pull/3963
 		goget := r.URL.Query().Get("go-get") == "1"
@@ -53,7 +53,7 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
 				suffix = suffix + "?" + r.URL.RawQuery
 			}
 			// /ipfs/cid/foo?bar must be redirected to /ipfs/cid/foo/?bar
-			redirectURL := originalUrlPath + suffix
+			redirectURL := originalURLPath + suffix
 			logger.Debugw("directory location moved permanently", "status", http.StatusMovedPermanently)
 			http.Redirect(w, r, redirectURL, http.StatusMovedPermanently)
 			return
@@ -125,7 +125,7 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
 		di := directoryItem{
 			Size:      "", // no size because we did not fetch child nodes
 			Name:      link.Name,
-			Path:      gopath.Join(originalUrlPath, link.Name),
+			Path:      gopath.Join(originalURLPath, link.Name),
 			Hash:      hash,
 			ShortHash: shortHash(hash),
 		}
@@ -149,7 +149,7 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
 
 	// construct the correct back link
 	// https://github.com/ipfs/kubo/issues/1365
-	var backLink string = originalUrlPath
+	backLink := originalURLPath
 
 	// don't go further up than /ipfs/$hash/
 	pathSplit := path.SplitList(contentPath.String())

gateway/core/corehttp/gateway_indexPage.go

@@ -117,8 +117,8 @@ func init() {
 
 	// custom template-escaping function to escape a full path, including '#' and '?'
 	urlEscape := func(rawUrl string) string {
-		pathUrl := url.URL{Path: rawUrl}
-		return pathUrl.String()
+		pathURL := url.URL{Path: rawUrl}
+		return pathURL.String()
 	}
 
 	// Directory listing template

gateway/core/corehttp/hostname.go

@@ -84,7 +84,8 @@ func HostnameOption() ServeOption {
 					if gw.UseSubdomains {
 						// Yes, redirect if applicable
 						// Example: dweb.link/ipfs/{cid} → {cid}.ipfs.dweb.link
-						newURL, err := toSubdomainURL(host, r.URL.Path, r, coreAPI)
+						useInlinedDNSLink := gw.InlineDNSLink.WithDefault(config.DefaultInlineDNSLink)
+						newURL, err := toSubdomainURL(host, r.URL.Path, r, useInlinedDNSLink, coreAPI)
 						if err != nil {
 							http.Error(w, err.Error(), http.StatusBadRequest)
 							return
@@ -132,6 +133,9 @@ func HostnameOption() ServeOption {
 				// Assemble original path prefix.
 				pathPrefix := "/" + ns + "/" + rootID
 
+				// Retrieve whether or not we should inline DNSLink.
+				useInlinedDNSLink := gw.InlineDNSLink.WithDefault(config.DefaultInlineDNSLink)
+
 				// Does this gateway _handle_ subdomains AND this path?
 				if !(gw.UseSubdomains && hasPrefix(pathPrefix, gw.Paths...)) {
 					// If not, resource does not exist, return 404
@@ -149,7 +153,7 @@ func HostnameOption() ServeOption {
 					}
 					if !strings.HasPrefix(r.Host, dnsCID) {
 						dnsPrefix := "/" + ns + "/" + dnsCID
-						newURL, err := toSubdomainURL(gwHostname, dnsPrefix+r.URL.Path, r, coreAPI)
+						newURL, err := toSubdomainURL(gwHostname, dnsPrefix+r.URL.Path, r, useInlinedDNSLink, coreAPI)
 						if err != nil {
 							http.Error(w, err.Error(), http.StatusBadRequest)
 							return
@@ -165,7 +169,7 @@ func HostnameOption() ServeOption {
 					// Do we need to fix multicodec in PeerID represented as CIDv1?
 					if isPeerIDNamespace(ns) {
 						if rootCID.Type() != cid.Libp2pKey {
-							newURL, err := toSubdomainURL(gwHostname, pathPrefix+r.URL.Path, r, coreAPI)
+							newURL, err := toSubdomainURL(gwHostname, pathPrefix+r.URL.Path, r, useInlinedDNSLink, coreAPI)
 							if err != nil {
 								http.Error(w, err.Error(), http.StatusBadRequest)
 								return
@@ -451,7 +455,7 @@ func toDNSLinkFQDN(dnsLabel string) (fqdn string) {
 }
 
 // Converts a hostname/path to a subdomain-based URL, if applicable.
-func toSubdomainURL(hostname, path string, r *http.Request, ipfs iface.CoreAPI) (redirURL string, err error) {
+func toSubdomainURL(hostname, path string, r *http.Request, inlineDNSLink bool, ipfs iface.CoreAPI) (redirURL string, err error) {
 	var scheme, ns, rootID, rest string
 
 	query := r.URL.RawQuery
@@ -554,7 +558,7 @@ func toSubdomainURL(hostname, path string, r *http.Request, ipfs iface.CoreAPI)
 		// can be loaded from a subdomain gateway with a wildcard TLS cert if
 		// represented as a single DNS label:
 		// https://my-v--long-example-com.ipns.dweb.link
-		if isHTTPS && ns == "ipns" && strings.Contains(rootID, ".") {
+		if (inlineDNSLink || isHTTPS) && ns == "ipns" && strings.Contains(rootID, ".") {
 			if isDNSLinkName(r.Context(), ipfs, rootID) {
 				// my.v-long.example.com → my-v--long-example-com
 				dnsLabel, err := toDNSLinkDNSLabel(rootID)

gateway/core/corehttp/hostname_test.go

@@ -36,35 +36,39 @@ func TestToSubdomainURL(t *testing.T) {
 
 	for _, test := range []struct {
 		// in:
-		request    *http.Request
-		gwHostname string
-		path       string
+		request       *http.Request
+		gwHostname    string
+		inlineDNSLink bool
+		path          string
 		// out:
 		url string
 		err error
 	}{
 		// DNSLink
-		{httpRequest, "localhost", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost/", nil},
+		{httpRequest, "localhost", false, "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost/", nil},
 		// Hostname with port
-		{httpRequest, "localhost:8080", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost:8080/", nil},
+		{httpRequest, "localhost:8080", false, "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost:8080/", nil},
 		// CIDv0 → CIDv1base32
-		{httpRequest, "localhost", "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.localhost/", nil},
+		{httpRequest, "localhost", false, "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.localhost/", nil},
 		// CIDv1 with long sha512
-		{httpRequest, "localhost", "/ipfs/bafkrgqe3ohjcjplc6n4f3fwunlj6upltggn7xqujbsvnvyw764srszz4u4rshq6ztos4chl4plgg4ffyyxnayrtdi5oc4xb2332g645433aeg", "", errors.New("CID incompatible with DNS label length limit of 63: kf1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5oj")},
+		{httpRequest, "localhost", false, "/ipfs/bafkrgqe3ohjcjplc6n4f3fwunlj6upltggn7xqujbsvnvyw764srszz4u4rshq6ztos4chl4plgg4ffyyxnayrtdi5oc4xb2332g645433aeg", "", errors.New("CID incompatible with DNS label length limit of 63: kf1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5oj")},
 		// PeerID as CIDv1 needs to have libp2p-key multicodec
-		{httpRequest, "localhost", "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", "http://k2k4r8n0flx3ra0y5dr8fmyvwbzy3eiztmtq6th694k5a3rznayp3e4o.ipns.localhost/", nil},
-		{httpRequest, "localhost", "/ipns/bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "http://k2k4r8l9ja7hkzynavdqup76ou46tnvuaqegbd04a4o1mpbsey0meucb.ipns.localhost/", nil},
+		{httpRequest, "localhost", false, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", "http://k2k4r8n0flx3ra0y5dr8fmyvwbzy3eiztmtq6th694k5a3rznayp3e4o.ipns.localhost/", nil},
+		{httpRequest, "localhost", false, "/ipns/bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "http://k2k4r8l9ja7hkzynavdqup76ou46tnvuaqegbd04a4o1mpbsey0meucb.ipns.localhost/", nil},
 		// PeerID: ed25519+identity multihash → CIDv1Base36
-		{httpRequest, "localhost", "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", "http://k51qzi5uqu5di608geewp3nqkg0bpujoasmka7ftkyxgcm3fh1aroup0gsdrna.ipns.localhost/", nil},
-		{httpRequest, "sub.localhost", "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.sub.localhost/", nil},
+		{httpRequest, "localhost", false, "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", "http://k51qzi5uqu5di608geewp3nqkg0bpujoasmka7ftkyxgcm3fh1aroup0gsdrna.ipns.localhost/", nil},
+		{httpRequest, "sub.localhost", false, "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.sub.localhost/", nil},
 		// HTTPS requires DNSLink name to fit in a single DNS label – see "Option C" from https://github.com/ipfs/in-web-browsers/issues/169
-		{httpRequest, "dweb.link", "/ipns/dnslink.long-name.example.com", "http://dnslink.long-name.example.com.ipns.dweb.link/", nil},
-		{httpsRequest, "dweb.link", "/ipns/dnslink.long-name.example.com", "https://dnslink-long--name-example-com.ipns.dweb.link/", nil},
-		{httpsProxiedRequest, "dweb.link", "/ipns/dnslink.long-name.example.com", "https://dnslink-long--name-example-com.ipns.dweb.link/", nil},
+		{httpRequest, "dweb.link", false, "/ipns/dnslink.long-name.example.com", "http://dnslink.long-name.example.com.ipns.dweb.link/", nil},
+		{httpsRequest, "dweb.link", false, "/ipns/dnslink.long-name.example.com", "https://dnslink-long--name-example-com.ipns.dweb.link/", nil},
+		{httpsProxiedRequest, "dweb.link", false, "/ipns/dnslink.long-name.example.com", "https://dnslink-long--name-example-com.ipns.dweb.link/", nil},
+		// HTTP requests can also be converted to fit into a single DNS label - https://github.com/ipfs/kubo/issues/9243
+		{httpRequest, "localhost", true, "/ipns/dnslink.long-name.example.com", "http://dnslink-long--name-example-com.ipns.localhost/", nil},
+		{httpRequest, "dweb.link", true, "/ipns/dnslink.long-name.example.com", "http://dnslink-long--name-example-com.ipns.dweb.link/", nil},
 	} {
-		url, err := toSubdomainURL(test.gwHostname, test.path, test.request, coreAPI)
+		url, err := toSubdomainURL(test.gwHostname, test.path, test.request, test.inlineDNSLink, coreAPI)
 		if url != test.url || !equalError(err, test.err) {
-			t.Errorf("(%s, %s) returned (%s, %v), expected (%s, %v)", test.gwHostname, test.path, url, err, test.url, test.err)
+			t.Errorf("(%s, %v, %s) returned (%s, %v), expected (%s, %v)", test.gwHostname, test.inlineDNSLink, test.path, url, err, test.url, test.err)
 		}
 	}
 }