feat(gateway)!: deserialised responses turned off by default (#252)

feat(gateway)!: new trustless mode, and by default

Author: hacdias

examples/gateway/common/handler.go

@@ -10,34 +10,48 @@ import (
 )
 
 func NewHandler(gwAPI gateway.IPFSBackend) http.Handler {
-	// Initialize the headers and gateway configuration. For this example, we do
-	// not add any special headers, but the required ones.
-	headers := map[string][]string{}
-	gateway.AddAccessControlHeaders(headers)
 	conf := gateway.Config{
-		Headers: headers,
-	}
+		// Initialize the headers. For this example, we do not add any special headers,
+		// only the required ones via gateway.AddAccessControlHeaders.
+		Headers: map[string][]string{},
 
-	// Initialize the public gateways that we will want to have available through
-	// Host header rewriting. This step is optional and only required if you're
-	// running multiple public gateways and want different settings and support
-	// for DNSLink and Subdomain Gateways.
-	noDNSLink := false // If you set DNSLink to point at the CID from CAR, you can load it!
-	publicGateways := map[string]*gateway.Specification{
-		// Support public requests with Host: CID.ipfs.example.net and ID.ipns.example.net
-		"example.net": {
-			Paths:         []string{"/ipfs", "/ipns"},
-			NoDNSLink:     noDNSLink,
-			UseSubdomains: true,
-		},
-		// Support local requests
-		"localhost": {
-			Paths:         []string{"/ipfs", "/ipns"},
-			NoDNSLink:     noDNSLink,
-			UseSubdomains: true,
+		// If you set DNSLink to point at the CID from CAR, you can load it!
+		NoDNSLink: false,
+
+		// For these examples we have the trusted mode enabled by default. That is,
+		// all types of requests will be accepted. By default, only Trustless Gateway
+		// requests work: https://specs.ipfs.tech/http-gateways/trustless-gateway/
+		DeserializedResponses: true,
+
+		// Initialize the public gateways that we will want to have available
+		// through Host header rewriting. This step is optional, but required
+		// if you're running multiple public gateways on different hostnames
+		// and want different settings such as support for Deserialized
+		// Responses on localhost, or DNSLink and Subdomain Gateways.
+		PublicGateways: map[string]*gateway.Specification{
+			// Support public requests with Host: CID.ipfs.example.net and ID.ipns.example.net
+			"example.net": {
+				Paths:         []string{"/ipfs", "/ipns"},
+				NoDNSLink:     false,
+				UseSubdomains: true,
+				// This subdomain gateway is used for testing and therefore we make non-trustless requests.
+				DeserializedResponses: true,
+			},
+			// Support local requests
+			"localhost": {
+				Paths:         []string{"/ipfs", "/ipns"},
+				NoDNSLink:     false,
+				UseSubdomains: true,
+				// Localhost is considered trusted, ok to allow deserialized responses
+				// as long it is not exposed to the internet.
+				DeserializedResponses: true,
+			},
 		},
 	}
 
+	// Add required access control headers to the configuration.
+	gateway.AddAccessControlHeaders(conf.Headers)
+
 	// Creates a mux to serve the gateway paths. This is not strictly necessary
 	// and gwHandler could be used directly. However, on the next step we also want
 	// to add prometheus metrics, hence needing the mux.
@@ -57,7 +71,7 @@ func NewHandler(gwAPI gateway.IPFSBackend) http.Handler {
 	// or example.net. If you want to expose the metrics on such gateways,
 	// you will have to add the path "/debug" to the variable Paths.
 	var handler http.Handler
-	handler = gateway.WithHostname(mux, gwAPI, publicGateways, noDNSLink)
+	handler = gateway.WithHostname(conf, gwAPI, mux)
 
 	// Then, wrap with the withConnect middleware. This is required since we use
 	// http.ServeMux which does not support CONNECT by default.

gateway/gateway.go

@@ -15,7 +15,69 @@ import (
 
 // Config is the configuration used when creating a new gateway handler.
 type Config struct {
+	// Headers is a map containing all the headers that should be sent by default
+	// in all requests. You can define custom headers, as well as add the recommended
+	// headers via AddAccessControlHeaders.
 	Headers map[string][]string
+
+	// DeserializedResponses configures this gateway to support returning data
+	// in deserialized format. By default, the gateway will only support
+	// trustless, verifiable [application/vnd.ipld.raw] and
+	// [application/vnd.ipld.car] responses, operating as a [Trustless Gateway].
+	//
+	// This global flag can be overridden per FQDN in PublicGateways map.
+	//
+	// [application/vnd.ipld.raw]: https://www.iana.org/assignments/media-types/application/vnd.ipld.raw
+	// [application/vnd.ipld.car]: https://www.iana.org/assignments/media-types/application/vnd.ipld.car
+	// [Trustless Gateway]: https://specs.ipfs.tech/http-gateways/trustless-gateway/
+	DeserializedResponses bool
+
+	// NoDNSLink configures the gateway to _not_ perform DNS TXT record lookups in
+	// response to requests with values in `Host` HTTP header. This flag can be
+	// overridden per FQDN in PublicGateways. To be used with WithHostname.
+	NoDNSLink bool
+
+	// PublicGateways configures the behavior of known public gateways. Each key is
+	// a fully qualified domain name (FQDN). To be used with WithHostname.
+	PublicGateways map[string]*Specification
+}
+
+// Specification is the specification of an IPFS Public Gateway.
+type Specification struct {
+	// Paths is explicit list of path prefixes that should be handled by
+	// this gateway. Example: `["/ipfs", "/ipns"]`
+	// Useful if you only want to support immutable `/ipfs`.
+	Paths []string
+
+	// UseSubdomains indicates whether or not this is a [Subdomain Gateway].
+	//
+	// If this flag is set, any `/ipns/$id` and/or `/ipfs/$id` paths in Paths
+	// will be permanently redirected to `http(s)://$id.[ipns|ipfs].$gateway/`.
+	//
+	// We do not support using both paths and subdomains for a single domain
+	// for security reasons ([Origin isolation]).
+	//
+	// [Subdomain Gateway]: https://specs.ipfs.tech/http-gateways/subdomain-gateway/
+	// [Origin isolation]: https://en.wikipedia.org/wiki/Same-origin_policy
+	UseSubdomains bool
+
+	// NoDNSLink configures this gateway to _not_ resolve DNSLink for the
+	// specific FQDN provided in `Host` HTTP header. Useful when you want to
+	// explicitly allow or refuse hosting a single hostname. To refuse all
+	// DNSLinks in `Host` processing, set NoDNSLink in Config instead. This setting
+	// overrides the global setting.
+	NoDNSLink bool
+
+	// InlineDNSLink configures this gateway to always inline DNSLink names
+	// (FQDN) into a single DNS label in order to interop with wildcard TLS certs
+	// and Origin per CID isolation provided by rules like https://publicsuffix.org
+	//
+	// This should be set to true if you use HTTPS.
+	InlineDNSLink bool
+
+	// DeserializedResponses configures this gateway to support returning data
+	// in deserialized format. This setting overrides the global setting.
+	DeserializedResponses bool
 }
 
 // TODO: Is this what we want for ImmutablePath?
@@ -221,7 +283,17 @@ func AddAccessControlHeaders(headers map[string][]string) {
 type RequestContextKey string
 
 const (
-	DNSLinkHostnameKey RequestContextKey = "dnslink-hostname"
+	// GatewayHostnameKey is the key for the hostname at which the gateway is
+	// operating. It may be a DNSLink, Subdomain or Regular gateway.
 	GatewayHostnameKey RequestContextKey = "gw-hostname"
-	ContentPathKey     RequestContextKey = "content-path"
+
+	// DNSLinkHostnameKey is the key for the hostname of a DNSLink Gateway:
+	// https://specs.ipfs.tech/http-gateways/dnslink-gateway/
+	DNSLinkHostnameKey RequestContextKey = "dnslink-hostname"
+
+	// SubdomainHostnameKey is the key for the hostname of a Subdomain Gateway:
+	// https://specs.ipfs.tech/http-gateways/subdomain-gateway/
+	SubdomainHostnameKey RequestContextKey = "subdomain-hostname"
+
+	ContentPathKey RequestContextKey = "content-path"
 )

gateway/gateway_test.go

@@ -198,14 +198,20 @@ func newTestServerAndNode(t *testing.T, ns mockNamesys) (*httptest.Server, *mock
 }
 
 func newTestServer(t *testing.T, api IPFSBackend) *httptest.Server {
-	config := Config{Headers: map[string][]string{}}
+	return newTestServerWithConfig(t, api, Config{
+		Headers:               map[string][]string{},
+		DeserializedResponses: true,
+	})
+}
+
+func newTestServerWithConfig(t *testing.T, api IPFSBackend, config Config) *httptest.Server {
 	AddAccessControlHeaders(config.Headers)
 
 	handler := NewHandler(config, api)
 	mux := http.NewServeMux()
 	mux.Handle("/ipfs/", handler)
 	mux.Handle("/ipns/", handler)
-	handler = WithHostname(mux, api, map[string]*Specification{}, false)
+	handler = WithHostname(config, api, mux)
 
 	ts := httptest.NewServer(handler)
 	t.Cleanup(func() { ts.Close() })
@@ -573,3 +579,128 @@ func TestIpnsBase58MultihashRedirect(t *testing.T) {
 		assert.Equal(t, "/ipns/k2k4r8ol4m8kkcqz509c1rcjwunebj02gcnm5excpx842u736nja8ger?keep=query", res.Header.Get("Location"))
 	})
 }
+
+func TestIpfsTrustlessMode(t *testing.T) {
+	api, root := newMockAPI(t)
+
+	ts := newTestServerWithConfig(t, api, Config{
+		Headers:   map[string][]string{},
+		NoDNSLink: false,
+		PublicGateways: map[string]*Specification{
+			"trustless.com": {
+				Paths: []string{"/ipfs", "/ipns"},
+			},
+			"trusted.com": {
+				Paths:                 []string{"/ipfs", "/ipns"},
+				DeserializedResponses: true,
+			},
+		},
+	})
+	t.Logf("test server url: %s", ts.URL)
+
+	trustedFormats := []string{"", "dag-json", "dag-cbor", "tar", "json", "cbor"}
+	trustlessFormats := []string{"raw", "car"}
+
+	doRequest := func(t *testing.T, path, host string, expectedStatus int) {
+		req, err := http.NewRequest(http.MethodGet, ts.URL+path, nil)
+		assert.Nil(t, err)
+
+		if host != "" {
+			req.Host = host
+		}
+
+		res, err := doWithoutRedirect(req)
+		assert.Nil(t, err)
+		defer res.Body.Close()
+		assert.Equal(t, expectedStatus, res.StatusCode)
+	}
+
+	doIpfsCidRequests := func(t *testing.T, formats []string, host string, expectedStatus int) {
+		for _, format := range formats {
+			doRequest(t, "/ipfs/"+root.String()+"/?format="+format, host, expectedStatus)
+		}
+	}
+
+	doIpfsCidPathRequests := func(t *testing.T, formats []string, host string, expectedStatus int) {
+		for _, format := range formats {
+			doRequest(t, "/ipfs/"+root.String()+"/EmptyDir/?format="+format, host, expectedStatus)
+		}
+	}
+
+	trustedTests := func(t *testing.T, host string) {
+		doIpfsCidRequests(t, trustlessFormats, host, http.StatusOK)
+		doIpfsCidRequests(t, trustedFormats, host, http.StatusOK)
+		doIpfsCidPathRequests(t, trustlessFormats, host, http.StatusOK)
+		doIpfsCidPathRequests(t, trustedFormats, host, http.StatusOK)
+	}
+
+	trustlessTests := func(t *testing.T, host string) {
+		doIpfsCidRequests(t, trustlessFormats, host, http.StatusOK)
+		doIpfsCidRequests(t, trustedFormats, host, http.StatusNotAcceptable)
+		doIpfsCidPathRequests(t, trustlessFormats, host, http.StatusNotAcceptable)
+		doIpfsCidPathRequests(t, trustedFormats, host, http.StatusNotAcceptable)
+	}
+
+	t.Run("Explicit Trustless Gateway", func(t *testing.T) {
+		t.Parallel()
+		trustlessTests(t, "trustless.com")
+	})
+
+	t.Run("Explicit Trusted Gateway", func(t *testing.T) {
+		t.Parallel()
+		trustedTests(t, "trusted.com")
+	})
+
+	t.Run("Implicit Default Trustless Gateway", func(t *testing.T) {
+		t.Parallel()
+		trustlessTests(t, "not.configured.com")
+		trustlessTests(t, "localhost")
+		trustlessTests(t, "127.0.0.1")
+		trustlessTests(t, "::1")
+	})
+}
+
+func TestIpnsTrustlessMode(t *testing.T) {
+	api, root := newMockAPI(t)
+	api.namesys["/ipns/trustless.com"] = path.FromCid(root)
+	api.namesys["/ipns/trusted.com"] = path.FromCid(root)
+
+	ts := newTestServerWithConfig(t, api, Config{
+		Headers:   map[string][]string{},
+		NoDNSLink: false,
+		PublicGateways: map[string]*Specification{
+			"trustless.com": {
+				Paths: []string{"/ipfs", "/ipns"},
+			},
+			"trusted.com": {
+				Paths:                 []string{"/ipfs", "/ipns"},
+				DeserializedResponses: true,
+			},
+		},
+	})
+	t.Logf("test server url: %s", ts.URL)
+
+	doRequest := func(t *testing.T, path, host string, expectedStatus int) {
+		req, err := http.NewRequest(http.MethodGet, ts.URL+path, nil)
+		assert.Nil(t, err)
+
+		if host != "" {
+			req.Host = host
+		}
+
+		res, err := doWithoutRedirect(req)
+		assert.Nil(t, err)
+		defer res.Body.Close()
+		assert.Equal(t, expectedStatus, res.StatusCode)
+	}
+
+	// DNSLink only. Not supported for trustless. Supported for trusted, except
+	// format=ipns-record which is unavailable for DNSLink.
+	doRequest(t, "/", "trustless.com", http.StatusNotAcceptable)
+	doRequest(t, "/EmptyDir/", "trustless.com", http.StatusNotAcceptable)
+	doRequest(t, "/?format=ipns-record", "trustless.com", http.StatusNotAcceptable)
+
+	doRequest(t, "/", "trusted.com", http.StatusOK)
+	doRequest(t, "/EmptyDir/", "trusted.com", http.StatusOK)
+	doRequest(t, "/?format=ipns-record", "trusted.com", http.StatusBadRequest)
+}