feat: enable using an HTTP block provider as a routing backend (#110)

* feat: enable using an HTTP block provider as a routing backend

* clarify http paths are not supported

Co-authored-by: Marcin Rataj <[email protected]>

* clarify only https and not http is allowed

Co-authored-by: Marcin Rataj <[email protected]>

* add changelog for http block providers flags

* feat: switch accelerated-dht bool option to dht string option to allow disabling the dht entirely

* fix: set Accept on requests

* fix: 5s timeout for HEAD checks

- finish sooner rather than waiting 30s (DefaultRoutingTimeout)
- dont return unactionable errors to end user, everything other than
  HTTP 200 should produce empty /routing/v1/providers result set

* test: TestHTTPBlockRouter

* docs: environment-variables.md

---------

Co-authored-by: Marcin Rataj <[email protected]>

Author: aschmahmann

CHANGELOG.md

@@ -15,8 +15,13 @@ The following emojis are used to highlight certain changes:
 
 ### Added
 
+- Added `http-block-provider-endpoints` and `http-block-provider-peerids` options to enable using a [trustless HTTP gateway](https://specs.ipfs.tech/http-gateways/trustless-gateway/) as a source for synthetic content routing records.
+  - When the configured gateway responds with HTTP 200 to an HTTP HEAD request for a block (`HEAD /ipfs/{cid}?format=raw`), `FindProviders` returns a provider record containing a predefined PeerID and the HTTP gateway as a multiaddr with `/tls/http` suffix.
+
 ### Changed
 
+- `accelerated-dht` option was removed and replaced with a `dht` option which enables toggling between the standard client, accelerated client and being disabled
+
 ### Removed
 
 ### Fixed

docs/environment-variables.md

@@ -4,10 +4,15 @@
 
 - [Configuration](#configuration)
   - [`SOMEGUY_LISTEN_ADDRESS`](#someguy_listen_address)
-  - [`SOMEGUY_ACCELERATED_DHT`](#someguy_accelerated_dht)
+  - [`SOMEGUY_DHT`](#someguy_dht)
+  - [`SOMEGUY_CACHED_ADDR_BOOK`](#someguy_cached_addr_book)
+  - [`SOMEGUY_CACHED_ADDR_BOOK_RECENT_TTL`](#someguy_cached_addr_book_recent_ttl)
+  - [`SOMEGUY_CACHED_ADDR_BOOK_ACTIVE_PROBING`](#someguy_cached_addr_book_active_probing)
   - [`SOMEGUY_PROVIDER_ENDPOINTS`](#someguy_provider_endpoints)
   - [`SOMEGUY_PEER_ENDPOINTS`](#someguy_peer_endpoints)
   - [`SOMEGUY_IPNS_ENDPOINTS`](#someguy_ipns_endpoints)
+  - [`SOMEGUY_HTTP_BLOCK_PROVIDER_ENDPOINTS`](#someguy_http_block_provider_endpoints)
+  - [`SOMEGUY_HTTP_BLOCK_PROVIDER_PEERIDS`](#someguy_http_block_provider_peerids)
   - [`SOMEGUY_LIBP2P_LISTEN_ADDRS`](#someguy_libp2p_listen_addrs)
   - [`SOMEGUY_LIBP2P_CONNMGR_LOW`](#someguy_libp2p_connmgr_low)
   - [`SOMEGUY_LIBP2P_CONNMGR_HIGH`](#someguy_libp2p_connmgr_high)
@@ -20,8 +25,8 @@
   - [`GOLOG_FILE`](#golog_file)
   - [`GOLOG_TRACING_FILE`](#golog_tracing_file)
 - [Tracing](#tracing)
-  - [`SOMEGUY_SAMPLING_FRACTION`](#someguy_sampling_fraction)
   - [`SOMEGUY_TRACING_AUTH`](#someguy_tracing_auth)
+  - [`SOMEGUY_SAMPLING_FRACTION`](#someguy_sampling_fraction)
 
 ## Configuration
 
@@ -31,11 +36,11 @@ The address to listen on.
 
 Default: `127.0.0.1:8190`
 
-### `SOMEGUY_ACCELERATED_DHT`
+### `SOMEGUY_DHT`
 
-Whether or not the Accelerated DHT is enabled or not.
+Controls DHT client mode: `standard`, `accelerated`, `disabled`
 
-Default: `true`
+Default: `accelerated`
 
 ### `SOMEGUY_CACHED_ADDR_BOOK`
 
@@ -73,6 +78,20 @@ Comma-separated list of other Delegated Routing V1 endpoints to proxy IPNS reque
 
 Default: none
 
+### `SOMEGUY_HTTP_BLOCK_PROVIDER_ENDPOINTS`
+
+Comma-separated list of [HTTP trustless gateway](https://specs.ipfs.tech/http-gateways/trustless-gateway/) for probing and generating synthetic provider records.
+
+When the configured gateway responds with HTTP 200 to an HTTP HEAD request for a block (`HEAD /ipfs/{cid}?format=raw`), `FindProviders` returns a provider record containing a PeerID from `SOMEGUY_HTTP_BLOCK_PROVIDER_PEERIDS` and the HTTP gateway endpoint as a multiaddr with `/tls/http` suffix.
+
+Default: none
+
+### `SOMEGUY_HTTP_BLOCK_PROVIDER_PEERIDS`
+
+Comma-separated list of [multibase-encoded peerIDs](https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#string-representation) to use in synthetic provider records returned for HTTP providers in `SOMEGUY_HTTP_BLOCK_PROVIDER_ENDPOINTS`.
+
+Default: none
+
 ### `SOMEGUY_LIBP2P_LISTEN_ADDRS`
 
 Multiaddresses for libp2p host to listen on (comma-separated).

http_block_router.go

@@ -0,0 +1,157 @@
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	drclient "github.com/ipfs/boxo/routing/http/client"
+	"github.com/ipfs/boxo/routing/http/types"
+	"github.com/ipfs/boxo/routing/http/types/iter"
+	"github.com/ipfs/go-cid"
+	"github.com/libp2p/go-libp2p/core/peer"
+	"github.com/multiformats/go-multiaddr"
+)
+
+type httpBlockRouter struct {
+	endpoint   string
+	endpointMa multiaddr.Multiaddr
+	peerID     peer.ID
+	httpClient *http.Client
+}
+
+const httpBlockRouterTimeout = 5 * time.Second
+
+// newHTTPBlockRouter returns a router backed by a trustless HTTP gateway
+// (https://specs.ipfs.tech/http-gateways/trustless-gateway/) at the specified
+// endpoint. If gateway responds to HTTP 200 to HTTP HEAD request, the
+// FindProviders returns a provider record with predefined peerID and gateway
+// URL represented as multiaddr  with /tls/http suffic.
+func newHTTPBlockRouter(endpoint string, p peer.ID, client *http.Client) (httpBlockRouter, error) {
+	if client == nil {
+		client = defaultHTTPBlockRouterClient(false)
+	}
+	if client.Timeout == 0 {
+		client.Timeout = httpBlockRouterTimeout
+	}
+
+	u, err := url.Parse(endpoint)
+	if err != nil {
+		return httpBlockRouter{}, fmt.Errorf("failed to parse endpoint %s: %w", endpoint, err)
+	}
+	if u.Scheme != "http" && u.Scheme != "https" {
+		return httpBlockRouter{}, fmt.Errorf("unsupported scheme %s, only http and https are supported", u.Scheme)
+	}
+
+	h := u.Hostname()
+	ip := net.ParseIP(h)
+	var hostComponent string
+	if ip == nil {
+		hostComponent = "dns"
+	} else if strings.Contains(h, ":") {
+		hostComponent = "ip6"
+	} else {
+		hostComponent = "ip4"
+	}
+
+	var port int
+	if u.Port() != "" {
+		if p, err := strconv.Atoi(u.Port()); err != nil {
+			return httpBlockRouter{}, fmt.Errorf("invalid port %s: %w", u.Port(), err)
+		} else {
+			port = p
+		}
+	} else {
+		if u.Scheme == "https" {
+			port = 443
+		} else {
+			port = 80
+		}
+	}
+
+	var tlsComponent string
+	if u.Scheme == "https" {
+		tlsComponent = "/tls"
+	} else if os.Getenv("DEBUG") == "true" {
+		// allow unencrypted HTTP for local debugging
+		tlsComponent = ""
+	} else {
+		return httpBlockRouter{}, fmt.Errorf("failed to parse endpoint %s: only HTTPS providers are allowed (unencrypted HTTP can't be used in web browsers)", endpoint)
+
+	}
+
+	var httpPathComponent string
+	if escPath := u.EscapedPath(); escPath != "" && escPath != "/" {
+		return httpBlockRouter{}, fmt.Errorf("failed to parse endpoint %s: only URLs without path are supported", endpoint)
+	}
+
+	endpointMaStr := fmt.Sprintf("/%s/%s/tcp/%d%s/http%s", hostComponent, h, port, tlsComponent, httpPathComponent)
+
+	ma, err := multiaddr.NewMultiaddr(endpointMaStr)
+	if err != nil {
+		return httpBlockRouter{}, fmt.Errorf("failed to parse endpoint %s: %w", endpoint, err)
+	}
+	return httpBlockRouter{
+		endpoint:   endpoint,
+		endpointMa: ma,
+		peerID:     p,
+		httpClient: client,
+	}, nil
+}
+
+func defaultHTTPBlockRouterClient(insecureSkipVerify bool) *http.Client {
+	transport := http.DefaultTransport
+	if insecureSkipVerify {
+		transport = &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true, // Disable TLS cert validation for tests
+			},
+		}
+	}
+	return &http.Client{
+		Timeout: httpBlockRouterTimeout, // timeout hanging HTTP HEAD sooner than boxo/routing/http/server.DefaultRoutingTimeout
+		Transport: &drclient.ResponseBodyLimitedTransport{
+			RoundTripper: transport,
+			LimitBytes:   1 << 12, // max 4KiB -- should be plenty for HEAD response
+			UserAgent:    "someguy/" + buildVersion(),
+		},
+	}
+}
+
+func (h httpBlockRouter) FindProviders(ctx context.Context, c cid.Cid, limit int) (iter.ResultIter[types.Record], error) {
+	req, err := http.NewRequestWithContext(ctx, "HEAD", fmt.Sprintf("%s/ipfs/%s?format=raw", h.endpoint, c), nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Accept", "application/vnd.ipld.raw")
+	httpClient := h.httpClient
+	if httpClient == nil {
+		httpClient = http.DefaultClient
+	}
+
+	resp, err := httpClient.Do(req)
+	if err == nil && resp.StatusCode == http.StatusOK {
+		return iter.ToResultIter(iter.FromSlice([]types.Record{
+			&types.PeerRecord{
+				Schema: types.SchemaPeer,
+				ID:     &h.peerID,
+				Addrs: []types.Multiaddr{
+					{Multiaddr: h.endpointMa},
+				},
+				Protocols: []string{"transport-ipfs-gateway-http"},
+				Extra:     nil,
+			},
+		})), nil
+	}
+	// everything that is not HTTP 200, including errors, produces empty response
+	return iter.ToResultIter(iter.FromSlice([]types.Record{})), nil
+}
+
+var _ providersRouter = (*httpBlockRouter)(nil)

http_block_router_test.go

@@ -0,0 +1,133 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/ipfs/boxo/routing/http/types"
+	"github.com/ipfs/boxo/routing/http/types/iter"
+	"github.com/ipfs/go-cid"
+	"github.com/libp2p/go-libp2p/core/peer"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHTTPBlockRouter(t *testing.T) {
+	t.Parallel()
+	debug := os.Getenv("DEBUG") == "true"
+
+	t.Run("FindProviders", func(t *testing.T) {
+		ctx := context.Background()
+		// Set up mock HTTP Provider (trustless gateway) that returns HTTP 200 for specific CID
+		testData := "Thu  8 May 01:07:03 CEST 2025"
+		testCid := cid.MustParse("bafkreie5zycmytdhd5bl4f5jqsayyiwshugf57d4hkd7eif3toh23fsy3i")
+		httpBlockGateway := newMockTrustlessGateway(testCid, testData, debug)
+		t.Cleanup(func() { httpBlockGateway.Close() })
+
+		// Test args
+		endpoint := httpBlockGateway.URL
+		peerId, _ := peer.Decode("12D3KooWCjfPiojcCUmv78Wd1NJzi4Mraj1moxigp7AfQVQvGLwH")
+		insecureSkipVerify := true
+		client := defaultHTTPBlockRouterClient(insecureSkipVerify)
+		httpHost, httpPort, err := splitHostPort(endpoint)
+		assert.NoError(t, err)
+		expectedAddr := fmt.Sprintf("/ip4/%s/tcp/%s/tls/http", httpHost, httpPort)
+
+		// Create Router
+		httpBlockRouter, err := newHTTPBlockRouter(endpoint, peerId, client)
+		assert.NoError(t, err)
+
+		t.Run("return gateway as HTTP provider if HTTP HEAD check returned HTTP 200", func(t *testing.T) {
+			t.Parallel()
+
+			// Ask Router for CID present on trustless gateway
+			it, err := httpBlockRouter.FindProviders(ctx, testCid, 10)
+			require.NoError(t, err)
+
+			results, err := iter.ReadAllResults(it)
+			require.NoError(t, err)
+			require.Len(t, results, 1)
+
+			// Verify returned provider points at http gateway URL
+			peerRecord := results[0].(*types.PeerRecord)
+			require.Equal(t, peerId, *peerRecord.ID)
+			require.Len(t, peerRecord.Addrs, 1)
+			assert.NoError(t, err)
+			require.Equal(t, expectedAddr, peerRecord.Addrs[0].String())
+		})
+
+		t.Run("return no results if HTTP HEAD check returned HTTP 404", func(t *testing.T) {
+			t.Parallel()
+
+			// This CID has no providers
+			failCid := cid.MustParse("bafkreie5keu4z5kgutjds5tz3ahdxhcdkn4hl2vr7snenml44ui7y4yfki")
+
+			// Ask Router for CID present on trustless gateway
+			it, err := httpBlockRouter.FindProviders(ctx, failCid, 10)
+			require.NoError(t, err)
+
+			results, err := iter.ReadAllResults(it)
+			require.NoError(t, err)
+			require.Len(t, results, 0)
+		})
+
+	})
+}
+
+// newMockTrustlessGateway pretends to be http provider that supports
+// block response https://specs.ipfs.tech/http-gateways/trustless-gateway/#block-responses-application-vnd-ipld-raw
+func newMockTrustlessGateway(c cid.Cid, body string, debug bool) *httptest.Server {
+	expectedPathPrefix := "/ipfs/" + c.String()
+	handler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		if debug {
+			fmt.Printf("mockTrustlessGateway %s %s\n", req.Method, req.URL.Path)
+		}
+		if strings.HasPrefix(req.URL.Path, expectedPathPrefix) {
+			w.Header().Set("Content-Type", "application/vnd.ipld.raw")
+			w.WriteHeader(http.StatusOK)
+			if req.Method == "GET" {
+				_, err := w.Write([]byte(body))
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "mockTrustlessGateway %s %s error: %v\n", req.Method, req.URL.Path, err)
+				}
+			}
+			return
+		} else if strings.HasPrefix(req.URL.Path, "/ipfs/bafkqaaa") {
+			// This is probe from https://specs.ipfs.tech/http-gateways/trustless-gateway/#dedicated-probe-paths
+			w.Header().Set("Content-Type", "application/vnd.ipld.raw")
+			w.WriteHeader(http.StatusOK)
+			return
+		} else {
+			http.Error(w, "Not Found", http.StatusNotFound)
+			return
+		}
+	})
+
+	// Make it HTTP/2 with self-signed TLS cert
+	srv := httptest.NewUnstartedServer(handler)
+	srv.EnableHTTP2 = true
+	srv.StartTLS()
+	return srv
+}
+
+func splitHostPort(httpUrl string) (ipAddr string, port string, err error) {
+	u, err := url.Parse(httpUrl)
+	if err != nil {
+		return "", "", err
+	}
+	if u.Scheme == "" || u.Host == "" {
+		return "", "", fmt.Errorf("invalid URL format: missing scheme or host")
+	}
+	ipAddr, port, err = net.SplitHostPort(u.Host)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to split host and port from %q: %w", u.Host, err)
+	}
+	return ipAddr, port, nil
+}