Skip to content

Commit fdc8994

Browse files
hsanjuanhsanjuan
committed
HTTP Block-retrieval support for rainbow
This adds http retrieval support based on latest boxo. It needs explicit enabling with an "--http-retrieval-enable" flag. Allowlist and number of workers can be configured with additional flags. A provider-ignore flag has been added as well. Example: - http-retrieval-allowlist: [dag.w3s.link] - routing-ignore-providers: [QmQzqxhK82kAmKvARFZSkUVS6fo9sySaiogAnx5EnZ6ZmC] (w3s elastic ipfs peer)
1 parent c858fe4 commit fdc8994

File tree

6 files changed

+117
-16
lines changed

6 files changed

+117
-16
lines changed

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ require (
1010
github.com/felixge/httpsnoop v1.0.4
1111
github.com/ipfs-shipyard/nopfs v0.0.14
1212
github.com/ipfs-shipyard/nopfs/ipfs v0.25.0
13-
github.com/ipfs/boxo v0.27.4
13+
github.com/ipfs/boxo v0.27.3-0.20250210095233-70538fd219ba
1414
github.com/ipfs/go-block-format v0.2.0
1515
github.com/ipfs/go-cid v0.4.1
1616
github.com/ipfs/go-datastore v0.6.0

‎go.sum‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -260,8 +260,36 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd
260260
github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU=
261261
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
262262
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
263-
github.com/ipfs/boxo v0.27.4 h1:6nC8lY5GnR6whAbW88hFz6L13wZUj2vr5BRe3iTvYBI=
264-
github.com/ipfs/boxo v0.27.4/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM=
263+
github.com/ipfs/boxo v0.27.3-0.20250127164459-08419db21bba h1:UjZ4+buzBhiWww0rJ47eQdN8AgBaeRe0h9ZqYsk/Ew8=
264+
github.com/ipfs/boxo v0.27.3-0.20250127164459-08419db21bba/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM=
265+
github.com/ipfs/boxo v0.27.3-0.20250130090023-3bc8ebd6d9b8 h1:G9JtVssDc6WaSmxEwoe4hoH2wUnMqYViMw4sK4pZrcg=
266+
github.com/ipfs/boxo v0.27.3-0.20250130090023-3bc8ebd6d9b8/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM=
267+
github.com/ipfs/boxo v0.27.3-0.20250204114410-7f3b0e1f32a9 h1:eEDINc2P42Bctfr3gOMPl36JMY0my7oCAHAmKxLvPCo=
268+
github.com/ipfs/boxo v0.27.3-0.20250204114410-7f3b0e1f32a9/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM=
269+
github.com/ipfs/boxo v0.27.3-0.20250204134905-aa1f7113c76b h1:yrL/B2ocwJJMUbKMmtZ7PHwv4QKODnrDcxTBvoxRjfw=
270+
github.com/ipfs/boxo v0.27.3-0.20250204134905-aa1f7113c76b/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM=
271+
github.com/ipfs/boxo v0.27.3-0.20250204170555-5a118b05f9b3 h1:ad4it5di9Z6+2+JQHv0seAIz1AKE1BUClTCPHhLMfzE=
272+
github.com/ipfs/boxo v0.27.3-0.20250204170555-5a118b05f9b3/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM=
273+
github.com/ipfs/boxo v0.27.3-0.20250204173852-a925bceda7a0 h1:Vw1kTaT2gars24F4vOlOVK48BzAb+miqH0R82WbzUiE=
274+
github.com/ipfs/boxo v0.27.3-0.20250204173852-a925bceda7a0/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM=
275+
github.com/ipfs/boxo v0.27.3-0.20250204175459-0ad5b9335f7a h1:mwmwDB+v2BRJ0dAFKXJq2XH06/LziHccXj+c+JrIH/Q=
276+
github.com/ipfs/boxo v0.27.3-0.20250204175459-0ad5b9335f7a/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM=
277+
github.com/ipfs/boxo v0.27.3-0.20250204212206-0a68017230f6 h1:DvTDhpGEiCzBkMQMowILxPlly4gZjOZczWHEimcZWAk=
278+
github.com/ipfs/boxo v0.27.3-0.20250204212206-0a68017230f6/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM=
279+
github.com/ipfs/boxo v0.27.3-0.20250205141539-def0f2bd33d5 h1:yzeq9HZ6fL9C1yAHJyet2HFwN7rzkE1WuYE5o5ywGiI=
280+
github.com/ipfs/boxo v0.27.3-0.20250205141539-def0f2bd33d5/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM=
281+
github.com/ipfs/boxo v0.27.3-0.20250205144801-3a75730dbb1b h1:fEUP/QSfKd5S4V8oMljsffTV1PuQqtdnOpnPQHdwSxE=
282+
github.com/ipfs/boxo v0.27.3-0.20250205144801-3a75730dbb1b/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM=
283+
github.com/ipfs/boxo v0.27.3-0.20250206094658-6890482eafce h1:GUTqmwowN0CueuQHYRQs5i3Dx3rZL8SGkm2K5s4EV2M=
284+
github.com/ipfs/boxo v0.27.3-0.20250206094658-6890482eafce/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM=
285+
github.com/ipfs/boxo v0.27.3-0.20250206095634-38fa962d384d h1:XqcbePxVsuGu6dBB+166l7nGD2MAW/MVFltf/9VO0Fw=
286+
github.com/ipfs/boxo v0.27.3-0.20250206095634-38fa962d384d/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM=
287+
github.com/ipfs/boxo v0.27.3-0.20250206223725-67bcdaba19c5 h1:HCbIV6rFALrQzpTLCqZRVTAFJbiQnaqURT+xkyGcmfs=
288+
github.com/ipfs/boxo v0.27.3-0.20250206223725-67bcdaba19c5/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM=
289+
github.com/ipfs/boxo v0.27.3-0.20250206224112-c780690065d0 h1:14Rdj3eTm8+JIRQq3zw2E5iU8/kSInian6DE71foPDE=
290+
github.com/ipfs/boxo v0.27.3-0.20250206224112-c780690065d0/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM=
291+
github.com/ipfs/boxo v0.27.3-0.20250210095233-70538fd219ba h1:Nf4bQytkNbiUvLRWqaBlBG+PwB10EW+mG65Xx8B3sQs=
292+
github.com/ipfs/boxo v0.27.3-0.20250210095233-70538fd219ba/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM=
265293
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
266294
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
267295
github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=

‎handler_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import (
88
"testing"
99
"time"
1010

11-
bsnet "github.com/ipfs/boxo/bitswap/network"
11+
bsnet "github.com/ipfs/boxo/bitswap/network/bsnet"
1212
bsserver "github.com/ipfs/boxo/bitswap/server"
1313
"github.com/ipfs/go-metrics-interface"
1414
"github.com/libp2p/go-libp2p"

‎main.go‎

Lines changed: 57 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -401,6 +401,30 @@ Generate an identity seed and launch a gateway:
401401
EnvVars: []string{"ROUTING_MAX_TIMEOUT"},
402402
Usage: "Maximum time for routing to find the maximum number of providers",
403403
},
404+
&cli.StringSliceFlag{
405+
Name: "routing-ignore-providers",
406+
EnvVars: []string{"ROUTING_IGNORE_PROVIDERS"},
407+
Usage: "Ignore provider records from the given peer IDs",
408+
},
409+
&cli.BoolFlag{
410+
Name: "http-retrieval-enable",
411+
Value: false,
412+
EnvVars: []string{"RAINBOW_HTTP_RETRIEVAL_ENABLE"},
413+
Usage: "Enable HTTP-retrieval of blocks.",
414+
},
415+
&cli.StringSliceFlag{
416+
Name: "http-retrieval-allowlist",
417+
Value: cli.NewStringSlice(),
418+
EnvVars: []string{"RAINBOW_HTTP_RETRIEVAL_ALLOWLIST"},
419+
Usage: "When HTTP retrieval is enabled, allow it only to the given hosts. Empty means 'everyone'",
420+
},
421+
&cli.IntFlag{
422+
Name: "http-retrieval-workers",
423+
Value: 32,
424+
EnvVars: []string{"RAINBOW_HTTP_RETRIEVAL_WORKERS"},
425+
Usage: "Number of workers to use for HTTP retrieval",
426+
},
427+
404428
&cli.StringSliceFlag{
405429
Name: "dnslink-resolvers",
406430
Value: cli.NewStringSlice(extraDNSLinkResolvers...),
@@ -527,6 +551,27 @@ share the same seed as long as the indexes are different.
527551
return err
528552
}
529553

554+
var routingIgnoreProviders []peer.ID
555+
for _, pstr := range cctx.StringSlice("routing-ignore-providers") {
556+
pid, err := peer.Decode(pstr)
557+
if err != nil {
558+
return fmt.Errorf("error parsing peer in routing-ignore-providers: %w", err)
559+
}
560+
routingIgnoreProviders = append(routingIgnoreProviders, pid)
561+
}
562+
563+
routerFilterProtocols := cctx.StringSlice("http-routers-filter-protocols")
564+
httpRetrievalEnable := cctx.Bool("http-retrieval-enable")
565+
httpRetrievalWorkers := cctx.Int("http-retrieval-workers")
566+
httpRetrievalAllowlist := cctx.StringSlice("http-retrieval-allowlist")
567+
if httpRetrievalEnable {
568+
routerFilterProtocols = append(routerFilterProtocols, httpRouterGatewayProtocol)
569+
fmt.Printf("HTTP block-retrievals enabled. Workers: %d. Allowlist set: %t\n",
570+
httpRetrievalWorkers,
571+
len(httpRetrievalAllowlist) == 0,
572+
)
573+
}
574+
530575
cfg := Config{
531576
DataDir: ddir,
532577
BlockstoreType: cctx.String("blockstore"),
@@ -540,7 +585,7 @@ share the same seed as long as the indexes are different.
540585
MaxFD: cctx.Int("libp2p-max-fd"),
541586
InMemBlockCache: cctx.Int64("inmem-block-cache"),
542587
RoutingV1Endpoints: cctx.StringSlice("http-routers"),
543-
RoutingV1FilterProtocols: cctx.StringSlice("http-routers-filter-protocols"),
588+
RoutingV1FilterProtocols: routerFilterProtocols,
544589
DHTRouting: dhtRouting,
545590
DHTSharedHost: cctx.Bool("dht-shared-host"),
546591
Bitswap: bitswap,
@@ -575,9 +620,15 @@ share the same seed as long as the indexes are different.
575620
WALMinSyncInterval: time.Second * time.Duration(cctx.Int("pebble-wal-min-sync-interval-sec")),
576621

577622
// Routing ProviderQueryManager config
578-
RoutingMaxRequests: cctx.Int("routing-max-requests"),
579-
RoutingMaxProviders: cctx.Int("routing-max-providers"),
580-
RoutingMaxTimeout: cctx.Duration("routing-max-timeout"),
623+
RoutingMaxRequests: cctx.Int("routing-max-requests"),
624+
RoutingMaxProviders: cctx.Int("routing-max-providers"),
625+
RoutingMaxTimeout: cctx.Duration("routing-max-timeout"),
626+
RoutingIgnoreProviders: routingIgnoreProviders,
627+
628+
// HTTP Retrieval config
629+
HTTPRetrievalEnable: httpRetrievalEnable,
630+
HTTPRetrievalAllowlist: httpRetrievalAllowlist,
631+
HTTPRetrievalWorkers: httpRetrievalWorkers,
581632
}
582633
var gnd *Node
583634

@@ -791,8 +842,8 @@ func parseCustomDNSLinkResolvers(customDNSResolvers []string) (madns.BasicResolv
791842
return nil, fmt.Errorf("invalid DNS resolver: %s", s)
792843
}
793844
domain := strings.TrimSpace(split[0])
794-
resolverUrl := strings.TrimSpace(split[1])
795-
customDNSResolverMap[domain] = resolverUrl
845+
resolverURL := strings.TrimSpace(split[1])
846+
customDNSResolverMap[domain] = resolverURL
796847
}
797848
dns, err := gateway.NewDNSResolver(customDNSResolverMap)
798849
if err != nil {

‎setup.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ func init() {
5252

5353
const cidContactEndpoint = "https://cid.contact"
5454

55+
const httpRouterGatewayProtocol = "transport-ipfs-gateway-http"
56+
5557
var httpRoutersFilterProtocols = []string{"unknown", "transport-bitswap"} // IPIP-484
5658

5759
var extraDNSLinkResolvers = []string{
@@ -113,6 +115,7 @@ type Config struct {
113115
TrustlessGatewayDomains []string
114116
RoutingV1Endpoints []string
115117
RoutingV1FilterProtocols []string
118+
RoutingIgnoreProviders []peer.ID
116119
DHTRouting DHTRouting
117120
DHTSharedHost bool
118121
IpnsMaxCacheTTL time.Duration
@@ -163,6 +166,11 @@ type Config struct {
163166
RoutingMaxRequests int
164167
RoutingMaxProviders int
165168
RoutingMaxTimeout time.Duration
169+
170+
// HTTP Retrieval configuration
171+
HTTPRetrievalEnable bool
172+
HTTPRetrievalAllowlist []string
173+
HTTPRetrievalWorkers int
166174
}
167175

168176
func SetupNoLibp2p(ctx context.Context, cfg Config, dnsCache *cachedDNS) (*Node, error) {

‎setup_bitswap.go‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,9 @@ import (
88

99
"github.com/ipfs/boxo/bitswap"
1010
bsclient "github.com/ipfs/boxo/bitswap/client"
11-
bsnet "github.com/ipfs/boxo/bitswap/network"
11+
"github.com/ipfs/boxo/bitswap/network"
12+
bsnet "github.com/ipfs/boxo/bitswap/network/bsnet"
13+
"github.com/ipfs/boxo/bitswap/network/httpnet"
1214
bsserver "github.com/ipfs/boxo/bitswap/server"
1315
"github.com/ipfs/boxo/blockstore"
1416
"github.com/ipfs/boxo/exchange"
@@ -24,14 +26,26 @@ import (
2426
func setupBitswapExchange(ctx context.Context, cfg Config, h host.Host, cr routing.ContentRouting, bstore blockstore.Blockstore) exchange.Interface {
2527
bsctx := metri.CtxScope(ctx, "ipfs_bitswap")
2628

29+
var exnet network.BitSwapNetwork
2730
bn := bsnet.NewFromIpfsHost(h)
2831

32+
if cfg.HTTPRetrievalEnable {
33+
htnet := httpnet.New(h,
34+
httpnet.WithHTTPWorkers(cfg.HTTPRetrievalWorkers),
35+
httpnet.WithAllowlist(cfg.HTTPRetrievalAllowlist),
36+
)
37+
exnet = network.New(h.Peerstore(), bn, htnet)
38+
} else {
39+
exnet = bn
40+
}
41+
2942
// Custom query manager with the content router and the host
3043
// and our custom options to overwrite the default.
31-
pqm, err := providerquerymanager.New(h, cr,
44+
pqm, err := providerquerymanager.New(exnet, cr,
3245
providerquerymanager.WithMaxInProcessRequests(cfg.RoutingMaxRequests),
3346
providerquerymanager.WithMaxProviders(cfg.RoutingMaxProviders),
3447
providerquerymanager.WithMaxTimeout(cfg.RoutingMaxTimeout),
48+
providerquerymanager.WithIgnoreProviders(cfg.RoutingIgnoreProviders...),
3549
)
3650
if err != nil {
3751
panic(err)
@@ -88,14 +102,14 @@ func setupBitswapExchange(ctx context.Context, cfg Config, h host.Host, cr routi
88102
)
89103

90104
// Initialize client+server
91-
bswap := bitswap.New(bsctx, bn, pqm, bstore, opts...)
92-
bn.Start(bswap)
105+
bswap := bitswap.New(bsctx, exnet, pqm, bstore, opts...)
106+
exnet.Start(bswap)
93107
return &noNotifyExchange{bswap}
94108
}
95109

96110
// By default, rainbow runs with bitswap client alone
97-
bswap := bsclient.New(bsctx, bn, pqm, bstore, clientOpts...)
98-
bn.Start(bswap)
111+
bswap := bsclient.New(bsctx, exnet, pqm, bstore, clientOpts...)
112+
exnet.Start(bswap)
99113
return bswap
100114
}
101115

0 commit comments

Comments
 (0)