Merge 'origin/master' into feat/10751-dag-width

Author: lidel

config/internal.go

@@ -14,5 +14,6 @@ type InternalBitswap struct {
 	EngineTaskWorkerCount       OptionalInteger
 	MaxOutstandingBytesPerPeer  OptionalInteger
 	ProviderSearchDelay         OptionalDuration
+	ProviderSearchMaxResults    OptionalInteger
 	WantHaveReplaceSize         OptionalInteger
 }

core/corehttp/webui.go

@@ -1,11 +1,12 @@
 package corehttp
 
 // WebUI version confirmed to work with this Kubo version
-const WebUIPath = "/ipfs/bafybeibpaa5kqrj4gkemiswbwndjqiryl65cks64ypwtyerxixu56gnvvm" // v4.6.0
+const WebUIPath = "/ipfs/bafybeibfd5kbebqqruouji6ct5qku3tay273g7mt24mmrfzrsfeewaal5y" // v4.7.0
 
 // WebUIPaths is a list of all past webUI paths.
 var WebUIPaths = []string{
 	WebUIPath,
+	"/ipfs/bafybeibpaa5kqrj4gkemiswbwndjqiryl65cks64ypwtyerxixu56gnvvm", // v4.6.0
 	"/ipfs/bafybeiata4qg7xjtwgor6r5dw63jjxyouenyromrrb4lrewxrlvav7gzgi", // v4.5.0
 	"/ipfs/bafybeigp3zm7cqoiciqk5anlheenqjsgovp7j7zq6hah4nu6iugdgb4nby", // v4.4.2
 	"/ipfs/bafybeiatztgdllxnp5p6zu7bdwhjmozsmd7jprff4bdjqjljxtylitvss4", // v4.4.1

core/node/bitswap.go

@@ -28,6 +28,7 @@ const (
 	DefaultEngineTaskWorkerCount       = 8
 	DefaultMaxOutstandingBytesPerPeer  = 1 << 20
 	DefaultProviderSearchDelay         = 1000 * time.Millisecond
+	DefaultMaxProviders                = 10 // matching BitswapClientDefaultMaxProviders from https://github.com/ipfs/boxo/blob/v0.29.1/bitswap/internal/defaults/defaults.go#L15
 	DefaultWantHaveReplaceSize         = 1024
 )
 
@@ -79,11 +80,13 @@ func Bitswap(provide bool) interface{} {
 
 		var provider routing.ContentDiscovery
 		if provide {
-			// We need to hardcode the default because it is an
-			// internal setting in boxo.
+			var maxProviders int = DefaultMaxProviders
+			if in.Cfg.Internal.Bitswap != nil {
+				maxProviders = int(in.Cfg.Internal.Bitswap.ProviderSearchMaxResults.WithDefault(DefaultMaxProviders))
+			}
 			pqm, err := rpqm.New(bitswapNetwork,
 				in.Rt,
-				rpqm.WithMaxProviders(10),
+				rpqm.WithMaxProviders(maxProviders),
 				rpqm.WithIgnoreProviders(in.Cfg.Routing.IgnoreProviders...),
 			)
 			if err != nil {

core/node/core.go

@@ -115,6 +115,7 @@ func FetcherConfig(bs blockservice.BlockService) FetchersOut {
 	// path resolution should not fetch new blocks via exchange.
 	offlineBs := blockservice.New(bs.Blockstore(), offline.Exchange(bs.Blockstore()))
 	offlineIpldFetcher := bsfetcher.NewFetcherConfig(offlineBs)
+	offlineIpldFetcher.SkipNotFound = true // carries onto the UnixFSFetcher below
 	offlineIpldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser)
 	offlineUnixFSFetcher := offlineIpldFetcher.WithReifier(unixfsnode.Reify)
 

core/node/provider.go

@@ -7,8 +7,10 @@ import (
 
 	"github.com/ipfs/boxo/blockstore"
 	"github.com/ipfs/boxo/fetcher"
+	"github.com/ipfs/boxo/mfs"
 	pin "github.com/ipfs/boxo/pinning/pinner"
 	provider "github.com/ipfs/boxo/provider"
+	"github.com/ipfs/go-cid"
 	"github.com/ipfs/kubo/repo"
 	irouting "github.com/ipfs/kubo/routing"
 	"go.uber.org/fx"
@@ -136,14 +138,8 @@ func OnlineProviders(useStrategicProviding bool, reprovideStrategy string, repro
 
 	var keyProvider fx.Option
 	switch reprovideStrategy {
-	case "all", "":
-		keyProvider = fx.Provide(newProvidingStrategy(false, false))
-	case "roots":
-		keyProvider = fx.Provide(newProvidingStrategy(true, true))
-	case "pinned":
-		keyProvider = fx.Provide(newProvidingStrategy(true, false))
-	case "flat":
-		keyProvider = fx.Provide(provider.NewBlockstoreProvider)
+	case "all", "", "roots", "pinned", "mfs", "pinned+mfs", "flat":
+		keyProvider = fx.Provide(newProvidingStrategy(reprovideStrategy))
 	default:
 		return fx.Error(fmt.Errorf("unknown reprovider strategy %q", reprovideStrategy))
 	}
@@ -159,32 +155,68 @@ func OfflineProviders() fx.Option {
 	return fx.Provide(provider.NewNoopProvider)
 }
 
-func newProvidingStrategy(onlyPinned, onlyRoots bool) interface{} {
+func mfsProvider(mfsRoot *mfs.Root, fetcher fetcher.Factory) provider.KeyChanFunc {
+	return func(ctx context.Context) (<-chan cid.Cid, error) {
+		err := mfsRoot.FlushMemFree(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("error flushing mfs, cannot provide MFS: %w", err)
+		}
+		rootNode, err := mfsRoot.GetDirectory().GetNode()
+		if err != nil {
+			return nil, fmt.Errorf("error loading mfs root, cannot provide MFS: %w", err)
+		}
+
+		kcf := provider.NewDAGProvider(rootNode.Cid(), fetcher)
+		return kcf(ctx)
+	}
+
+}
+
+func mfsRootProvider(mfsRoot *mfs.Root) provider.KeyChanFunc {
+	return func(ctx context.Context) (<-chan cid.Cid, error) {
+		rootNode, err := mfsRoot.GetDirectory().GetNode()
+		if err != nil {
+			return nil, fmt.Errorf("error loading mfs root, cannot provide MFS: %w", err)
+		}
+		ch := make(chan cid.Cid, 1)
+		ch <- rootNode.Cid()
+		close(ch)
+		return ch, nil
+	}
+}
+
+func newProvidingStrategy(strategy string) interface{} {
 	type input struct {
 		fx.In
-		Pinner      pin.Pinner
-		Blockstore  blockstore.Blockstore
-		IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
+		Pinner               pin.Pinner
+		Blockstore           blockstore.Blockstore
+		OfflineIPLDFetcher   fetcher.Factory `name:"offlineIpldFetcher"`
+		OfflineUnixFSFetcher fetcher.Factory `name:"offlineUnixfsFetcher"`
+		MFSRoot              *mfs.Root
 	}
 	return func(in input) provider.KeyChanFunc {
-		// Pinner-related CIDs will be buffered in memory to avoid
-		// deadlocking the pinner when the providing process is slow.
-
-		if onlyRoots {
-			return provider.NewBufferedProvider(
-				provider.NewPinnedProvider(true, in.Pinner, in.IPLDFetcher),
+		switch strategy {
+		case "roots":
+			return provider.NewBufferedProvider(provider.NewPinnedProvider(true, in.Pinner, in.OfflineIPLDFetcher))
+		case "pinned":
+			return provider.NewBufferedProvider(provider.NewPinnedProvider(false, in.Pinner, in.OfflineIPLDFetcher))
+		case "pinned+mfs":
+			return provider.NewPrioritizedProvider(
+				provider.NewBufferedProvider(provider.NewPinnedProvider(false, in.Pinner, in.OfflineIPLDFetcher)),
+				mfsProvider(in.MFSRoot, in.OfflineUnixFSFetcher),
 			)
-		}
-
-		if onlyPinned {
-			return provider.NewBufferedProvider(
-				provider.NewPinnedProvider(false, in.Pinner, in.IPLDFetcher),
+		case "mfs":
+			return mfsProvider(in.MFSRoot, in.OfflineUnixFSFetcher)
+		case "flat":
+			return provider.NewBlockstoreProvider(in.Blockstore)
+		default: // "all", ""
+			return provider.NewPrioritizedProvider(
+				provider.NewPrioritizedProvider(
+					provider.NewBufferedProvider(provider.NewPinnedProvider(true, in.Pinner, in.OfflineIPLDFetcher)),
+					mfsRootProvider(in.MFSRoot),
+				),
+				provider.NewBlockstoreProvider(in.Blockstore),
 			)
 		}
-
-		return provider.NewPrioritizedProvider(
-			provider.NewBufferedProvider(provider.NewPinnedProvider(true, in.Pinner, in.IPLDFetcher)),
-			provider.NewBlockstoreProvider(in.Blockstore),
-		)
 	}
 }

docs/changelogs/v0.35.md

@@ -10,6 +10,9 @@ This release  was brought to you by the [Shipyard](http://ipshipyard.com/) team.
 
 - [Overview](#overview)
 - [🔦 Highlights](#-highlights)
+  - [Dedicated `Reprovider.Strategy` for MFS](#dedicated-reproviderstrategy-for-mfs)
+  - [Additional new configuration options](#additional-new-configuration-options)
+  - [Grid view in WebUI](#grid-view-in-webui)
   - [Enhanced DAG-Shaping Controls for `ipfs add`](#enhanced-dag-shaping-controls-for-ipfs-add)
     - [New `ipfs add` Options](#new-ipfs-add-options)
     - [Persistent `Import.*` Configuration](#persistent-import-configuration)
@@ -22,12 +25,26 @@ This release  was brought to you by the [Shipyard](http://ipshipyard.com/) team.
 
 ### 🔦 Highlights
 
-##### `Routing.IgnoreProviders`
+#### Dedicated `Reprovider.Strategy` for MFS
 
-This new option allows ignoring specific peer IDs when returned by the content
-routing system as providers of content. See the
-[documentation](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingignoreproviders)
-for for information.
+The [Mutable File System (MFS)](https://docs.ipfs.tech/concepts/glossary/#mfs) in Kubo is a UnixFS filesystem managed with [`ipfs files`](https://docs.ipfs.tech/reference/kubo/cli/#ipfs-files) commands. It supports familiar file operations like cp and mv within a folder-tree structure, automatically updating a MerkleDAG and a "root CID" that reflects the current MFS state. Files in MFS are protected from garbage collection, offering a simpler alternative to `ipfs pin`. This makes it a popular choice for tools like [IPFS Desktop](https://docs.ipfs.tech/install/ipfs-desktop/) and the [WebUI](https://github.com/ipfs/ipfs-webui/#readme).
+
+Previously, the `pinned` reprovider strategy required manual pin management: each dataset update meant pinning the new version and unpinning the old one. Now, new strategies—`mfs` and `pinned+mfs`—let users limit announcements to data explicitly placed in MFS. This simplifies updating datasets and announcing only the latest version to the Amino DHT.
+
+Users relying on the `pinned` strategy can switch to `pinned+mfs` and use MFS alone to manage updates and announcements, eliminating the need for manual pinning and unpinning. We hope this makes it easier to publish just the data that matters to you.
+
+See [`Reprovider.Strategy`](https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy) for more details.
+
+#### Additional new configuration options
+
+- [`Internal.Bitswap.ProviderSearchMaxResults`](https://github.com/ipfs/kubo/blob/master/docs/config.md##internalbitswapprovidersearchmaxresults) for adjusting the maximum number of providers bitswap client should aim at before it stops searching for new ones.
+- [`Routing.IgnoreProviders`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingignoreproviders) allows ignoring specific peer IDs when returned by the content routing system as providers of content.
+
+#### Grid view in WebUI
+
+The WebUI, accessible at http://127.0.0.1:5001/webui/, now includes support for the grid view on the _Files_ screen:
+
+> ![image](https://github.com/user-attachments/assets/80dcf0d0-8103-426f-ae91-416fb25d32b6)
 
 #### Enhanced DAG-Shaping Controls for `ipfs add`
 
@@ -64,6 +81,8 @@ The release updated configuration [profiles](https://github.com/ipfs/kubo/blob/m
 
 #### 📦️ Important dependency updates
 
+- update `ipfs-webui` to [v4.7.0](https://github.com/ipfs/ipfs-webui/releases/tag/v4.7.0)
+
 ### 📝 Changelog
 
 ### 👨‍👩‍👧‍👦 Contributors

docs/config.md

@@ -79,7 +79,8 @@ config file at runtime.
       - [`Internal.Bitswap.EngineBlockstoreWorkerCount`](#internalbitswapengineblockstoreworkercount)
       - [`Internal.Bitswap.EngineTaskWorkerCount`](#internalbitswapenginetaskworkercount)
       - [`Internal.Bitswap.MaxOutstandingBytesPerPeer`](#internalbitswapmaxoutstandingbytesperpeer)
-    - [`Internal.Bitswap.ProviderSearchDelay`](#internalbitswapprovidersearchdelay)
+      - [`Internal.Bitswap.ProviderSearchDelay`](#internalbitswapprovidersearchdelay)
+      - [`Internal.Bitswap.ProviderSearchMaxResults`](#internalbitswapprovidersearchmaxresults)
     - [`Internal.UnixFSShardingSizeThreshold`](#internalunixfsshardingsizethreshold)
   - [`Ipns`](#ipns)
     - [`Ipns.RepublishPeriod`](#ipnsrepublishperiod)
@@ -1185,14 +1186,21 @@ deteriorate the quality provided to less aggressively-wanting peers.
 
 Type: `optionalInteger` (byte count, `null` means default which is 1MB)
 
-### `Internal.Bitswap.ProviderSearchDelay`
+#### `Internal.Bitswap.ProviderSearchDelay`
 
 This parameter determines how long to wait before looking for providers outside of bitswap.
 Other routing systems like the Amino DHT are able to provide results in less than a second, so lowering
 this number will allow faster peers lookups in some cases.
 
 Type: `optionalDuration` (`null` means default which is 1s)
 
+#### `Internal.Bitswap.ProviderSearchMaxResults`
+
+Maximum number of providers bitswap client should aim at before it stops searching for new ones.
+Setting to 0 means unlimited.
+
+Type: `optionalInteger` (`null` means default which is 10)
+
 ### `Internal.UnixFSShardingSizeThreshold`
 
 **MOVED:** see [`Import.UnixFSHAMTDirectorySizeThreshold`](#importunixfshamtdirectorysizethreshold)
@@ -1583,21 +1591,26 @@ Type: `optionalDuration` (unset for the default)
 Tells reprovider what should be announced. Valid strategies are:
 
 - `"all"` - announce all CIDs of stored blocks
-  - Order: root blocks of direct and recursive pins are announced first, then the rest of blockstore
-- `"pinned"` - only announce pinned CIDs recursively (both roots and child blocks)
+  - Order: root blocks of direct and recursive pins and MFS root are announced first, then the rest of blockstore
+- `"pinned"` - only announce recursively pinned CIDs (`ipfs pin add -r`, both roots and child blocks)
   - Order: root blocks of direct and recursive pins are announced first, then the child blocks of recursive pins
-- `"roots"` - only announce the root block of explicitly pinned CIDs
+- `"roots"` - only announce the root block of explicitly pinned CIDs (`ipfs pin add`)
   - **⚠️  BE CAREFUL:** node with `roots` strategy will not announce child blocks.
     It makes sense only for use cases where the entire DAG is fetched in full,
     and a graceful resume does not have to be guaranteed: the lack of child
     announcements means an interrupted retrieval won't be able to find
     providers for the missing block in the middle of a file, unless the peer
     happens to already be connected to a provider and ask for child CID over
     bitswap.
+- `"mfs"` - announce only the local CIDs that are part of the MFS (`ipfs files`)
+   - Note: MFS is lazy-loaded. Only the MFS blocks present in local datastore are announced.
+- `"pinned+mfs"` - a combination of the `pinned` and `mfs` strategies.
+  - **ℹ️ NOTE:** This is the suggested strategy for users who run without GC and don't want to provide everything in cache.
+  - Order: first `pinned` and then the locally available part of `mfs`.
 - `"flat"` - same as `all`, announce all CIDs of stored blocks, but without prioritizing anything.
 
 > [!IMPORTANT]
-> Reproviding larger pinsets using the `all`, `pinned`, or `roots` strategies requires additional memory, with an estimated ~1 GiB of RAM per 20 million items for reproviding to the Amino DHT.
+> Reproviding larger pinsets using the `all`, `mfs`, `pinned`, `pinned+mfs` or `roots` strategies requires additional memory, with an estimated ~1 GiB of RAM per 20 million items for reproviding to the Amino DHT.
 > This is due to the use of a buffered provider, which avoids holding a lock on the entire pinset during the reprovide cycle.
 > The `flat` strategy can be used to lower memory requirements, but only recommended if memory utilization is too high, prioritization of pins is not necessary, and it is acceptable to announce every block cached in the local repository.
 

docs/examples/kubo-as-a-library/go.mod

@@ -7,7 +7,7 @@ go 1.24
 replace github.com/ipfs/kubo => ./../../..
 
 require (
-	github.com/ipfs/boxo v0.29.2-0.20250407221841-f8115e4506c3
+	github.com/ipfs/boxo v0.29.2-0.20250411173252-654578d290f8
 	github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
 	github.com/libp2p/go-libp2p v0.41.1
 	github.com/multiformats/go-multiaddr v0.15.0

docs/examples/kubo-as-a-library/go.sum

@@ -298,8 +298,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd
 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.29.2-0.20250407221841-f8115e4506c3 h1:j8yvRgkHJMtaszUNxTUYLnKPaUVrfomBBhDdDNzKxnY=
-github.com/ipfs/boxo v0.29.2-0.20250407221841-f8115e4506c3/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI=
+github.com/ipfs/boxo v0.29.2-0.20250411173252-654578d290f8 h1:Vxe2X6RO2Xf+YoFZbLOlOW3ErpmVrTCckyIJJJArc+c=
+github.com/ipfs/boxo v0.29.2-0.20250411173252-654578d290f8/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=

go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/hashicorp/go-version v1.7.0
 	github.com/ipfs-shipyard/nopfs v0.0.14
 	github.com/ipfs-shipyard/nopfs/ipfs v0.25.0
-	github.com/ipfs/boxo v0.29.2-0.20250407221841-f8115e4506c3
+	github.com/ipfs/boxo v0.29.2-0.20250411173252-654578d290f8
 	github.com/ipfs/go-block-format v0.2.0
 	github.com/ipfs/go-cid v0.5.0
 	github.com/ipfs/go-cidutil v0.1.0