feat: Reprovider.Strategy: rename "flat" to "all" (#10928)

* Reprovider strategy: rename "flat" to "all".

Value "flat" now parses to "all". Behaviour from "all" removed.

Fixes #10864 which has detailed explanation.

* core/node/provider.go: remove unused function mfsRootProvider

It was used in the "all" strategy.

* docs: improve reprovider.strategy=all changelog framing

- highlight memory efficiency improvements
- clarify this removes v0.28 workaround
- update config.md memory requirements
- fix announce-on profile typo

* feat: deprecate Reprovider.Strategy=flat

- add deprecation warning in daemon.go when flat strategy is detected
- document that flat is deprecated in ParseReproviderStrategy comment
- add explicit test case for flat -> all mapping
- flat continues to work but users are warned to migrate to all

---------

Co-authored-by: Marcin Rataj <[email protected]>

Author: hsanjuan

cmd/ipfs/kubo/daemon.go

@@ -479,6 +479,10 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
 	if cfg.Provider.Strategy.WithDefault("") != "" && cfg.Reprovider.Strategy.IsDefault() {
 		log.Fatal("Invalid config. Remove unused Provider.Strategy and set Reprovider.Strategy instead. Documentation: https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy")
 	}
+	// Check for deprecated "flat" strategy
+	if cfg.Reprovider.Strategy.WithDefault("") == "flat" {
+		log.Error("Reprovider.Strategy='flat' is deprecated and will be removed in the next release. Please update your config to use 'all' instead.")
+	}
 	if cfg.Experimental.StrategicProviding {
 		log.Error("Experimental.StrategicProviding was removed. Remove it from your config and set Provider.Enabled=false to remove this message. Documentation: https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#strategic-providing")
 		cfg.Experimental.StrategicProviding = false

config/reprovider.go

@@ -13,11 +13,10 @@ const (
 type ReproviderStrategy int
 
 const (
-	ReproviderStrategyAll    ReproviderStrategy = 1 << iota // 1  (0b00001)
-	ReproviderStrategyFlat                                  // 2  (0b00010)
-	ReproviderStrategyPinned                                // 4  (0b00100)
-	ReproviderStrategyRoots                                 // 8  (0b01000)
-	ReproviderStrategyMFS                                   // 16 (0b10000)
+	ReproviderStrategyAll ReproviderStrategy = 1 << iota
+	ReproviderStrategyPinned
+	ReproviderStrategyRoots
+	ReproviderStrategyMFS
 )
 
 // Reprovider configuration describes how CID from local datastore are periodically re-announced to routing systems.
@@ -31,10 +30,8 @@ func ParseReproviderStrategy(s string) ReproviderStrategy {
 	var strategy ReproviderStrategy
 	for _, part := range strings.Split(s, "+") {
 		switch part {
-		case "all", "": // special case, does not mix with others
+		case "all", "flat", "": // special case, does not mix with others ("flat" is deprecated, maps to "all")
 			return ReproviderStrategyAll
-		case "flat":
-			strategy |= ReproviderStrategyFlat
 		case "pinned":
 			strategy |= ReproviderStrategyPinned
 		case "roots":

config/reprovider_test.go

@@ -0,0 +1,27 @@
+package config
+
+import "testing"
+
+func TestParseReproviderStrategy(t *testing.T) {
+	tests := []struct {
+		input  string
+		expect ReproviderStrategy
+	}{
+		{"all", ReproviderStrategyAll},
+		{"pinned", ReproviderStrategyPinned},
+		{"mfs", ReproviderStrategyMFS},
+		{"pinned+mfs", ReproviderStrategyPinned | ReproviderStrategyMFS},
+		{"invalid", 0},
+		{"all+invalid", ReproviderStrategyAll},
+		{"", ReproviderStrategyAll},
+		{"flat", ReproviderStrategyAll}, // deprecated, maps to "all"
+		{"flat+all", ReproviderStrategyAll},
+	}
+
+	for _, tt := range tests {
+		result := ParseReproviderStrategy(tt.input)
+		if result != tt.expect {
+			t.Errorf("ParseReproviderStrategy(%q) = %d, want %d", tt.input, result, tt.expect)
+		}
+	}
+}

core/coreapi/unixfs.go

@@ -108,7 +108,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options
 
 	// wrap the DAGService in a providingDAG service which provides every block written.
 	// note about strategies:
-	//   - "all"/"flat" gets handled directly at the blockstore so no need to provide
+	//   - "all" gets handled directly at the blockstore so no need to provide
 	//   - "roots" gets handled in the pinner
 	//   - "mfs" gets handled in mfs
 	// We need to provide the "pinned" cases only. Added blocks are not

core/node/provider.go

@@ -184,19 +184,6 @@ func mfsProvider(mfsRoot *mfs.Root, fetcher fetcher.Factory) provider.KeyChanFun
 	}
 }
 
-func mfsRootProvider(mfsRoot *mfs.Root) provider.KeyChanFunc {
-	return func(ctx context.Context) (<-chan cid.Cid, error) {
-		rootNode, err := mfsRoot.GetDirectory().GetNode()
-		if err != nil {
-			return nil, fmt.Errorf("error loading mfs root, cannot provide MFS: %w", err)
-		}
-		ch := make(chan cid.Cid, 1)
-		ch <- rootNode.Cid()
-		close(ch)
-		return ch, nil
-	}
-}
-
 type provStrategyIn struct {
 	fx.In
 	Pinner               pin.Pinner
@@ -219,8 +206,7 @@ type provStrategyOut struct {
 // - "roots": Only root CIDs of pinned content
 // - "pinned": All pinned content (roots + children)
 // - "mfs": Only MFS content
-// - "flat": All blocks, no prioritization
-// - "all": Prioritized: pins first, then MFS roots, then all blocks
+// - "all": all blocks
 func createKeyProvider(strategyFlag config.ReproviderStrategy, in provStrategyIn) provider.KeyChanFunc {
 	switch strategyFlag {
 	case config.ReproviderStrategyRoots:
@@ -234,28 +220,11 @@ func createKeyProvider(strategyFlag config.ReproviderStrategy, in provStrategyIn
 		)
 	case config.ReproviderStrategyMFS:
 		return mfsProvider(in.MFSRoot, in.OfflineUnixFSFetcher)
-	case config.ReproviderStrategyFlat:
+	default: // "all", "", "flat" (compat)
 		return in.Blockstore.AllKeysChan
-	default: // "all", ""
-		return createAllStrategyProvider(in)
 	}
 }
 
-// createAllStrategyProvider creates the complex "all" strategy provider.
-// This implements a three-tier priority system:
-// 1. Root blocks of direct and recursive pins (highest priority)
-// 2. MFS root (medium priority)
-// 3. All other blocks in blockstore (lowest priority)
-func createAllStrategyProvider(in provStrategyIn) provider.KeyChanFunc {
-	return provider.NewPrioritizedProvider(
-		provider.NewPrioritizedProvider(
-			provider.NewBufferedProvider(dspinner.NewPinnedProvider(true, in.Pinner, in.OfflineIPLDFetcher)),
-			mfsRootProvider(in.MFSRoot),
-		),
-		in.Blockstore.AllKeysChan,
-	)
-}
-
 // detectStrategyChange checks if the reproviding strategy has changed from what's persisted.
 // Returns: (previousStrategy, hasChanged, error)
 func detectStrategyChange(ctx context.Context, strategy string, ds datastore.Datastore) (string, bool, error) {

core/node/storage.go

@@ -39,13 +39,12 @@ func BaseBlockstoreCtor(
 		opts := []blockstore.Option{blockstore.WriteThrough(writeThrough)}
 
 		// Blockstore providing integration:
-		// When strategy includes "all" or "flat", the blockstore directly provides blocks as they're Put.
+		// When strategy includes "all" the blockstore directly provides blocks as they're Put.
 		// Important: Provide calls from blockstore are intentionally BLOCKING.
 		// The Provider implementation (not the blockstore) should handle concurrency/queuing.
 		// This avoids spawning unbounded goroutines for concurrent block additions.
 		strategyFlag := config.ParseReproviderStrategy(providingStrategy)
-		shouldProvide := config.ReproviderStrategyAll | config.ReproviderStrategyFlat
-		if strategyFlag&shouldProvide != 0 {
+		if strategyFlag&config.ReproviderStrategyAll != 0 {
 			opts = append(opts, blockstore.Provider(prov))
 		}
 

docs/changelogs/v0.37.md

@@ -19,6 +19,7 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team.
   - [📌 Named pins in `ipfs add` command](#-named-pins-in-ipfs-add-command)
   - [Custom sequence numbers in `ipfs name publish`](#custom-sequence-numbers-in-ipfs-name-publish)
   - [⚙️ `Reprovider.Strategy` is now consistently respected](#-reprovider-strategy-is-now-consistently-respected)
+  - [⚙️ `Reprovider.Strategy=all`: improved memory efficiency](#-reproviderstrategyall-improved-memory-efficiency)
   - [Removed unnecessary dependencies](#removed-unnecessary-dependencies)
   - [Improved `ipfs cid`](#improved-ipfs-cid)
   - [Deprecated `ipfs stats reprovide`](#deprecated-ipfs-stats-reprovide)
@@ -151,7 +152,7 @@ $ ipfs pin ls --names
 bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi recursive testname
 ```
 
-#### ⚙️ `Reprovider.Strategy` is now consistently respected
+#### ⚙️ `Reprovider.Strategy` is now consistently respected.
 
 Prior to this version, files added, blocks received etc. were "provided" to the network (announced on the DHT) regardless of the ["reproviding strategy" setting](https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy). For example:
 
@@ -165,6 +166,15 @@ This was inefficient as content that should not be provided was getting provided
 > [!NOTE]
 > **Behavior change:** The `--offline` flag no longer affects providing behavior. Both `ipfs add` and `ipfs --offline add` now provide blocks according to the reproviding strategy when run against an online daemon (previously `--offline add` did not provide). Since `ipfs add` has been nearly as fast as offline mode [since v0.35](https://github.com/ipfs/kubo/blob/master/docs/changelogs/v0.35.md#fast-ipfs-add-in-online-mode), `--offline` is rarely needed. To run truly offline operations, use `ipfs --offline daemon`.
 
+#### ⚙️ `Reprovider.Strategy=all`: improved memory efficiency
+
+The memory cost of `Reprovider.Strategy=all` no longer grows with the number of pins. The strategy now processes blocks directly from the datastore in undefined order, eliminating the memory pressure tied to the number of pins.
+
+As part of this improvement, the `flat` reprovider strategy has been renamed to `all` (the default). This cleanup removes the workaround introduced in v0.28 for pin root prioritization. With the introduction of more granular strategies like [`pinned+mfs`](https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy), we can now optimize the default `all` strategy for lower memory usage without compromising users who need pin root prioritization ([rationale](https://github.com/ipfs/kubo/pull/10928#issuecomment-3211040182)).
+
+> [!NOTE]
+> **Migration guidance:** If you experience undesired announcement delays of root CIDs with the new `all` strategy, switch to `pinned+mfs` for root prioritization.
+
 #### Removed unnecessary dependencies
 
 Kubo has been cleaned up by removing unnecessary dependencies and packages:

docs/config.md

@@ -2063,7 +2063,6 @@ Type: `optionalDuration` (unset for the default)
 Tells reprovider what should be announced. Valid strategies are:
 
 - `"all"` - announce all CIDs of stored blocks
-  - Order: root blocks of direct and recursive pins and MFS root are announced first, then the rest of blockstore
 - `"pinned"` - only announce recursively pinned CIDs (`ipfs pin add -r`, both roots and child blocks)
   - Order: root blocks of direct and recursive pins are announced first, then the child blocks of recursive pins
 - `"roots"` - only announce the root block of explicitly pinned CIDs (`ipfs pin add`)
@@ -2079,15 +2078,13 @@ Tells reprovider what should be announced. Valid strategies are:
 - `"pinned+mfs"` - a combination of the `pinned` and `mfs` strategies.
   - **ℹ️ NOTE:** This is the suggested strategy for users who run without GC and don't want to provide everything in cache.
   - Order: first `pinned` and then the locally available part of `mfs`.
-- `"flat"` - same as `all`, announce all CIDs of stored blocks, but without prioritizing anything.
 
 **Strategy changes automatically clear the provide queue.** When you change `Reprovider.Strategy` and restart Kubo, the provide queue is automatically cleared to ensure only content matching your new strategy is announced. You can also manually clear the queue using `ipfs provide clear`.
 
 **Memory requirements:**
 
-- Reproviding larger pinsets using the `all`, `mfs`, `pinned`, `pinned+mfs` or `roots` strategies requires additional memory, with an estimated ~1 GiB of RAM per 20 million items for reproviding to the Amino DHT.
+- Reproviding larger pinsets using the `mfs`, `pinned`, `pinned+mfs` or `roots` strategies requires additional memory, with an estimated ~1 GiB of RAM per 20 million items for reproviding to the Amino DHT.
 - This is due to the use of a buffered provider, which avoids holding a lock on the entire pinset during the reprovide cycle.
-- The `flat` strategy can be used to lower memory requirements, but only recommended if memory utilization is too high, prioritization of pins is not necessary, and it is acceptable to announce every block cached in the local repository.
 
 Default: `"all"`
 
@@ -3446,7 +3443,7 @@ Disables [Reprovider](#reprovider) system (and announcing to Amino DHT).
 
 ### `announce-on` profile
 
-(Re-)enables [Reprovider](#reprovider) system (reverts [`announce-off` profile](#annouce-off-profile).
+(Re-)enables [Reprovider](#reprovider) system (reverts [`announce-off` profile](#announce-off-profile)).
 
 ### `legacy-cid-v0` profile
 

test/cli/provider_test.go

@@ -187,18 +187,6 @@ func TestProvider(t *testing.T) {
 		expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...)
 	})
 
-	t.Run("Provide with 'flat' strategy", func(t *testing.T) {
-		t.Parallel()
-
-		nodes := initNodes(t, 2, func(n *harness.Node) {
-			n.SetIPFSConfig("Reprovider.Strategy", "flat")
-		})
-		defer nodes.StopDaemons()
-
-		cid := nodes[0].IPFSAddStr("flat strategy")
-		expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...)
-	})
-
 	t.Run("Provide with 'pinned' strategy", func(t *testing.T) {
 		t.Parallel()
 
@@ -308,24 +296,6 @@ func TestProvider(t *testing.T) {
 		expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...)
 	})
 
-	t.Run("Reprovides with 'flat' strategy", func(t *testing.T) {
-		t.Parallel()
-
-		nodes := initNodesWithoutStart(t, 2, func(n *harness.Node) {
-			n.SetIPFSConfig("Reprovider.Strategy", "flat")
-		})
-
-		cid := nodes[0].IPFSAddStr(time.Now().String())
-
-		nodes = nodes.StartDaemons().Connect()
-		defer nodes.StopDaemons()
-		expectNoProviders(t, cid, nodes[1:]...)
-
-		nodes[0].IPFS("routing", "reprovide")
-
-		expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...)
-	})
-
 	t.Run("Reprovides with 'pinned' strategy", func(t *testing.T) {
 		t.Parallel()