test: legacy-cid-v1, UnixFSChunker, UnixFSFileMaxLinks

basic smoke test confirming UnixFSFileMaxLinks sets
to 'ipfs add' defaults and informs the DAG shape

Author: lidel

test/cli/add_test.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/ipfs/kubo/config"
 	"github.com/ipfs/kubo/test/cli/harness"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -106,13 +107,53 @@ func TestAdd(t *testing.T) {
 		require.Equal(t, shortStringCidV0, cidStr)
 	})
 
-	t.Run("ipfs init --profile=legacy-cid-v1 produces modern CIDv1", func(t *testing.T) {
+	t.Run("ipfs init --profile=legacy-cid-v0 applies UnixFSChunker=size-262144 and UnixFSFileMaxLinks=174", func(t *testing.T) {
+		t.Parallel()
+		node := harness.NewT(t).NewNode().Init("--profile=legacy-cid-v0")
+		node.StartDaemon()
+		defer node.StopDaemon()
+
+		// Add 44544KiB file:
+		// 174 * 256KiB should fit in single DAG layer
+		cidStr := node.IPFSAddFromSeed("44544KiB", "v0-seed")
+		root, err := node.InspectPBNode(cidStr)
+		assert.NoError(t, err)
+		require.Equal(t, 174, len(root.Links))
+
+		// add 256KiB (one more block), it should force rebalancing DAG and moving most to second layer
+		cidStr = node.IPFSAddFromSeed("44800KiB", "v0-seed")
+		root, err = node.InspectPBNode(cidStr)
+		assert.NoError(t, err)
+		require.Equal(t, 2, len(root.Links))
+	})
+
+	t.Run("ipfs init --profile=legacy-cid-v1 produces CIDv1 with raw leaves", func(t *testing.T) {
 		t.Parallel()
 		node := harness.NewT(t).NewNode().Init("--profile=legacy-cid-v1")
 		node.StartDaemon()
 		defer node.StopDaemon()
 
 		cidStr := node.IPFSAddStr(shortString)
-		require.Equal(t, shortStringCidV1, cidStr)
+		require.Equal(t, shortStringCidV1, cidStr) // raw leaf
+	})
+
+	t.Run("ipfs init --profile=legacy-cid-v1 applies UnixFSChunker=size-1048576 and UnixFSFileMaxLinks=174", func(t *testing.T) {
+		t.Parallel()
+		node := harness.NewT(t).NewNode().Init("--profile=legacy-cid-v1")
+		node.StartDaemon()
+		defer node.StopDaemon()
+
+		// Add 174MiB file:
+		// 174 * 1MiB should fit in single layer
+		cidStr := node.IPFSAddFromSeed("174MiB", "v1-seed")
+		root, err := node.InspectPBNode(cidStr)
+		assert.NoError(t, err)
+		require.Equal(t, 174, len(root.Links))
+
+		// add +1MiB (one more block), it should force rebalancing DAG and moving most to second layer
+		cidStr = node.IPFSAddFromSeed("175MiB", "v1-seed")
+		root, err = node.InspectPBNode(cidStr)
+		assert.NoError(t, err)
+		require.Equal(t, 2, len(root.Links))
 	})
 }

test/cli/harness/ipfs.go

@@ -76,6 +76,17 @@ func (n *Node) IPFSAddStr(content string, args ...string) string {
 	return n.IPFSAdd(strings.NewReader(content), args...)
 }
 
+// IPFSAddDeterministic produces a CID of a file of a certain size, filled with deterministically generated bytes based  on some seed.
+// This ensures deterministic CID on the other end, that can be used in tests.
+func (n *Node) IPFSAddFromSeed(size string, seed string, args ...string) string {
+	log.Debugf("node %d adding %s of deterministic pseudo-random data with seed %q and args: %v", n.ID, size, seed, args)
+	reader, err := createRandomReader(size, seed)
+	if err != nil {
+		panic(err)
+	}
+	return n.IPFSAdd(reader, args...)
+}
+
 func (n *Node) IPFSAdd(content io.Reader, args ...string) string {
 	log.Debugf("node %d adding with args: %v", n.ID, args)
 	fullArgs := []string{"add", "-q"}
@@ -108,3 +119,15 @@ func (n *Node) IPFSDagImport(content io.Reader, cid string, args ...string) erro
 	})
 	return res.Err
 }
+
+/*
+func (n *Node) IPFSDagExport(cid string, car *os.File) error {
+	log.Debugf("node %d dag export of %s to %q with args: %v", n.ID, cid, car.Name())
+	res := n.Runner.MustRun(RunRequest{
+		Path:    n.IPFSBin,
+		Args:    []string{"dag", "export", cid},
+		CmdOpts: []CmdOpt{RunWithStdout(car)},
+	})
+	return res.Err
+}
+*/

test/cli/harness/pbinspect.go

@@ -0,0 +1,54 @@
+package harness
+
+import (
+	"bytes"
+	"encoding/json"
+)
+
+// InspectPBNode uses dag-json output of 'ipfs dag get' to inspect
+// "Logical Format" of DAG-PB as defined in
+// https://web.archive.org/web/20250403194752/https://ipld.io/specs/codecs/dag-pb/spec/#logical-format
+// (mainly used for inspecting Links without depending on any libraries)
+func (n *Node) InspectPBNode(cid string) (PBNode, error) {
+	log.Debugf("node %d dag get %s as dag-json", n.ID, cid)
+
+	var root PBNode
+	var dagJsonOutput bytes.Buffer
+	res := n.Runner.MustRun(RunRequest{
+		Path:    n.IPFSBin,
+		Args:    []string{"dag", "get", "--output-codec=dag-json", cid},
+		CmdOpts: []CmdOpt{RunWithStdout(&dagJsonOutput)},
+	})
+	if res.Err != nil {
+		return root, res.Err
+	}
+
+	err := json.Unmarshal(dagJsonOutput.Bytes(), &root)
+	if err != nil {
+		return root, err
+	}
+	return root, nil
+
+}
+
+// Define structs to match the JSON for
+type PBHash struct {
+	Slash string `json:"/"`
+}
+
+type PBLink struct {
+	Hash  PBHash `json:"Hash"`
+	Name  string `json:"Name"`
+	Tsize int    `json:"Tsize"`
+}
+
+type PBData struct {
+	Slash struct {
+		Bytes string `json:"bytes"`
+	} `json:"/"`
+}
+
+type PBNode struct {
+	Data  PBData   `json:"Data"`
+	Links []PBLink `json:"Links"`
+}

test/cli/harness/random_reader.go

@@ -0,0 +1,46 @@
+package harness
+
+import (
+	"crypto/sha256"
+	"io"
+
+	"github.com/dustin/go-humanize"
+	"golang.org/x/crypto/chacha20"
+)
+
+type randomReader struct {
+	cipher    *chacha20.Cipher
+	remaining int64
+}
+
+func (r *randomReader) Read(p []byte) (int, error) {
+	if r.remaining <= 0 {
+		return 0, io.EOF
+	}
+	n := int64(len(p))
+	if n > r.remaining {
+		n = r.remaining
+	}
+	// Generate random bytes directly into the provided buffer
+	r.cipher.XORKeyStream(p[:n], make([]byte, n))
+	r.remaining -= n
+	return int(n), nil
+}
+
+// createRandomReader produces specified number of pseudo-random bytes
+// from a seed.
+func createRandomReader(sizeStr string, seed string) (io.Reader, error) {
+	size, err := humanize.ParseBytes(sizeStr)
+	if err != nil {
+		return nil, err
+	}
+	// Hash the seed string to a 32-byte key for ChaCha20
+	key := sha256.Sum256([]byte(seed))
+	// Use ChaCha20 for deterministic random bytes
+	var nonce [chacha20.NonceSize]byte // Zero nonce for simplicity
+	cipher, err := chacha20.NewUnauthenticatedCipher(key[:chacha20.KeySize], nonce[:])
+	if err != nil {
+		return nil, err
+	}
+	return &randomReader{cipher: cipher, remaining: int64(size)}, nil
+}