Feat/allow small pieces (#479)

feat: allow configurable min-piece-size to reduce excessive padding for small segments

This change introduces support for specifying a `--min-piece-size` when preparing data, improving behavior for small DAGs, remainder CARs, and small preparations that would otherwise be padded to the full target piece size (e.g. 32GiB). Such excessive padding leads to inefficiencies and causes sectors to be rejected by Storage Providers or fail verified deal requirements.

### Key changes:
- Add support for `--min-piece-size` (default: 256B, subject to adjustment)
- Pass both `min` and `target` piece sizes to `GetCommp`, enabling finer control over padding
- Retain power-of-2 padding via `target size`, but allow flexibility by setting it to `0`

This helps avoid generating 90%+ padding pieces and reduces transfer times in many cases.

### Notes:
- Default behavior remains unchanged if `--min-piece-size` is not set
- Full support for non-padded pieces now depends on both chunker accuracy and downstream deal acceptance
- `pieceType` is now tracked in metadata (e.g., data vs. DAG)

### Out of scope:
- No cross-preparation aggregation; that responsibility remains with SPs
- Edge cases like aggregating under-1MiB pieces are not yet solved

Closes #473

Co-authored-by: Arkadiy Kukarkin <[email protected]>

Author: parkan

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.23.6-bullseye as builder
+FROM golang:1.23.6-bullseye AS builder
 WORKDIR /app
 COPY go.* ./
 RUN go mod download

client/swagger/models/dataprep_create_request.go

@@ -263,7 +263,7 @@ func TestBasicDataPrep(t *testing.T) {
 		require.True(t, listPiecesResp.IsSuccess())
 		require.Len(t, listPiecesResp.Payload, 1)
 		require.Len(t, listPiecesResp.Payload[0].Pieces, 1)
-		require.Equal(t, "baga6ea4seaqoahdvfwkrp64ecsxbjvyuqcwpz3o7ctxrjanlv2x4u2cq2qjf2ji", listPiecesResp.Payload[0].Pieces[0].PieceCid)
+		require.Equal(t, "baga6ea4seaqhmks2wnochilik4updmit54agfi5mjf6r7ehotu36ksdp46uxahi", listPiecesResp.Payload[0].Pieces[0].PieceCid)
 		// Start daggen
 		startDagGenResp, err := client.Job.StartDagGen(&job.StartDagGenParams{
 			ID:      "prep",
@@ -285,7 +285,9 @@ func TestBasicDataPrep(t *testing.T) {
 		require.True(t, listPiecesResp.IsSuccess())
 		require.Len(t, listPiecesResp.Payload, 1)
 		require.Len(t, listPiecesResp.Payload[0].Pieces, 2)
-		require.Equal(t, "baga6ea4seaqoahdvfwkrp64ecsxbjvyuqcwpz3o7ctxrjanlv2x4u2cq2qjf2ji", listPiecesResp.Payload[0].Pieces[0].PieceCid)
-		require.Equal(t, "baga6ea4seaqbkouoyih2elxfrztq3gr23rpvgpx5e3fnud2rhvvzf4b7tneeyki", listPiecesResp.Payload[0].Pieces[1].PieceCid)
+		// data piece, full size
+		require.Equal(t, "baga6ea4seaqhmks2wnochilik4updmit54agfi5mjf6r7ehotu36ksdp46uxahi", listPiecesResp.Payload[0].Pieces[0].PieceCid)
+		// dag piece, min piece size
+		require.Equal(t, "baga6ea4seaqfoo2k3wmwp7gvxnc7hbjpb7ovtvt52tehwfvzxbreljcebbnwgiq", listPiecesResp.Payload[0].Pieces[1].PieceCid)
 	})
 }

client/swagger/models/model_car.go

@@ -55,6 +55,12 @@ var CreateCmd = &cli.Command{
 			Value:       "",
 			DefaultText: "Determined by --max-size",
 		},
+		&cli.StringFlag{
+			Name:        "min-piece-size",
+			Usage:       "The minimum size of a piece. Pieces smaller than this will be padded up to this size. It's recommended to leave this as the default",
+			Value:       "1MiB",
+			DefaultText: "1MiB",
+		},
 		&cli.BoolFlag{
 			Name:  "delete-after-export",
 			Usage: "Whether to delete the source files after export to CAR files",
@@ -83,6 +89,7 @@ var CreateCmd = &cli.Command{
 		outputStorages := c.StringSlice("output")
 		maxSizeStr := c.String("max-size")
 		pieceSizeStr := c.String("piece-size")
+		minPieceSizeStr := c.String("min-piece-size")
 		for _, sourcePath := range c.StringSlice("local-source") {
 			source, err := createStorageIfNotExist(c.Context, db, sourcePath)
 			if err != nil {
@@ -103,8 +110,9 @@ var CreateCmd = &cli.Command{
 			OutputStorages:    outputStorages,
 			MaxSizeStr:        maxSizeStr,
 			PieceSizeStr:      pieceSizeStr,
-			DeleteAfterExport: c.Bool("delete-after-export"),
+			MinPieceSizeStr:   minPieceSizeStr,
 			Name:              name,
+			DeleteAfterExport: c.Bool("delete-after-export"),
 			NoInline:          c.Bool("no-inline"),
 			NoDag:             c.Bool("no-dag"),
 		})

client/swagger/models/model_preparation.go

@@ -451,6 +451,10 @@ func TestDataPrep(t *testing.T) {
 							require.Equal(t, pieceCID, calculatedPieceCID)
 							err = os.WriteFile(filepath.Join(downloadDir, pieceCID+".car"), downloaded, 0777)
 							require.NoError(t, err)
+
+							// Verify piece size is a power of two
+							pieceSize := uint64(len(downloaded))
+							require.True(t, util.IsPowerOfTwo(pieceSize), "piece size %d is not a power of two", pieceSize)
 						}
 
 						// Download all pieces using local download server
@@ -499,7 +503,7 @@ func TestNoDuplicatedOutput(t *testing.T) {
 		_, _, err = runner.Run(ctx, fmt.Sprintf("singularity storage create local --name source --path %s", testutil.EscapePath(source)))
 		require.NoError(t, err)
 
-		_, _, err = runner.Run(ctx, fmt.Sprintf("singularity prep create --name test-prep --delete-after-export --source source --local-output %s --max-size=500KiB", testutil.EscapePath(output)))
+		_, _, err = runner.Run(ctx, fmt.Sprintf("singularity prep create --name test-prep --delete-after-export --source source --local-output %s --max-size=500KiB --min-piece-size=256KiB", testutil.EscapePath(output)))
 		require.NoError(t, err)
 
 		// Start scanning