porep: No-overhead unsealed creation in Filalize (#174)

* porep: No-overhead unsealed creation in Filalize

* Fix unsealed path in finalize

Author: magik6k

lib/ffi/sdr_funcs.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/KarpelesLab/reflink"
 	"github.com/ipfs/go-cid"
@@ -477,14 +478,32 @@ func (sb *SealCalls) LocalStorage(ctx context.Context) ([]storiface.StoragePath,
 	return sb.sectors.localStore.Local(ctx)
 }
 
-func (sb *SealCalls) FinalizeSector(ctx context.Context, sector storiface.SectorRef, keepUnsealed bool) error {
-	alloc := storiface.FTNone
-	if keepUnsealed {
-		// note: In Curio we don't write the unsealed file in any of the previous stages, it's only written here from tree-d
-		alloc = storiface.FTUnsealed
+func changePathType(path string, newType storiface.SectorFileType) (string, error) {
+	// /some/parent/[type]/filename -> /some/parent/[newType]/filename
+
+	dir, file := filepath.Split(path)
+	if dir == "" {
+		return "", xerrors.Errorf("path has too few components: %s", path)
+	}
+
+	components := strings.Split(strings.TrimRight(dir, string(filepath.Separator)), string(filepath.Separator))
+	if len(components) < 1 {
+		return "", xerrors.Errorf("path has too few components: %s", path)
 	}
 
-	sectorPaths, pathIDs, releaseSector, err := sb.sectors.AcquireSector(ctx, nil, sector, storiface.FTCache, alloc, storiface.PathSealing)
+	newTypeName := newType.String()
+
+	components[len(components)-1] = newTypeName
+	newPath := filepath.Join(append(components, file)...)
+
+	if filepath.IsAbs(path) {
+		newPath = filepath.Join(string(filepath.Separator), newPath)
+	}
+
+	return newPath, nil
+}
+func (sb *SealCalls) FinalizeSector(ctx context.Context, sector storiface.SectorRef, keepUnsealed bool) error {
+	sectorPaths, pathIDs, releaseSector, err := sb.sectors.AcquireSector(ctx, nil, sector, storiface.FTCache, storiface.FTNone, storiface.PathSealing)
 	if err != nil {
 		return xerrors.Errorf("acquiring sector paths: %w", err)
 	}
@@ -496,6 +515,21 @@ func (sb *SealCalls) FinalizeSector(ctx context.Context, sector storiface.Sector
 	}
 
 	if keepUnsealed {
+		// We are going to be moving the unsealed file, no need to allocate storage specifically for it
+		sectorPaths.Unsealed, err = changePathType(sectorPaths.Cache, storiface.FTUnsealed)
+		if err != nil {
+			return xerrors.Errorf("changing path type: %w", err)
+		}
+
+		pathIDs.Unsealed = pathIDs.Cache // this is just an uuid string
+
+		defer func() {
+			// We don't pass FTUnsealed to Acquire, so releaseSector won't declare it. Do it here.
+			if err := sb.sectors.sindex.StorageDeclareSector(ctx, storiface.ID(pathIDs.Unsealed), sector.ID, storiface.FTUnsealed, true); err != nil {
+				log.Errorf("declare unsealed sector error: %+v", err)
+			}
+		}()
+
 		// tree-d contains exactly unsealed data in the prefix, so
 		// * we move it to a temp file
 		// * we truncate the temp file to the sector size
@@ -579,7 +613,12 @@ afterUnsealedMove:
 		return xerrors.Errorf("clearing cache: %w", err)
 	}
 
-	if err := sb.ensureOneCopy(ctx, sector.ID, pathIDs, storiface.FTCache|alloc); err != nil {
+	maybeUns := storiface.FTUnsealed
+	if !keepUnsealed {
+		maybeUns = storiface.FTNone
+	}
+
+	if err := sb.ensureOneCopy(ctx, sector.ID, pathIDs, storiface.FTCache|maybeUns); err != nil {
 		return xerrors.Errorf("ensure one copy: %w", err)
 	}
 

lib/ffi/sdr_funcs_test.go

@@ -0,0 +1,85 @@
+package ffi
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/filecoin-project/lotus/storage/sealer/storiface"
+)
+
+func TestChangePathType(t *testing.T) {
+	tests := []struct {
+		name        string
+		path        string
+		newType     storiface.SectorFileType
+		expected    string
+		expectError bool
+	}{
+		{
+			name:        "Valid relative path change",
+			path:        filepath.Join("some", "parent", "sealed", "filename"),
+			newType:     storiface.FTCache,
+			expected:    filepath.Join("some", "parent", "cache", "filename"),
+			expectError: false,
+		},
+		{
+			name:        "Valid absolute path change",
+			path:        filepath.Join("/", "some", "parent", "sealed", "filename"),
+			newType:     storiface.FTCache,
+			expected:    filepath.Join("/", "some", "parent", "cache", "filename"),
+			expectError: false,
+		},
+		{
+			name:        "Same type, no change (relative)",
+			path:        filepath.Join("some", "parent", "sealed", "filename"),
+			newType:     storiface.FTSealed,
+			expected:    filepath.Join("some", "parent", "sealed", "filename"),
+			expectError: false,
+		},
+		{
+			name:        "Same type, no change (absolute)",
+			path:        filepath.Join("/", "some", "parent", "sealed", "filename"),
+			newType:     storiface.FTSealed,
+			expected:    filepath.Join("/", "some", "parent", "sealed", "filename"),
+			expectError: false,
+		},
+		{
+			name:        "Too few components",
+			path:        "filename",
+			newType:     storiface.FTCache,
+			expected:    "",
+			expectError: true,
+		},
+		{
+			name:        "Empty path",
+			path:        "",
+			newType:     storiface.FTCache,
+			expected:    "",
+			expectError: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, err := changePathType(tt.path, tt.newType)
+
+			if tt.expectError {
+				if err == nil {
+					t.Errorf("Expected an error, but got nil")
+				}
+			} else {
+				if err != nil {
+					t.Errorf("Unexpected error: %v", err)
+				}
+				if result != tt.expected {
+					t.Errorf("Expected %s, but got %s", tt.expected, result)
+				}
+
+				// Check if the absolute/relative nature of the path is preserved
+				if filepath.IsAbs(tt.path) != filepath.IsAbs(result) {
+					t.Errorf("Absolute/relative nature of path not preserved. Original: %s, Result: %s", tt.path, result)
+				}
+			}
+		})
+	}
+}