feat: automatically detect if genesis CAR is compressed (#12885)

Automatically detect if genesis CAR is compressed

When `--genesis` path is set, automatically detect if the genesis file
is ZSTD compressed and decompress it.

Author: masih

CHANGELOG.md

@@ -15,6 +15,8 @@
 
 - chore: switch to pure-go zstd decoder for snapshot imports.  ([filecoin-project/lotus#12857](https://github.com/filecoin-project/lotus/pull/12857))
 
+- feat: automatically detect if the genesis is zstd compressed. ([filecoin-project/lotus#12885](https://github.com/filecoin-project/lotus/pull/12885)
+
 # UNRELEASED v.1.32.0
 
 See https://github.com/filecoin-project/lotus/blob/release/v1.32.0/CHANGELOG.md

build/genesis.go

@@ -1,12 +1,14 @@
 package build
 
 import (
+	"bytes"
 	"embed"
 	"io"
 	"path"
 
 	logging "github.com/ipfs/go-log/v2"
 	"github.com/klauspost/compress/zstd"
+	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/build/buildconstants"
 )
@@ -17,26 +19,51 @@ var log = logging.Logger("build")
 //go:embed genesis/*.car.zst
 var genesisCars embed.FS
 
+var zstdHeader = []byte{0x28, 0xb5, 0x2f, 0xfd}
+
 func MaybeGenesis() []byte {
 	file, err := genesisCars.Open(path.Join("genesis", buildconstants.GenesisFile))
 	if err != nil {
 		log.Warnf("opening built-in genesis: %s", err)
 		return nil
 	}
 	defer file.Close() //nolint
-
-	decoder, err := zstd.NewReader(file)
+	decompressed, err := DecompressAsZstd(file)
 	if err != nil {
-		log.Warnf("creating zstd decoder: %s", err)
+		log.Warnf("decompressing genesis: %s", err)
 		return nil
 	}
+	return decompressed
+}
+
+func DecompressAsZstd(target io.Reader) ([]byte, error) {
+	decoder, err := zstd.NewReader(target)
+	if err != nil {
+		return nil, xerrors.Errorf("creating zstd decoder: %w", err)
+	}
 	defer decoder.Close() //nolint
 
-	decompressedBytes, err := io.ReadAll(decoder)
+	decompressed, err := io.ReadAll(decoder)
 	if err != nil {
-		log.Warnf("reading decompressed genesis file: %s", err)
-		return nil
+		return nil, xerrors.Errorf("reading decompressed genesis file: %w", err)
 	}
+	return decompressed, nil
+}
 
-	return decompressedBytes
+func IsZstdCompressed(file io.ReadSeeker) (_ bool, _err error) {
+	pos, err := file.Seek(0, io.SeekCurrent)
+	if err != nil {
+		return false, xerrors.Errorf("getting current position: %w", err)
+	}
+	defer func() {
+		_, err := file.Seek(pos, io.SeekStart)
+		if _err == nil && err != nil {
+			_err = xerrors.Errorf("seeking back to original offset: %w", err)
+		}
+	}()
+	header := make([]byte, 4)
+	if _, err := file.Read(header); err != nil {
+		return false, xerrors.Errorf("failed to read file header: %w", err)
+	}
+	return bytes.Equal(header, zstdHeader), nil
 }

build/genesis_test.go

@@ -0,0 +1,109 @@
+package build_test
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"os"
+	"testing"
+
+	"github.com/klauspost/compress/zstd"
+	"github.com/stretchr/testify/require"
+
+	"github.com/filecoin-project/lotus/build"
+)
+
+func TestGenesis(t *testing.T) {
+	for _, test := range []struct {
+		path string
+	}{
+		{path: "genesis/butterflynet.car.zst"},
+		{path: "genesis/calibnet.car.zst"},
+		{path: "genesis/interopnet.car.zst"},
+		{path: "genesis/mainnet.car.zst"},
+	} {
+		t.Run(test.path, func(t *testing.T) {
+			subject, err := os.Open(test.path)
+			require.NoError(t, err)
+
+			gotIsCompressed, err := build.IsZstdCompressed(subject)
+			require.NoError(t, err)
+			require.True(t, gotIsCompressed)
+
+			gotDecompressed, err := build.DecompressAsZstd(subject)
+			require.NoError(t, err)
+			require.NotEmpty(t, gotDecompressed)
+
+			gotIsCompressed, err = build.IsZstdCompressed(bytes.NewReader(gotDecompressed))
+			require.NoError(t, err)
+			require.False(t, gotIsCompressed)
+		})
+	}
+}
+
+func TestGenesis_ZstdCheck(t *testing.T) {
+	for _, test := range []struct {
+		name           string
+		given          func(t *testing.T) io.ReadSeeker
+		wantCompressed bool
+		wantErr        bool
+	}{
+		{
+			name: "arbitraryLongEnough",
+			given: func(t *testing.T) io.ReadSeeker {
+				return bytes.NewReader([]byte("fish"))
+			},
+		},
+		{
+			name: "arbitraryShort",
+			given: func(t *testing.T) io.ReadSeeker {
+				return bytes.NewReader([]byte("🐠"))
+			},
+		},
+		{
+			name: "arbitraryZstdCompressed",
+			given: func(t *testing.T) io.ReadSeeker {
+				var buf bytes.Buffer
+				writer, err := zstd.NewWriter(&buf)
+				require.NoError(t, err)
+				written, err := writer.Write([]byte("fish"))
+				require.NoError(t, err)
+				require.NotZero(t, written)
+				require.NoError(t, writer.Close())
+				return bytes.NewReader(buf.Bytes())
+			},
+			wantCompressed: true,
+		},
+		{
+			name:    "failingPositionReset",
+			given:   func(t *testing.T) io.ReadSeeker { return failOnSeekStart{} },
+			wantErr: true,
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			target := test.given(t)
+			gotCompressed, gotErr := build.IsZstdCompressed(target)
+			require.Equal(t, test.wantCompressed, gotCompressed)
+			require.Equal(t, test.wantErr, gotErr != nil)
+
+			if !test.wantErr {
+				gotPosition, err := target.Seek(0, io.SeekCurrent)
+				require.NoError(t, err)
+				require.Zero(t, gotPosition)
+			}
+		})
+	}
+}
+
+var _ io.ReadSeeker = (*failOnSeekStart)(nil)
+
+type failOnSeekStart struct{}
+
+func (failOnSeekStart) Read([]byte) (int, error) { return 0, nil }
+
+func (failOnSeekStart) Seek(_ int64, whence int) (int64, error) {
+	if whence == io.SeekStart {
+		return 0, errors.New("pursue the horizon; forsake the dawn; the start is long gone")
+	}
+	return 0, nil
+}

cli/lotus/daemon.go

@@ -107,7 +107,7 @@ var DaemonCmd = &cli.Command{
 		},
 		&cli.StringFlag{
 			Name:  "genesis",
-			Usage: "genesis file to use for first node run",
+			Usage: "genesis file to use for first node run, which may be a zstd compressed CAR or an uncompressed CAR file.",
 		},
 		&cli.BoolFlag{
 			Name:  "bootstrap",
@@ -253,9 +253,9 @@ var DaemonCmd = &cli.Command{
 		}
 
 		var genBytes []byte
-		if cctx.String("genesis") != "" {
-			genBytes, err = os.ReadFile(cctx.String("genesis"))
-			if err != nil {
+		genesisPath := cctx.String("genesis")
+		if genesisPath != "" {
+			if genBytes, err = readGenesis(genesisPath); err != nil {
 				return xerrors.Errorf("reading genesis: %w", err)
 			}
 		} else {
@@ -470,6 +470,23 @@ var DaemonCmd = &cli.Command{
 	},
 }
 
+// readGenesis detects if the path points to a zstd compressed file and if so
+// automatically decompress it. Otherwise, defaults to reading the path as is.
+func readGenesis(path string) ([]byte, error) {
+	genesisFile, err := os.Open(filepath.Clean(path))
+	if err != nil {
+		return nil, xerrors.Errorf("opening genesis file: %w", err)
+	}
+	defer func() { _ = genesisFile.Close() }()
+
+	if compressed, err := build.IsZstdCompressed(genesisFile); err != nil {
+		return nil, xerrors.Errorf("checking genesis header: %w", err)
+	} else if compressed {
+		return build.DecompressAsZstd(genesisFile)
+	}
+	return io.ReadAll(genesisFile)
+}
+
 func importKey(ctx context.Context, api lapi.FullNode, f string) error {
 	f, err := homedir.Expand(f)
 	if err != nil {

documentation/en/cli-lotus.md

@@ -64,7 +64,7 @@ COMMANDS:
 
 OPTIONS:
    --api value               (default: "1234")
-   --genesis value           genesis file to use for first node run
+   --genesis value           genesis file to use for first node run, which may be a zstd compressed CAR or an uncompressed CAR file.
    --bootstrap               (default: true)
    --import-chain value      on first run, load chain from given file or url and validate
    --import-snapshot value   import chain state from a given chain export file or url