feat(Issue 121): Adding lz4 as compression option.

Adding lz4 dependency and adding
to the options for compression.

Adding lz4 logic in for decompression
and compression.

Unit tests still need to be written.

Author: cjtaylor1990

cmd/init.go

@@ -173,7 +173,7 @@ func init() {
 	initCmd.Flags().StringVar(&hashAlgorithm, "hash", "sha256", "Hash algorithm (sha256, blake3)")
 
 	// Compression vars
-	initCmd.Flags().StringVar(&compressionType, "compression", "none", "Compression type (none, gzip, zstd)")
+	initCmd.Flags().StringVar(&compressionType, "compression", "none", "Compression type (none, gzip, zstd, lz4)")
 
 	// Sync vars
 	initCmd.Flags().StringVar(&syncMode, "sync-mode", "manual", "Synchronization mode (manual, auto)")

go.mod

@@ -121,6 +121,7 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/libp2p/go-libp2p v0.41.1
 	github.com/multiformats/go-multiaddr v0.15.0
+	github.com/pierrec/lz4/v4 v4.1.22
 	github.com/spf13/cobra v1.9.1
 	github.com/spf13/pflag v1.0.6
 	golang.org/x/sys v0.36.0 // indirect

go.sum

@@ -232,6 +232,8 @@ github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/
 github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
+github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
+github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pion/datachannel v1.5.10 h1:ly0Q26K1i6ZkGf42W7D4hQYR90pZwzFOjTq5AuCKk4o=
 github.com/pion/datachannel v1.5.10/go.mod h1:p/jJfC9arb29W7WrxyKbepTU20CFgyx5oLo8Rs4Py/M=
 github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s=

internal/chunk/prompt.go

@@ -99,7 +99,7 @@ func PromptCompressionConfig(configuration *config.VaultConfig) error {
 	// Compression prompt with descriptions
 	compressionPrompt := promptui.Select{
 		Label: "Compression algorithm",
-		Items: []string{"none", "gzip", "zstd"},
+		Items: []string{"none", "gzip", "zstd", "lz4"},
 		Templates: &promptui.SelectTemplates{
 			Selected: "Compression: {{ . }}",
 			Active:   "▸ {{ . }}",
@@ -108,7 +108,8 @@ func PromptCompressionConfig(configuration *config.VaultConfig) error {
 {{ "Details:" | faint }}
 {{ if eq . "none" }}No compression (faster but larger files)
 {{ else if eq . "gzip" }}Gzip compression (good balance of speed/compression)
-{{ else if eq . "zstd" }}Zstandard compression (better compression but slower){{ end }}
+{{ else if eq . "zstd" }}Zstandard compression (better compression but slower)
+{{ else if eq . "lz4" }}LZ4 compression (very fast but lower compression ratio){{ end }}
 `,
 		},
 	}

internal/compression/Compressor.go

@@ -7,6 +7,7 @@ import (
 	"io"
 
 	"github.com/klauspost/compress/zstd"
+	"github.com/pierrec/lz4/v4"
 	"github.com/substantialcattle5/sietch/internal/constants"
 )
 
@@ -32,6 +33,16 @@ func CompressData(data []byte, algorithm string) ([]byte, error) {
 		}
 		defer encoder.Close()
 		return encoder.EncodeAll(data, nil), nil
+	case constants.CompressionTypeLz4:
+		var buf bytes.Buffer
+		writer := lz4.NewWriter(&buf)
+		if _, err := writer.Write(data); err != nil {
+			return nil, fmt.Errorf("failed to write lz4 data: %w", err)
+		}
+		if err := writer.Close(); err != nil {
+			return nil, fmt.Errorf("failed to close lz4 writer: %w", err)
+		}
+		return buf.Bytes(), nil
 	default:
 		return nil, fmt.Errorf("unsupported compression algorithm: %s", algorithm)
 	}
@@ -83,6 +94,19 @@ func DecompressData(data []byte, algorithm string) ([]byte, error) {
 		}
 
 		return decompressed, nil
+
+	case constants.CompressionTypeLz4:
+		reader := lz4.NewReader(bytes.NewReader(data))
+		if reader.Size() >= constants.MaxDecompressionSize {
+			return nil, fmt.Errorf("decompressed data exceeds maximum size limit (%d bytes) - potential decompression bomb", constants.MaxDecompressionSize)
+		}
+		var buf bytes.Buffer
+		_, err := io.Copy(&buf, reader)
+		if err != nil && err != io.EOF {
+			return nil, fmt.Errorf("failed to decompress lz4 data: %w", err)
+		}
+		return buf.Bytes(), nil
+
 	default:
 		return nil, fmt.Errorf("unsupported compression algorithm: %s", algorithm)
 	}

internal/config/vault.go

@@ -171,7 +171,7 @@ type ChunkRef struct {
 	Index           int    `yaml:"index"`                      // Position in the file
 	Deduplicated    bool   `yaml:"deduplicated,omitempty"`     // Whether this chunk was deduplicated
 	Compressed      bool   `yaml:"compressed,omitempty"`       // Whether this chunk was compressed
-	CompressionType string `yaml:"compression_type,omitempty"` // Compression algorithm used (e.g., "gzip", "zstd", "none")
+	CompressionType string `yaml:"compression_type,omitempty"` // Compression algorithm used (e.g., "gzip", "zstd", "lz4", "none")
 	IV              string `yaml:"iv,omitempty"`               // Per-chunk IV if used
 	Integrity       string `yaml:"integrity,omitempty"`        // Integrity check value (e.g., HMAC)
 }

internal/constants/constants.go

@@ -82,6 +82,7 @@ const (
 	//** Constants for compression
 	CompressionTypeGzip = "gzip"
 	CompressionTypeZstd = "zstd"
+	CompressionTypeLz4  = "lz4"
 	CompressionTypeNone = "none"
 
 	// Maximum decompression size to prevent decompression bombs