feat(storage): add compression primitives: FrameTable, V4 header #2246
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,131 @@ | ||
| package storage | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
|
|
||
| "github.com/e2b-dev/infra/packages/shared/pkg/featureflags" | ||
| ) | ||
|
|
||
| // CompressConfig is the base compression configuration, loaded from environment | ||
| // variables at startup. Feature flags can override individual fields at runtime | ||
| // via ResolveCompressConfig. | ||
| type CompressConfig struct { | ||
| Enabled bool `env:"COMPRESS_ENABLED" envDefault:"false"` | ||
| Type string `env:"COMPRESS_TYPE" envDefault:"zstd"` | ||
| Level int `env:"COMPRESS_LEVEL" envDefault:"2"` | ||
| FrameSizeKB int `env:"COMPRESS_FRAME_SIZE_KB" envDefault:"2048"` | ||
| TargetPartSizeMB int `env:"COMPRESS_TARGET_PART_SIZE_MB" envDefault:"50"` | ||
| FrameEncodeWorkers int `env:"COMPRESS_FRAME_ENCODE_WORKERS" envDefault:"4"` | ||
| EncoderConcurrency int `env:"COMPRESS_ENCODER_CONCURRENCY" envDefault:"1"` | ||
| } | ||
|
Comment on lines
+13
to
+21
Contributor
There was a problem hiding this comment. isn't this defined by the feature flags already?
Contributor
Author
There was a problem hiding this comment. we need to override the FF settings for the integration tests, and also to configure the service in the future without FF |
||
|
|
||
| // CompressionType returns the parsed CompressionType. | ||
| func (c *CompressConfig) CompressionType() CompressionType { | ||
| if c == nil { | ||
| return CompressionNone | ||
| } | ||
|
|
||
| return ParseCompressionType(c.Type) | ||
| } | ||
|
|
||
| // FrameSize returns the frame size in bytes. | ||
| func (c *CompressConfig) FrameSize() int { | ||
| if c == nil || c.FrameSizeKB <= 0 { | ||
| return DefaultCompressFrameSize | ||
| } | ||
|
|
||
| return c.FrameSizeKB * 1024 | ||
| } | ||
|
|
||
| // TargetPartSize returns the target part size in bytes. | ||
| func (c *CompressConfig) TargetPartSize() int64 { | ||
| if c == nil || c.TargetPartSizeMB <= 0 { | ||
| return int64(gcpMultipartUploadChunkSize) | ||
| } | ||
|
|
||
| return int64(c.TargetPartSizeMB) * (1 << 20) | ||
| } | ||
|
|
||
| // IsEnabled reports whether compression is configured and active. | ||
| func (c *CompressConfig) IsEnabled() bool { | ||
| return c != nil && c.Enabled && c.CompressionType() != CompressionNone | ||
| } | ||
|
|
||
| // Validate checks that the config is internally consistent. | ||
| func (c *CompressConfig) Validate() error { | ||
| if c == nil || !c.IsEnabled() { | ||
| return nil | ||
| } | ||
|
|
||
| fs := c.FrameSize() | ||
| if fs <= 0 { | ||
| return fmt.Errorf("frame size must be positive, got %d KB", c.FrameSizeKB) | ||
| } | ||
| if MemoryChunkSize%fs != 0 && fs%MemoryChunkSize != 0 { | ||
| return fmt.Errorf("frame size (%d) must be a divisor or multiple of MemoryChunkSize (%d)", fs, MemoryChunkSize) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // Resolve returns a pointer to this config if compression is enabled, or nil. | ||
| // Callers use nil to mean "no compression". | ||
| func (c *CompressConfig) Resolve() *CompressConfig { | ||
| if c == nil || !c.IsEnabled() { | ||
| return nil | ||
| } | ||
|
|
||
| return c | ||
| } | ||
|
|
||
| // CompressConfigFromLDValue parses the LaunchDarkly CompressConfigFlag JSON | ||
| // into a CompressConfig. Returns nil if the flag disables compression. | ||
| func CompressConfigFromLDValue(ff *featureflags.Client, ctx context.Context) *CompressConfig { | ||
| if ff == nil { | ||
| return nil | ||
| } | ||
|
|
||
| v := ff.JSONFlag(ctx, featureflags.CompressConfigFlag).AsValueMap() | ||
|
|
||
| if !v.Get("compressBuilds").BoolValue() { | ||
| return nil | ||
| } | ||
|
|
||
| ct := v.Get("compressionType").StringValue() | ||
| if ParseCompressionType(ct) == CompressionNone { | ||
| return nil | ||
| } | ||
|
|
||
| return &CompressConfig{ | ||
| Enabled: true, | ||
| Type: ct, | ||
| Level: v.Get("compressionLevel").IntValue(), | ||
| FrameSizeKB: v.Get("frameSizeKB").IntValue(), | ||
| TargetPartSizeMB: v.Get("targetPartSizeMB").IntValue(), | ||
| FrameEncodeWorkers: v.Get("frameEncodeWorkers").IntValue(), | ||
| EncoderConcurrency: v.Get("encoderConcurrency").IntValue(), | ||
| } | ||
| } | ||
|
|
||
| // ResolveCompressConfig returns the effective compression config for a given | ||
| // file type and use case. Feature flags override the base config when active. | ||
| // Returns nil when compression is disabled. | ||
| // | ||
| // fileType and useCase are added to the LD evaluation context so that | ||
|
Check warning on line 115 in packages/shared/pkg/storage/compress_config.go
|
||
levb marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| // LaunchDarkly targeting rules can differentiate (e.g. compress memfile | ||
| // but not rootfs, or compress builds but not pauses). | ||
| func ResolveCompressConfig(ctx context.Context, base CompressConfig, ff *featureflags.Client, fileType, useCase string) *CompressConfig { | ||
| if ff != nil { | ||
| ctx = featureflags.AddToContext(ctx, | ||
| featureflags.CompressFileTypeContext(fileType), | ||
| featureflags.CompressUseCaseContext(useCase), | ||
| ) | ||
|
|
||
| if override := CompressConfigFromLDValue(ff, ctx); override != nil { | ||
| return override | ||
| } | ||
| } | ||
|
|
||
| return base.Resolve() | ||
| } | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.