-
Notifications
You must be signed in to change notification settings - Fork 275
feat(storage): add compression primitives: FrameTable, V4 header #2246
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Closed
Closed
Changes from all commits
Commits
Show all changes
7 commits
Select commit
Hold shift + click to select a range
7691321
feat(storage): add compression primitives — FrameTable, codec pools, …
levb cf85907
feat(storage): add compressed upload pipeline, CompressConfig, and pa…
levb 3cb0bb1
feat(header): V4 header format with FrameTable serialization, BuildFi…
levb 4c6962a
refactor(storage): cleanup compression primitives for PR review
levb 2c65691
chore: auto-commit generated changes
github-actions[bot] 00149d4
fix(storage): address PR review — LZ4 streaming API, upload goroutine…
levb be428aa
Merge branch 'lev-compression-primitives' of github.com:e2b-dev/infra…
levb File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,131 @@ | ||
| package storage | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
|
|
||
| "github.com/e2b-dev/infra/packages/shared/pkg/featureflags" | ||
| ) | ||
|
|
||
| // CompressConfig is the base compression configuration, loaded from environment | ||
| // variables at startup. Feature flags can override individual fields at runtime | ||
| // via ResolveCompressConfig. | ||
| type CompressConfig struct { | ||
| Enabled bool `env:"COMPRESS_ENABLED" envDefault:"false"` | ||
| Type string `env:"COMPRESS_TYPE" envDefault:"zstd"` | ||
| Level int `env:"COMPRESS_LEVEL" envDefault:"2"` | ||
| FrameSizeKB int `env:"COMPRESS_FRAME_SIZE_KB" envDefault:"2048"` | ||
| TargetPartSizeMB int `env:"COMPRESS_TARGET_PART_SIZE_MB" envDefault:"50"` | ||
| FrameEncodeWorkers int `env:"COMPRESS_FRAME_ENCODE_WORKERS" envDefault:"4"` | ||
| EncoderConcurrency int `env:"COMPRESS_ENCODER_CONCURRENCY" envDefault:"1"` | ||
| } | ||
|
|
||
| // CompressionType returns the parsed CompressionType. | ||
| func (c *CompressConfig) CompressionType() CompressionType { | ||
| if c == nil { | ||
| return CompressionNone | ||
| } | ||
|
|
||
| return ParseCompressionType(c.Type) | ||
| } | ||
|
|
||
| // FrameSize returns the frame size in bytes. | ||
| func (c *CompressConfig) FrameSize() int { | ||
| if c == nil || c.FrameSizeKB <= 0 { | ||
| return DefaultCompressFrameSize | ||
| } | ||
|
|
||
| return c.FrameSizeKB * 1024 | ||
| } | ||
|
|
||
| // TargetPartSize returns the target part size in bytes. | ||
| func (c *CompressConfig) TargetPartSize() int64 { | ||
| if c == nil || c.TargetPartSizeMB <= 0 { | ||
| return int64(gcpMultipartUploadChunkSize) | ||
| } | ||
|
|
||
| return int64(c.TargetPartSizeMB) * (1 << 20) | ||
| } | ||
|
|
||
| // IsEnabled reports whether compression is configured and active. | ||
| func (c *CompressConfig) IsEnabled() bool { | ||
| return c != nil && c.Enabled && c.CompressionType() != CompressionNone | ||
| } | ||
|
|
||
| // Validate checks that the config is internally consistent. | ||
| func (c *CompressConfig) Validate() error { | ||
| if c == nil || !c.IsEnabled() { | ||
| return nil | ||
| } | ||
|
|
||
| fs := c.FrameSize() | ||
| if fs <= 0 { | ||
| return fmt.Errorf("frame size must be positive, got %d KB", c.FrameSizeKB) | ||
| } | ||
| if MemoryChunkSize%fs != 0 && fs%MemoryChunkSize != 0 { | ||
| return fmt.Errorf("frame size (%d) must be a divisor or multiple of MemoryChunkSize (%d)", fs, MemoryChunkSize) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // Resolve returns a pointer to this config if compression is enabled, or nil. | ||
| // Callers use nil to mean "no compression". | ||
| func (c *CompressConfig) Resolve() *CompressConfig { | ||
| if c == nil || !c.IsEnabled() { | ||
| return nil | ||
| } | ||
|
|
||
| return c | ||
| } | ||
|
|
||
| // CompressConfigFromLDValue parses the LaunchDarkly CompressConfigFlag JSON | ||
| // into a CompressConfig. Returns nil if the flag disables compression. | ||
| func CompressConfigFromLDValue(ctx context.Context, ff *featureflags.Client) *CompressConfig { | ||
| if ff == nil { | ||
| return nil | ||
| } | ||
|
|
||
| v := ff.JSONFlag(ctx, featureflags.CompressConfigFlag).AsValueMap() | ||
|
|
||
| if !v.Get("compressBuilds").BoolValue() { | ||
| return nil | ||
| } | ||
|
|
||
| ct := v.Get("compressionType").StringValue() | ||
| if ParseCompressionType(ct) == CompressionNone { | ||
| return nil | ||
| } | ||
|
|
||
| return &CompressConfig{ | ||
| Enabled: true, | ||
| Type: ct, | ||
| Level: v.Get("compressionLevel").IntValue(), | ||
| FrameSizeKB: v.Get("frameSizeKB").IntValue(), | ||
| TargetPartSizeMB: v.Get("targetPartSizeMB").IntValue(), | ||
| FrameEncodeWorkers: v.Get("frameEncodeWorkers").IntValue(), | ||
| EncoderConcurrency: v.Get("encoderConcurrency").IntValue(), | ||
| } | ||
| } | ||
|
|
||
| // ResolveCompressConfig returns the effective compression config for a given | ||
| // file type and use case. Feature flags override the base config when active. | ||
| // Returns nil when compression is disabled. | ||
| // | ||
| // fileType and useCase are added to the LD evaluation context so that | ||
levb marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| // LaunchDarkly targeting rules can differentiate (e.g. compress memfile | ||
| // but not rootfs, or compress builds but not pauses). | ||
| func ResolveCompressConfig(ctx context.Context, base CompressConfig, ff *featureflags.Client, fileType, useCase string) *CompressConfig { | ||
| if ff != nil { | ||
| ctx = featureflags.AddToContext(ctx, | ||
| featureflags.CompressFileTypeContext(fileType), | ||
| featureflags.CompressUseCaseContext(useCase), | ||
| ) | ||
|
|
||
| if override := CompressConfigFromLDValue(ctx, ff); override != nil { | ||
| return override | ||
| } | ||
| } | ||
|
|
||
| return base.Resolve() | ||
| } | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
isn't this defined by the feature flags already?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
we need to override the FF settings for the integration tests, and also to configure the service in the future without FF