Skip to content

Commit 114763a

Browse files
litianningdatadoglitianningdatadog
authored
feat: eat: Add hierarchical configurable compression levels (#800)
feat: Add hierarchical configurable compression levels - Add global compression_level config parameter (0-9, default: 6) with fallback hierarchy - Support 2-level compression configuration: global level first, then module-specific - This makes configuration more convenient - set once globally or override per module - Apply compression configuration to metrics flushers and trace processor - Add environment variable DD_COMPRESSION_LEVEL for global setting Test - Configuration: <img width="966" height="742" alt="image" src="https://github.com/user-attachments/assets/b33c0fd3-2b02-4838-8660-fc9ea9493998" /> - ([log](https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1#logsV2:log-groups/log-group/$252Faws$252Flambda$252Fltn1-fullinstrument-bn-cold-python310-lambda/log-events/2025$252F08$252F25$252F$255B$2524LATEST$255D9c19719435bc48839f6f005d2b58b552)) Configuration: <img width="965" height="568" alt="image" src="https://github.com/user-attachments/assets/dfef594a-549f-4773-879d-549234f03fb7" />
1 parent 983ce6f commit 114763a

File tree

8 files changed

+114
-14
lines changed

8 files changed

+114
-14
lines changed

‎bottlecap/Cargo.lock‎

Lines changed: 3 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎bottlecap/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ datadog-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev =
6262
datadog-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "9405db9cb4ef733f3954c3ee77ce71a502e98e50" , features = ["mini_agent"] }
6363
datadog-trace-normalization = { git = "https://github.com/DataDog/libdatadog", rev = "9405db9cb4ef733f3954c3ee77ce71a502e98e50" }
6464
datadog-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "9405db9cb4ef733f3954c3ee77ce71a502e98e50" }
65-
dogstatsd = { git = "https://github.com/DataDog/serverless-components", rev = "abfec752b0638a9e4096e1465acd4bb2651edfa7", default-features = false }
65+
dogstatsd = { git = "https://github.com/DataDog/serverless-components", rev = "b0b0cb9310d8d8f2038c00a46e3267e21dc3e287", default-features = false }
6666
datadog-fips = { git = "https://github.com/DataDog/serverless-components", rev = "fa1d2f4ea2c4c2596144a1f362935e56cf0cb3c7", default-features = false }
6767
libddwaf = { version = "1.26.0", git = "https://github.com/DataDog/libddwaf-rust", rev = "1d57bf0ca49782723e556ba327ee7f378978aaa7", default-features = false, features = ["serde", "dynamic"] }
6868

‎bottlecap/src/bin/bottlecap/main.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1035,6 +1035,7 @@ fn start_metrics_flushers(
10351035
https_proxy: config.proxy_https.clone(),
10361036
timeout: Duration::from_secs(config.flush_timeout),
10371037
retry_strategy: DsdRetryStrategy::Immediate(3),
1038+
compression_level: config.metrics_config_compression_level,
10381039
};
10391040
flushers.push(MetricsFlusher::new(flusher_config));
10401041

@@ -1063,6 +1064,7 @@ fn start_metrics_flushers(
10631064
https_proxy: config.proxy_https.clone(),
10641065
timeout: Duration::from_secs(config.flush_timeout),
10651066
retry_strategy: DsdRetryStrategy::Immediate(3),
1067+
compression_level: config.metrics_config_compression_level,
10661068
};
10671069
flushers.push(MetricsFlusher::new(additional_flusher_config));
10681070
}

‎bottlecap/src/config/env.rs‎

Lines changed: 43 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,12 @@ pub struct EnvConfig {
107107
/// @env `DD_TAGS`
108108
#[serde(deserialize_with = "deserialize_key_value_pairs")]
109109
pub tags: HashMap<String, String>,
110+
/// @env `DD_COMPRESSION_LEVEL`
111+
///
112+
/// Global level `compression_level` parameter accepts values from 0 (no compression)
113+
/// to 9 (maximum compression but higher resource usage). This value is effective only if
114+
/// the individual component doesn't specify its own.
115+
pub compression_level: Option<i32>,
110116

111117
// Logs
112118
/// @env `DD_LOGS_CONFIG_LOGS_DD_URL`
@@ -229,6 +235,12 @@ pub struct EnvConfig {
229235
#[serde(deserialize_with = "deserialize_optional_bool_from_anything")]
230236
pub trace_propagation_http_baggage_enabled: Option<bool>,
231237

238+
/// @env `DD_METRICS_CONFIG_COMPRESSION_LEVEL`
239+
/// The metrics compresses traces before sending them. The `compression_level` parameter
240+
/// accepts values from 0 (no compression) to 9 (maximum compression but
241+
/// higher resource usage).
242+
pub metrics_config_compression_level: Option<i32>,
243+
232244
// OTLP
233245
//
234246
// - APM / Traces
@@ -393,10 +405,18 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) {
393405
merge_string!(config, env_config, url);
394406
merge_hashmap!(config, env_config, additional_endpoints);
395407

408+
merge_option_to_value!(config, env_config, compression_level);
409+
396410
// Logs
397411
merge_string!(config, env_config, logs_config_logs_dd_url);
398412
merge_option!(config, env_config, logs_config_processing_rules);
399413
merge_option_to_value!(config, env_config, logs_config_use_compression);
414+
merge_option_to_value!(
415+
config,
416+
logs_config_compression_level,
417+
env_config,
418+
compression_level
419+
);
400420
merge_option_to_value!(config, env_config, logs_config_compression_level);
401421
merge_vec!(config, env_config, logs_config_additional_endpoints);
402422

@@ -414,6 +434,12 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) {
414434
env_config,
415435
apm_config_obfuscation_http_remove_paths_with_digits
416436
);
437+
merge_option_to_value!(
438+
config,
439+
apm_config_compression_level,
440+
env_config,
441+
compression_level
442+
);
417443
merge_option_to_value!(config, env_config, apm_config_compression_level);
418444
merge_vec!(config, env_config, apm_features);
419445
merge_hashmap!(config, env_config, apm_additional_endpoints);
@@ -429,6 +455,15 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) {
429455
merge_option_to_value!(config, env_config, trace_propagation_extract_first);
430456
merge_option_to_value!(config, env_config, trace_propagation_http_baggage_enabled);
431457

458+
// Metrics
459+
merge_option_to_value!(
460+
config,
461+
metrics_config_compression_level,
462+
env_config,
463+
compression_level
464+
);
465+
merge_option_to_value!(config, env_config, metrics_config_compression_level);
466+
432467
// OTLP
433468
merge_option_to_value!(config, env_config, otlp_config_traces_enabled);
434469
merge_option_to_value!(
@@ -588,6 +623,7 @@ mod tests {
588623
jail.set_env("DD_SERVICE", "test-service");
589624
jail.set_env("DD_VERSION", "1.0.0");
590625
jail.set_env("DD_TAGS", "team:test-team,project:test-project");
626+
jail.set_env("DD_COMPRESSION_LEVEL", "4");
591627

592628
// Logs
593629
jail.set_env("DD_LOGS_CONFIG_LOGS_DD_URL", "https://logs.datadoghq.com");
@@ -596,7 +632,7 @@ mod tests {
596632
r#"[{"type":"exclude_at_match","name":"exclude","pattern":"exclude"}]"#,
597633
);
598634
jail.set_env("DD_LOGS_CONFIG_USE_COMPRESSION", "false");
599-
jail.set_env("DD_LOGS_CONFIG_COMPRESSION_LEVEL", "3");
635+
jail.set_env("DD_LOGS_CONFIG_COMPRESSION_LEVEL", "1");
600636
jail.set_env(
601637
"DD_LOGS_CONFIG_ADDITIONAL_ENDPOINTS",
602638
"[{\"api_key\": \"apikey2\", \"Host\": \"agent-http-intake.logs.datadoghq.com\", \"Port\": 443, \"is_reliable\": true}]",
@@ -615,7 +651,7 @@ mod tests {
615651
"DD_APM_CONFIG_OBFUSCATION_HTTP_REMOVE_PATHS_WITH_DIGITS",
616652
"true",
617653
);
618-
jail.set_env("DD_APM_CONFIG_COMPRESSION_LEVEL", "3");
654+
jail.set_env("DD_APM_CONFIG_COMPRESSION_LEVEL", "2");
619655
jail.set_env(
620656
"DD_APM_FEATURES",
621657
"enable_otlp_compute_top_level_by_span_kind,enable_stats_by_span_kind",
@@ -632,6 +668,7 @@ mod tests {
632668
"env:^test.*$ debug:^true$",
633669
);
634670

671+
jail.set_env("DD_METRICS_CONFIG_COMPRESSION_LEVEL", "3");
635672
// Trace Propagation
636673
jail.set_env("DD_TRACE_PROPAGATION_STYLE", "datadog");
637674
jail.set_env("DD_TRACE_PROPAGATION_STYLE_EXTRACT", "b3");
@@ -721,6 +758,7 @@ mod tests {
721758
site: "test-site".to_string(),
722759
api_key: "test-api-key".to_string(),
723760
log_level: LogLevel::Debug,
761+
compression_level: 4,
724762
flush_timeout: 42,
725763
proxy_https: Some("https://proxy.example.com".to_string()),
726764
proxy_no_proxy: vec!["localhost".to_string(), "127.0.0.1".to_string()],
@@ -752,7 +790,7 @@ mod tests {
752790
replace_placeholder: None,
753791
}]),
754792
logs_config_use_compression: false,
755-
logs_config_compression_level: 3,
793+
logs_config_compression_level: 1,
756794
logs_config_additional_endpoints: vec![LogsAdditionalEndpoint {
757795
api_key: "apikey2".to_string(),
758796
host: "agent-http-intake.logs.datadoghq.com".to_string(),
@@ -772,7 +810,7 @@ mod tests {
772810
),
773811
apm_config_obfuscation_http_remove_query_string: true,
774812
apm_config_obfuscation_http_remove_paths_with_digits: true,
775-
apm_config_compression_level: 3,
813+
apm_config_compression_level: 2,
776814
apm_features: vec![
777815
"enable_otlp_compute_top_level_by_span_kind".to_string(),
778816
"enable_stats_by_span_kind".to_string(),
@@ -808,6 +846,7 @@ mod tests {
808846
trace_propagation_extract_first: true,
809847
trace_propagation_http_baggage_enabled: true,
810848
trace_aws_service_representation_enabled: true,
849+
metrics_config_compression_level: 3,
811850
otlp_config_traces_enabled: false,
812851
otlp_config_traces_span_name_as_resource_name: true,
813852
otlp_config_traces_span_name_remappings: HashMap::from([(

‎bottlecap/src/config/mod.rs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,10 @@ pub struct Config {
245245
// Timeout for the request to flush data to Datadog endpoint
246246
pub flush_timeout: u64,
247247

248+
// Global config of compression levels.
249+
// It would be overridden by the setup for the individual component
250+
pub compression_level: i32,
251+
248252
// Proxy
249253
pub proxy_https: Option<String>,
250254
pub proxy_no_proxy: Vec<String>,
@@ -291,6 +295,9 @@ pub struct Config {
291295
pub trace_propagation_http_baggage_enabled: bool,
292296
pub trace_aws_service_representation_enabled: bool,
293297

298+
// Metrics
299+
pub metrics_config_compression_level: i32,
300+
294301
// OTLP
295302
//
296303
// - APM / Traces
@@ -368,6 +375,8 @@ impl Default for Config {
368375
version: None,
369376
tags: HashMap::new(),
370377

378+
compression_level: 6,
379+
371380
// Logs
372381
logs_config_logs_dd_url: String::default(),
373382
logs_config_processing_rules: None,
@@ -397,6 +406,9 @@ impl Default for Config {
397406
trace_propagation_extract_first: false,
398407
trace_propagation_http_baggage_enabled: false,
399408

409+
// Metrics
410+
metrics_config_compression_level: 6,
411+
400412
// OTLP
401413
otlp_config_traces_enabled: true,
402414
otlp_config_traces_span_name_as_resource_name: false,

‎bottlecap/src/config/yaml.rs‎

Lines changed: 51 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ pub struct YamlConfig {
3838

3939
pub flush_timeout: Option<u64>,
4040

41+
pub compression_level: Option<i32>,
42+
4143
// Proxy
4244
pub proxy: ProxyConfig,
4345
// nit: this should probably be in the endpoints section
@@ -78,6 +80,9 @@ pub struct YamlConfig {
7880
#[serde(deserialize_with = "deserialize_optional_bool_from_anything")]
7981
pub trace_propagation_http_baggage_enabled: Option<bool>,
8082

83+
// Metrics
84+
pub metrics_config: MetricsConfig,
85+
8186
// OTLP
8287
pub otlp_config: Option<OtlpConfig>,
8388

@@ -131,6 +136,15 @@ pub struct LogsConfig {
131136
pub additional_endpoints: Vec<LogsAdditionalEndpoint>,
132137
}
133138

139+
/// Metrics specific config
140+
///
141+
#[derive(Debug, PartialEq, Deserialize, Clone, Copy, Default)]
142+
#[serde(default)]
143+
#[allow(clippy::module_name_repetitions)]
144+
pub struct MetricsConfig {
145+
pub compression_level: Option<i32>,
146+
}
147+
134148
/// APM Config
135149
///
136150
@@ -373,6 +387,7 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) {
373387
merge_option!(config, yaml_config, version);
374388
merge_hashmap!(config, yaml_config, tags);
375389

390+
merge_option_to_value!(config, yaml_config, compression_level);
376391
// Proxy
377392
merge_option!(config, proxy_https, yaml_config.proxy, https);
378393
merge_option_to_value!(config, proxy_no_proxy, yaml_config.proxy, no_proxy);
@@ -401,6 +416,12 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) {
401416
yaml_config.logs_config,
402417
use_compression
403418
);
419+
merge_option_to_value!(
420+
config,
421+
logs_config_compression_level,
422+
yaml_config,
423+
compression_level
424+
);
404425
merge_option_to_value!(
405426
config,
406427
logs_config_compression_level,
@@ -414,6 +435,20 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) {
414435
additional_endpoints
415436
);
416437

438+
merge_option_to_value!(
439+
config,
440+
metrics_config_compression_level,
441+
yaml_config,
442+
compression_level
443+
);
444+
445+
merge_option_to_value!(
446+
config,
447+
metrics_config_compression_level,
448+
yaml_config.metrics_config,
449+
compression_level
450+
);
451+
417452
// APM
418453
merge_hashmap!(config, yaml_config, service_mapping);
419454
merge_string!(config, apm_dd_url, yaml_config.apm_config, apm_dd_url);
@@ -423,6 +458,12 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) {
423458
yaml_config.apm_config,
424459
replace_tags
425460
);
461+
merge_option_to_value!(
462+
config,
463+
apm_config_compression_level,
464+
yaml_config,
465+
compression_level
466+
);
426467
merge_option_to_value!(
427468
config,
428469
apm_config_compression_level,
@@ -667,7 +708,7 @@ site: "test-site"
667708
api_key: "test-api-key"
668709
log_level: "debug"
669710
flush_timeout: 42
670-
711+
compression_level: 4
671712
# Proxy
672713
proxy:
673714
https: "https://proxy.example.com"
@@ -699,7 +740,7 @@ logs_config:
699740
type: "exclude_at_match"
700741
pattern: "test-pattern"
701742
use_compression: false
702-
compression_level: 3
743+
compression_level: 1
703744
additional_endpoints:
704745
- api_key: "apikey2"
705746
Host: "agent-http-intake.logs.datadoghq.com"
@@ -714,7 +755,7 @@ apm_config:
714755
http:
715756
remove_query_string: true
716757
remove_paths_with_digits: true
717-
compression_level: 3
758+
compression_level: 2
718759
features:
719760
- "enable_otlp_compute_top_level_by_span_kind"
720761
- "enable_stats_by_span_kind"
@@ -734,6 +775,9 @@ trace_propagation_extract_first: true
734775
trace_propagation_http_baggage_enabled: true
735776
trace_aws_service_representation_enabled: true
736777
778+
metrics_config:
779+
compression_level: 3
780+
737781
# OTLP
738782
otlp_config:
739783
receiver:
@@ -801,6 +845,7 @@ extension_version: "compatibility"
801845
api_key: "test-api-key".to_string(),
802846
log_level: LogLevel::Debug,
803847
flush_timeout: 42,
848+
compression_level: 4,
804849
proxy_https: Some("https://proxy.example.com".to_string()),
805850
proxy_no_proxy: vec!["localhost".to_string(), "127.0.0.1".to_string()],
806851
http_protocol: Some("http1".to_string()),
@@ -831,7 +876,7 @@ extension_version: "compatibility"
831876
replace_placeholder: None,
832877
}]),
833878
logs_config_use_compression: false,
834-
logs_config_compression_level: 3,
879+
logs_config_compression_level: 1,
835880
logs_config_additional_endpoints: vec![LogsAdditionalEndpoint {
836881
api_key: "apikey2".to_string(),
837882
host: "agent-http-intake.logs.datadoghq.com".to_string(),
@@ -846,7 +891,7 @@ extension_version: "compatibility"
846891
apm_replace_tags: Some(vec![]),
847892
apm_config_obfuscation_http_remove_query_string: true,
848893
apm_config_obfuscation_http_remove_paths_with_digits: true,
849-
apm_config_compression_level: 3,
894+
apm_config_compression_level: 2,
850895
apm_features: vec![
851896
"enable_otlp_compute_top_level_by_span_kind".to_string(),
852897
"enable_stats_by_span_kind".to_string(),
@@ -866,6 +911,7 @@ extension_version: "compatibility"
866911
trace_propagation_extract_first: true,
867912
trace_propagation_http_baggage_enabled: true,
868913
trace_aws_service_representation_enabled: true,
914+
metrics_config_compression_level: 3,
869915
otlp_config_traces_enabled: false,
870916
otlp_config_traces_span_name_as_resource_name: true,
871917
otlp_config_traces_span_name_remappings: HashMap::from([(

‎bottlecap/src/traces/trace_processor.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -361,7 +361,7 @@ impl TraceProcessor for ServerlessTraceProcessor {
361361
};
362362

363363
let builder = SendDataBuilder::new(body_size, payload, header_tags, &endpoint)
364-
.with_compression(Compression::Zstd(6))
364+
.with_compression(Compression::Zstd(config.apm_config_compression_level))
365365
.with_retry_strategy(RetryStrategy::new(
366366
1,
367367
100,

0 commit comments

Comments
 (0)