diff --git a/bottlecap/Cargo.lock b/bottlecap/Cargo.lock index bb35bc136..82c3a5192 100644 --- a/bottlecap/Cargo.lock +++ b/bottlecap/Cargo.lock @@ -624,7 +624,7 @@ dependencies = [ [[package]] name = "datadog-protos" version = "0.1.0" -source = "git+https://github.com/DataDog/saluki-backport/?rev=3c5d87ab82dea4a1a98ef0c60fb3659ca35c2751#3c5d87ab82dea4a1a98ef0c60fb3659ca35c2751" +source = "git+https://github.com/DataDog/saluki/?rev=c89b58e5784b985819baf11f13f7d35876741222#c89b58e5784b985819baf11f13f7d35876741222" dependencies = [ "bytes", "prost 0.13.4", @@ -650,7 +650,7 @@ dependencies = [ [[package]] name = "datadog-trace-mini-agent" version = "14.3.1" -source = "git+https://github.com/DataDog/libdatadog?rev=dbc8869c6de4ccd78a770524d250420f397fc7b3#dbc8869c6de4ccd78a770524d250420f397fc7b3" +source = "git+https://github.com/DataDog/libdatadog?rev=6e2d4462873e35e49a2c9befde958d96c4ed9de4#6e2d4462873e35e49a2c9befde958d96c4ed9de4" dependencies = [ "anyhow", "async-trait", @@ -669,7 +669,7 @@ dependencies = [ [[package]] name = "datadog-trace-normalization" version = "14.3.1" -source = "git+https://github.com/DataDog/libdatadog?rev=dbc8869c6de4ccd78a770524d250420f397fc7b3#dbc8869c6de4ccd78a770524d250420f397fc7b3" +source = "git+https://github.com/DataDog/libdatadog?rev=6e2d4462873e35e49a2c9befde958d96c4ed9de4#6e2d4462873e35e49a2c9befde958d96c4ed9de4" dependencies = [ "anyhow", "datadog-trace-protobuf", @@ -678,7 +678,7 @@ dependencies = [ [[package]] name = "datadog-trace-obfuscation" version = "14.3.1" -source = "git+https://github.com/DataDog/libdatadog?rev=dbc8869c6de4ccd78a770524d250420f397fc7b3#dbc8869c6de4ccd78a770524d250420f397fc7b3" +source = "git+https://github.com/DataDog/libdatadog?rev=6e2d4462873e35e49a2c9befde958d96c4ed9de4#6e2d4462873e35e49a2c9befde958d96c4ed9de4" dependencies = [ "anyhow", "datadog-trace-protobuf", @@ -695,7 +695,7 @@ dependencies = [ [[package]] name = "datadog-trace-protobuf" version = "14.3.1" -source = "git+https://github.com/DataDog/libdatadog?rev=dbc8869c6de4ccd78a770524d250420f397fc7b3#dbc8869c6de4ccd78a770524d250420f397fc7b3" +source = "git+https://github.com/DataDog/libdatadog?rev=6e2d4462873e35e49a2c9befde958d96c4ed9de4#6e2d4462873e35e49a2c9befde958d96c4ed9de4" dependencies = [ "prost 0.11.9", "serde", @@ -705,7 +705,7 @@ dependencies = [ [[package]] name = "datadog-trace-utils" version = "14.3.1" -source = "git+https://github.com/DataDog/libdatadog?rev=dbc8869c6de4ccd78a770524d250420f397fc7b3#dbc8869c6de4ccd78a770524d250420f397fc7b3" +source = "git+https://github.com/DataDog/libdatadog?rev=6e2d4462873e35e49a2c9befde958d96c4ed9de4#6e2d4462873e35e49a2c9befde958d96c4ed9de4" dependencies = [ "anyhow", "bytes", @@ -732,7 +732,7 @@ dependencies = [ [[package]] name = "ddcommon" version = "14.3.1" -source = "git+https://github.com/DataDog/libdatadog?rev=dbc8869c6de4ccd78a770524d250420f397fc7b3#dbc8869c6de4ccd78a770524d250420f397fc7b3" +source = "git+https://github.com/DataDog/libdatadog?rev=6e2d4462873e35e49a2c9befde958d96c4ed9de4#6e2d4462873e35e49a2c9befde958d96c4ed9de4" dependencies = [ "anyhow", "cc", @@ -761,10 +761,10 @@ dependencies = [ [[package]] name = "ddsketch-agent" version = "0.1.0" -source = "git+https://github.com/DataDog/saluki-backport/?rev=3c5d87ab82dea4a1a98ef0c60fb3659ca35c2751#3c5d87ab82dea4a1a98ef0c60fb3659ca35c2751" +source = "git+https://github.com/DataDog/saluki/?rev=c89b58e5784b985819baf11f13f7d35876741222#c89b58e5784b985819baf11f13f7d35876741222" dependencies = [ - "datadog-protos 0.1.0 (git+https://github.com/DataDog/saluki-backport/?rev=3c5d87ab82dea4a1a98ef0c60fb3659ca35c2751)", - "float_eq", + "datadog-protos 0.1.0 (git+https://github.com/DataDog/saluki/?rev=c89b58e5784b985819baf11f13f7d35876741222)", + "float-cmp", "ordered-float", "smallvec", ] @@ -780,6 +780,27 @@ dependencies = [ "smallvec", ] +[[package]] +name = "derive_more" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", + "unicode-xid", +] + [[package]] name = "digest" version = "0.10.7" @@ -826,10 +847,11 @@ dependencies = [ [[package]] name = "dogstatsd" version = "14.3.1" -source = "git+https://github.com/DataDog/libdatadog?rev=dbc8869c6de4ccd78a770524d250420f397fc7b3#dbc8869c6de4ccd78a770524d250420f397fc7b3" +source = "git+https://github.com/DataDog/libdatadog?rev=6e2d4462873e35e49a2c9befde958d96c4ed9de4#6e2d4462873e35e49a2c9befde958d96c4ed9de4" dependencies = [ - "datadog-protos 0.1.0 (git+https://github.com/DataDog/saluki-backport/?rev=3c5d87ab82dea4a1a98ef0c60fb3659ca35c2751)", - "ddsketch-agent 0.1.0 (git+https://github.com/DataDog/saluki-backport/?rev=3c5d87ab82dea4a1a98ef0c60fb3659ca35c2751)", + "datadog-protos 0.1.0 (git+https://github.com/DataDog/saluki/?rev=c89b58e5784b985819baf11f13f7d35876741222)", + "ddsketch-agent 0.1.0 (git+https://github.com/DataDog/saluki/?rev=c89b58e5784b985819baf11f13f7d35876741222)", + "derive_more", "fnv", "hashbrown 0.14.5", "lazy_static", @@ -956,12 +978,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "float_eq" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28a80e3145d8ad11ba0995949bbcf48b9df2be62772b3d351ef017dff6ecb853" - [[package]] name = "fnv" version = "1.0.7" @@ -3105,7 +3121,7 @@ dependencies = [ [[package]] name = "tinybytes" version = "14.3.1" -source = "git+https://github.com/DataDog/libdatadog?rev=dbc8869c6de4ccd78a770524d250420f397fc7b3#dbc8869c6de4ccd78a770524d250420f397fc7b3" +source = "git+https://github.com/DataDog/libdatadog?rev=6e2d4462873e35e49a2c9befde958d96c4ed9de4#6e2d4462873e35e49a2c9befde958d96c4ed9de4" dependencies = [ "serde", ] diff --git a/bottlecap/Cargo.toml b/bottlecap/Cargo.toml index cca65ca2c..6e9cb9434 100644 --- a/bottlecap/Cargo.toml +++ b/bottlecap/Cargo.toml @@ -9,13 +9,13 @@ async-trait = { version = "0.1", default-features = false } chrono = { version = "0.4", features = ["serde", "std", "now"], default-features = false } datadog-protos = { version = "0.1.0", default-features = false, git = "https://github.com/DataDog/saluki/" } ddsketch-agent = { version = "0.1.0", default-features = false, git = "https://github.com/DataDog/saluki/" } -ddcommon = { git = "https://github.com/DataDog/libdatadog", rev = "dbc8869c6de4ccd78a770524d250420f397fc7b3" } -datadog-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "dbc8869c6de4ccd78a770524d250420f397fc7b3" } -datadog-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "dbc8869c6de4ccd78a770524d250420f397fc7b3" } -datadog-trace-mini-agent = { git = "https://github.com/DataDog/libdatadog", rev = "dbc8869c6de4ccd78a770524d250420f397fc7b3" } -datadog-trace-normalization = { git = "https://github.com/DataDog/libdatadog", rev = "dbc8869c6de4ccd78a770524d250420f397fc7b3" } -datadog-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "dbc8869c6de4ccd78a770524d250420f397fc7b3" } -dogstatsd = { git = "https://github.com/DataDog/libdatadog", rev = "dbc8869c6de4ccd78a770524d250420f397fc7b3" } +ddcommon = { git = "https://github.com/DataDog/libdatadog", rev = "6e2d4462873e35e49a2c9befde958d96c4ed9de4" } +datadog-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "6e2d4462873e35e49a2c9befde958d96c4ed9de4" } +datadog-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "6e2d4462873e35e49a2c9befde958d96c4ed9de4" } +datadog-trace-mini-agent = { git = "https://github.com/DataDog/libdatadog", rev = "6e2d4462873e35e49a2c9befde958d96c4ed9de4" } +datadog-trace-normalization = { git = "https://github.com/DataDog/libdatadog", rev = "6e2d4462873e35e49a2c9befde958d96c4ed9de4" } +datadog-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "6e2d4462873e35e49a2c9befde958d96c4ed9de4" } +dogstatsd = { git = "https://github.com/DataDog/libdatadog", rev = "6e2d4462873e35e49a2c9befde958d96c4ed9de4" } figment = { version = "0.10", default-features = false, features = ["yaml", "env"] } hyper = { version = "0.14", default-features = false, features = ["server"] } lazy_static = { version = "1.5", default-features = false } diff --git a/bottlecap/src/bin/bottlecap/main.rs b/bottlecap/src/bin/bottlecap/main.rs index 91d399a62..160a6001a 100644 --- a/bottlecap/src/bin/bottlecap/main.rs +++ b/bottlecap/src/bin/bottlecap/main.rs @@ -46,8 +46,9 @@ use decrypt::resolve_secrets; use dogstatsd::{ aggregator::Aggregator as MetricsAggregator, constants::CONTEXTS, + datadog::{DdDdUrl, DdUrl, MetricsIntakeUrlPrefix, MetricsIntakeUrlPrefixOverride, Site}, dogstatsd::{DogStatsD, DogStatsDConfig}, - flusher::{build_fqdn_metrics, Flusher as MetricsFlusher}, + flusher::{Flusher as MetricsFlusher, FlusherConfig as MetricsFlusherConfig}, metric::{SortedTags, EMPTY_TAGS}, }; use reqwest::Client; @@ -163,6 +164,41 @@ fn build_function_arn(account_id: &str, region: &str, function_name: &str) -> St async fn main() -> Result<()> { let (aws_config, config) = load_configs(); + let site = Site::new(config.site.clone()).map_err(|e| { + Error::new( + std::io::ErrorKind::InvalidData, + format!("Failed to parse DD_SITE: {e:?}"), + ) + })?; + let dd_url = match config.url.clone() { + Some(dd_url) => Some(DdUrl::new(dd_url).map_err(|e| { + Error::new( + std::io::ErrorKind::InvalidData, + format!("Failed to parse DD_URL: {e:?}"), + ) + })?), + None => None, + }; + let dd_dd_url = match config.dd_url.clone() { + Some(dd_dd_url) => Some(DdDdUrl::new(dd_dd_url).map_err(|e| { + Error::new( + std::io::ErrorKind::InvalidData, + format!("Failed to parse DD_DD_URL: {e:?}"), + ) + })?), + None => None, + }; + let metrics_intake_url_prefix = MetricsIntakeUrlPrefix::new( + Some(site), + MetricsIntakeUrlPrefixOverride::maybe_new(dd_url, dd_dd_url), + ) + .map_err(|e| { + Error::new( + std::io::ErrorKind::InvalidData, + format!("Failed to create intake url prefix: {e:?}"), + ) + })?; + enable_logging_subsystem(&config); let client = reqwest::Client::builder().no_proxy().build().map_err(|e| { Error::new( @@ -176,7 +212,16 @@ async fn main() -> Result<()> { .map_err(|e| Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?; if let Some(resolved_api_key) = resolve_secrets(Arc::clone(&config), &aws_config).await { - match extension_loop_active(&aws_config, &config, &client, &r, resolved_api_key).await { + match extension_loop_active( + &aws_config, + &config, + &client, + &r, + resolved_api_key, + metrics_intake_url_prefix, + ) + .await + { Ok(()) => { debug!("Extension loop completed successfully"); Ok(()) @@ -260,6 +305,7 @@ async fn extension_loop_active( client: &Client, r: &RegisterResponse, resolved_api_key: String, + metrics_intake_url_prefix: MetricsIntakeUrlPrefix, ) -> Result<()> { let mut event_bus = EventBus::run(); @@ -284,13 +330,13 @@ async fn extension_loop_active( ) .expect("failed to create aggregator"), )); - let mut metrics_flusher = MetricsFlusher::new( - resolved_api_key.clone(), - Arc::clone(&metrics_aggr), - build_fqdn_metrics(config.site.clone()), - config.https_proxy.clone(), - Duration::from_secs(config.flush_timeout), - ); + let mut metrics_flusher = MetricsFlusher::new(MetricsFlusherConfig { + api_key: resolved_api_key.clone(), + aggregator: Arc::clone(&metrics_aggr), + metrics_intake_url_prefix, + https_proxy: config.https_proxy.clone(), + timeout: Duration::from_secs(config.flush_timeout), + }); let trace_flusher = Arc::new(trace_flusher::ServerlessTraceFlusher { buffer: Arc::new(TokioMutex::new(Vec::new())), diff --git a/bottlecap/src/config/mod.rs b/bottlecap/src/config/mod.rs index 598ba1e21..d44b4b84f 100644 --- a/bottlecap/src/config/mod.rs +++ b/bottlecap/src/config/mod.rs @@ -39,8 +39,6 @@ pub struct FallbackConfig { otlp_config_receiver_protocols_http_endpoint: Option, otlp_config_receiver_protocols_grpc_endpoint: Option, // intake urls - url: Option, - dd_url: Option, logs_config_logs_dd_url: Option, // APM, as opposed to logs, does not use the `apm_config` prefix for env vars apm_dd_url: Option, @@ -107,6 +105,11 @@ pub struct Config { pub trace_propagation_style_extract: Vec, pub trace_propagation_extract_first: bool, pub trace_propagation_http_baggage_enabled: bool, + // Intake URLs + // These two are for metrics intake. The official config is `DD_DD_URL` but `DD_URL` can also + // be used for backwards compatibility. `DD_DD_URL` takes precedence. + pub url: Option, + pub dd_url: Option, } impl Default for Config { @@ -142,6 +145,9 @@ impl Default for Config { trace_propagation_style_extract: vec![], trace_propagation_extract_first: false, trace_propagation_http_baggage_enabled: false, + // Intake URLs + url: None, + dd_url: None, } } } @@ -208,9 +214,7 @@ fn fallback(figment: &Figment, yaml_figment: &Figment) -> Result<(), ConfigError } // Intake URLs - if config.url.is_some() - || config.dd_url.is_some() - || config.logs_config_logs_dd_url.is_some() + if config.logs_config_logs_dd_url.is_some() || config.apm_dd_url.is_some() || yaml_config .logs_config diff --git a/bottlecap/tests/metrics_integration_test.rs b/bottlecap/tests/metrics_integration_test.rs index 8bca4362a..26ea34483 100644 --- a/bottlecap/tests/metrics_integration_test.rs +++ b/bottlecap/tests/metrics_integration_test.rs @@ -1,7 +1,8 @@ use bottlecap::config::Config; use bottlecap::metrics::enhanced::lambda::Lambda as enhanced_metrics; use dogstatsd::aggregator::Aggregator as MetricsAggregator; -use dogstatsd::flusher::Flusher as MetricsFlusher; +use dogstatsd::datadog::{DdDdUrl, MetricsIntakeUrlPrefix, MetricsIntakeUrlPrefixOverride}; +use dogstatsd::flusher::{Flusher as MetricsFlusher, FlusherConfig as MetricsFlusherConfig}; use dogstatsd::metric::SortedTags; use httpmock::prelude::*; use std::sync::{Arc, Mutex}; @@ -36,13 +37,20 @@ async fn test_enhanced_metrics() { MetricsAggregator::new(SortedTags::parse("aTagKey:aTagValue").unwrap(), 1024) .expect("failed to create aggregator"), )); - let mut metrics_flusher = MetricsFlusher::new( - dd_api_key.to_string(), - metrics_aggr.clone(), - server.base_url(), - None, - std::time::Duration::from_secs(5), - ); + let mut metrics_flusher = MetricsFlusher::new(MetricsFlusherConfig { + api_key: dd_api_key.to_string(), + aggregator: metrics_aggr.clone(), + metrics_intake_url_prefix: MetricsIntakeUrlPrefix::new( + None, + MetricsIntakeUrlPrefixOverride::maybe_new( + None, + Some(DdDdUrl::new(server.base_url()).expect("failed to create dd url")), + ), + ) + .expect("failed to create metrics intake url prefix"), + https_proxy: None, + timeout: std::time::Duration::from_secs(5), + }); let lambda_enhanced_metrics = enhanced_metrics::new(Arc::clone(&metrics_aggr), Arc::clone(&arc_config));