feat: use docker cli instead of relying on testcontainers (#1224)

* feat: use docker cli instead of relying on tst containers

# Motivations
Both testcontainers and httpmock have a lot of dependencies which we don't use elsewhere (some of them duplicates with incompatible versions).

This makes building tests quite a bit slower...

Also since testcontainer uses the docker sock, which is not available by default on mac, running the tests on a brand new mac install requires some extra setup.

In total this PR drops 60 tests dependencies on the data-pipeline crare

# Changes

* testcontainers can be replaced by running a few command with the docker cli directly,
* httpmock has been updated to the latest version, which uses hyper 1.0 instead of 0.14, which means deps are reusable now

Author: paullegranddc

Cargo.lock

@@ -43,7 +43,7 @@ members = [
   "tinybytes",
   "dogstatsd-client",
   "datadog-log",
-  "datadog-log-ffi"
+  "datadog-log-ffi",
 ]
 
 # https://doc.rust-lang.org/cargo/reference/resolver.html
@@ -65,6 +65,17 @@ edition = "2021"
 version = "21.0.0"
 license = "Apache-2.0"
 
+[workspace.dependencies]
+hyper = { version = "1.6", features = [
+  "http1",
+  "client",
+], default-features = false }
+hyper-util = { version = "0.1.10", features = [
+  "http1",
+  "client",
+  "client-legacy",
+] }
+
 [profile.dev]
 debug = 2 # full debug info
 

Cargo.toml

@@ -23,7 +23,7 @@ cbindgen = ["build_common/cbindgen", "ddcommon-ffi/cbindgen"]
 build_common = { path = "../build-common" }
 
 [dev-dependencies]
-httpmock = "0.7.0"
+httpmock = "0.8.0-alpha.1"
 rmp-serde = "1.1.1"
 datadog-trace-utils = { path = "../datadog-trace-utils" }
 

LICENSE-3rdparty.yml

@@ -1080,8 +1080,8 @@ mod tests {
             let mock_metrics = server.mock(|when, then| {
                 when.method(POST)
                     .path("/telemetry/proxy/api/v2/apmtelemetry")
-                    .body_contains(r#""runtime_id":"foo""#)
-                    .body_contains(r#""metric":"trace_api."#);
+                    .body_includes(r#""runtime_id":"foo""#)
+                    .body_includes(r#""metric":"trace_api."#);
                 then.status(200)
                     .header("content-type", "application/json")
                     .body("");
@@ -1134,7 +1134,7 @@ mod tests {
 
             ddog_trace_exporter_free(exporter);
             // It should receive 1 metrics payload (excluding heartbeats)
-            mock_metrics.assert_hits(1);
+            mock_metrics.assert_calls(1);
         }
     }
 

data-pipeline-ffi/Cargo.toml

@@ -12,8 +12,8 @@ autobenches = false
 [dependencies]
 anyhow = { version = "1.0" }
 arc-swap = "1.7.1"
-hyper = { version = "1.6", features = ["http1", "client"] }
-hyper-util = { version = "0.1", features = ["client", "client-legacy"] }
+hyper = { workspace = true }
+hyper-util = { workspace = true }
 http = "1.0"
 http-body-util = "0.1"
 tracing = { version = "0.1", default-features = false }
@@ -55,7 +55,7 @@ criterion = "0.5.1"
 datadog-trace-utils = { path = "../datadog-trace-utils", features = [
     "test-utils",
 ] }
-httpmock = "0.7.0"
+httpmock = "0.8.0-alpha.1"
 rand = "0.8.5"
 regex = "1.5"
 tempfile = "3.3.0"

data-pipeline-ffi/src/trace_exporter.rs

@@ -382,7 +382,7 @@ mod single_threaded_tests {
             .await
             .unwrap();
 
-        mock.assert_hits(2);
+        mock.assert_calls(2);
         assert!(
             matches!(new_state_info_status, FetchInfoStatus::NewState(info) if *info == AgentInfo {
                         state_hash: TEST_INFO_HASH.to_string(),
@@ -466,7 +466,7 @@ mod single_threaded_tests {
             .await;
 
         // Wait for second fetch
-        while mock_v2.hits_async().await == 0 {
+        while mock_v2.calls_async().await == 0 {
             tokio::time::sleep(Duration::from_millis(100)).await;
         }
 
@@ -533,13 +533,13 @@ mod single_threaded_tests {
         const SLEEP_DURATION_MS: u64 = 10;
 
         let mut attempts = 0;
-        while mock.hits_async().await == 0 && attempts < MAX_ATTEMPTS {
+        while mock.calls_async().await == 0 && attempts < MAX_ATTEMPTS {
             attempts += 1;
             tokio::time::sleep(Duration::from_millis(SLEEP_DURATION_MS)).await;
         }
 
         // Should trigger a fetch since the state is different
-        mock.assert_hits_async(1).await;
+        mock.assert_calls_async(1).await;
 
         // Wait for the cache to be updated with proper timeout
         let mut attempts = 0;
@@ -616,6 +616,6 @@ mod single_threaded_tests {
         tokio::time::sleep(Duration::from_millis(500)).await;
 
         // Should not trigger a fetch since the state is the same
-        mock.assert_hits_async(0).await;
+        mock.assert_calls_async(0).await;
     }
 }

data-pipeline/Cargo.toml

@@ -253,7 +253,7 @@ mod tests {
                 when.method(POST)
                     .header("Content-type", "application/msgpack")
                     .path("/v0.6/stats")
-                    .body_contains("libdatadog-test");
+                    .body_includes("libdatadog-test");
                 then.status(200).body("");
             })
             .await;
@@ -312,7 +312,7 @@ mod tests {
                 when.method(POST)
                     .header("Content-type", "application/msgpack")
                     .path("/v0.6/stats")
-                    .body_contains("libdatadog-test");
+                    .body_includes("libdatadog-test");
                 then.status(200).body("");
             })
             .await;
@@ -349,7 +349,7 @@ mod tests {
                 when.method(POST)
                     .header("Content-type", "application/msgpack")
                     .path("/v0.6/stats")
-                    .body_contains("libdatadog-test");
+                    .body_includes("libdatadog-test");
                 then.status(200).body("");
             })
             .await;

data-pipeline/src/agent_info/fetcher.rs

@@ -368,10 +368,10 @@ mod tests {
         client.start().await;
         let _ = client.send(&data);
         client.shutdown().await;
-        while telemetry_srv.hits_async().await == 0 {
+        while telemetry_srv.calls_async().await == 0 {
             sleep(Duration::from_millis(10)).await;
         }
-        telemetry_srv.assert_hits_async(1).await;
+        telemetry_srv.assert_calls_async(1).await;
     }
 
     #[cfg_attr(miri, ignore)]
@@ -397,10 +397,10 @@ mod tests {
         client.start().await;
         let _ = client.send(&data);
         client.shutdown().await;
-        while telemetry_srv.hits_async().await == 0 {
+        while telemetry_srv.calls_async().await == 0 {
             sleep(Duration::from_millis(10)).await;
         }
-        telemetry_srv.assert_hits_async(1).await;
+        telemetry_srv.assert_calls_async(1).await;
     }
 
     #[cfg_attr(miri, ignore)]
@@ -426,10 +426,10 @@ mod tests {
         client.start().await;
         let _ = client.send(&data);
         client.shutdown().await;
-        while telemetry_srv.hits_async().await == 0 {
+        while telemetry_srv.calls_async().await == 0 {
             sleep(Duration::from_millis(10)).await;
         }
-        telemetry_srv.assert_hits_async(1).await;
+        telemetry_srv.assert_calls_async(1).await;
     }
 
     #[cfg_attr(miri, ignore)]
@@ -455,10 +455,10 @@ mod tests {
         client.start().await;
         let _ = client.send(&data);
         client.shutdown().await;
-        while telemetry_srv.hits_async().await == 0 {
+        while telemetry_srv.calls_async().await == 0 {
             sleep(Duration::from_millis(10)).await;
         }
-        telemetry_srv.assert_hits_async(1).await;
+        telemetry_srv.assert_calls_async(1).await;
     }
 
     #[cfg_attr(miri, ignore)]
@@ -484,10 +484,10 @@ mod tests {
         client.start().await;
         let _ = client.send(&data);
         client.shutdown().await;
-        while telemetry_srv.hits_async().await == 0 {
+        while telemetry_srv.calls_async().await == 0 {
             sleep(Duration::from_millis(10)).await;
         }
-        telemetry_srv.assert_hits_async(1).await;
+        telemetry_srv.assert_calls_async(1).await;
     }
 
     #[cfg_attr(miri, ignore)]
@@ -513,10 +513,10 @@ mod tests {
         client.start().await;
         let _ = client.send(&data);
         client.shutdown().await;
-        while telemetry_srv.hits_async().await == 0 {
+        while telemetry_srv.calls_async().await == 0 {
             sleep(Duration::from_millis(10)).await;
         }
-        telemetry_srv.assert_hits_async(1).await;
+        telemetry_srv.assert_calls_async(1).await;
     }
 
     #[cfg_attr(miri, ignore)]
@@ -542,10 +542,10 @@ mod tests {
         client.start().await;
         let _ = client.send(&data);
         client.shutdown().await;
-        while telemetry_srv.hits_async().await == 0 {
+        while telemetry_srv.calls_async().await == 0 {
             sleep(Duration::from_millis(10)).await;
         }
-        telemetry_srv.assert_hits_async(1).await;
+        telemetry_srv.assert_calls_async(1).await;
     }
 
     #[cfg_attr(miri, ignore)]
@@ -571,10 +571,10 @@ mod tests {
         client.start().await;
         let _ = client.send(&data);
         client.shutdown().await;
-        while telemetry_srv.hits_async().await == 0 {
+        while telemetry_srv.calls_async().await == 0 {
             sleep(Duration::from_millis(10)).await;
         }
-        telemetry_srv.assert_hits_async(1).await;
+        telemetry_srv.assert_calls_async(1).await;
     }
 
     #[test]
@@ -698,7 +698,7 @@ mod tests {
 
         let telemetry_srv = server
             .mock_async(|when, then| {
-                when.method(POST).body_contains(r#""runtime_id":"foo""#);
+                when.method(POST).body_includes(r#""runtime_id":"foo""#);
                 then.status(200).body("");
             })
             .await;
@@ -765,10 +765,10 @@ mod tests {
             })
             .unwrap();
         client.shutdown().await;
-        while telemetry_srv.hits_async().await == 0 {
+        while telemetry_srv.calls_async().await == 0 {
             sleep(Duration::from_millis(10)).await;
         }
         // One payload generate-metrics
-        telemetry_srv.assert_hits_async(1).await;
+        telemetry_srv.assert_calls_async(1).await;
     }
 }