ref(tests): Rework integration tests structure (#313)

* ref(tests): Rework structure of integration tests

* Improve

* Improve

* Improve

* Improve

* Improve

* Improve

* Improve

* Improve

* Improve

* Improve

* Improve

Author: iambriccardo

.github/workflows/ci.yml

@@ -88,13 +88,22 @@ jobs:
 
       - name: Run tests
         run: |
-          cargo test --workspace --no-fail-fast \
-            --all-features -- --skip integration::bigquery_test
+          cargo test \
+            --workspace \
+            --all-features \
+            --no-fail-fast \
+            --exclude etl-destinations \
+          && \
+          cargo test \
+            -p etl-destinations \
+            --no-default-features \
+            --no-fail-fast \
+            --features iceberg
 
   test-full:
     name: Test Full
     runs-on: ubuntu-latest
-    # Run on non forks.
+    # Run on non-forks.
     if: github.event.pull_request.head.repo.fork == false
     permissions:
       contents: read

etl-api/tests/common/mod.rs

@@ -1,69 +1,24 @@
-use etl_api::configs::destination::FullApiDestinationConfig;
 use etl_api::routes::destinations::{
     CreateDestinationRequest, CreateDestinationResponse, ReadDestinationResponse,
     ReadDestinationsResponse, UpdateDestinationRequest,
 };
 use etl_api::routes::pipelines::{CreatePipelineRequest, CreatePipelineResponse};
-use etl_config::SerializableSecretString;
 use etl_telemetry::tracing::init_test_tracing;
 use reqwest::StatusCode;
 
 use crate::{
-    common::test_app::{TestApp, spawn_test_app},
-    integration::{
-        images_test::create_default_image, pipelines_test::new_pipeline_config,
-        sources_test::create_source, tenants_test::create_tenant,
+    support::mocks::create_default_image,
+    support::mocks::destinations::{
+        create_destination, create_destination_with_config, new_destination_config, new_name,
+        updated_destination_config, updated_name,
     },
+    support::mocks::pipelines::new_pipeline_config,
+    support::mocks::sources::create_source,
+    support::mocks::tenants::create_tenant,
+    support::test_app::spawn_test_app,
 };
 
-pub fn new_name() -> String {
-    "BigQuery Destination".to_string()
-}
-
-pub fn new_destination_config() -> FullApiDestinationConfig {
-    FullApiDestinationConfig::BigQuery {
-        project_id: "project-id".to_string(),
-        dataset_id: "dataset-id".to_string(),
-        service_account_key: SerializableSecretString::from("service-account-key".to_string()),
-        max_staleness_mins: None,
-        max_concurrent_streams: Some(1),
-    }
-}
-
-pub fn updated_name() -> String {
-    "BigQuery Destination (Updated)".to_string()
-}
-
-pub fn updated_destination_config() -> FullApiDestinationConfig {
-    FullApiDestinationConfig::BigQuery {
-        project_id: "project-id-updated".to_string(),
-        dataset_id: "dataset-id-updated".to_string(),
-        service_account_key: SerializableSecretString::from(
-            "service-account-key-updated".to_string(),
-        ),
-        max_staleness_mins: Some(10),
-        max_concurrent_streams: Some(1),
-    }
-}
-
-pub async fn create_destination_with_config(
-    app: &TestApp,
-    tenant_id: &str,
-    name: String,
-    config: FullApiDestinationConfig,
-) -> i64 {
-    let destination = CreateDestinationRequest { name, config };
-    let response = app.create_destination(tenant_id, &destination).await;
-    let response: CreateDestinationResponse = response
-        .json()
-        .await
-        .expect("failed to deserialize response");
-    response.id
-}
-
-pub async fn create_destination(app: &TestApp, tenant_id: &str) -> i64 {
-    create_destination_with_config(app, tenant_id, new_name(), new_destination_config()).await
-}
+mod support;
 
 #[tokio::test(flavor = "multi_thread")]
 async fn destination_can_be_created() {
@@ -122,7 +77,7 @@ async fn an_existing_destination_can_be_read() {
 }
 
 #[tokio::test(flavor = "multi_thread")]
-async fn a_non_existing_destination_cant_be_read() {
+async fn non_existing_destination_cannot_be_read() {
     init_test_tracing();
     // Arrange
     let app = spawn_test_app().await;
@@ -176,7 +131,7 @@ async fn an_existing_destination_can_be_updated() {
 }
 
 #[tokio::test(flavor = "multi_thread")]
-async fn a_non_existing_destination_cant_be_updated() {
+async fn non_existing_destination_cannot_be_updated() {
     init_test_tracing();
     // Arrange
     let app = spawn_test_app().await;
@@ -221,7 +176,7 @@ async fn an_existing_destination_can_be_deleted() {
 }
 
 #[tokio::test(flavor = "multi_thread")]
-async fn a_non_existing_destination_cant_be_deleted() {
+async fn non_existing_destination_cannot_be_deleted() {
     init_test_tracing();
     // Arrange
     let app = spawn_test_app().await;

etl-api/tests/integration/destination_test.rs renamed to etl-api/tests/destinations.rs

@@ -7,19 +7,23 @@ use etl_api::routes::pipelines::{CreatePipelineRequest, ReadPipelineResponse};
 use etl_telemetry::tracing::init_test_tracing;
 use reqwest::StatusCode;
 
-use crate::common::database::{create_test_source_database, run_etl_migrations_on_source_database};
+use crate::support::database::{
+    create_test_source_database, run_etl_migrations_on_source_database,
+};
 use crate::{
-    common::test_app::spawn_test_app,
-    integration::destination_test::{
+    support::mocks::create_default_image,
+    support::mocks::destinations::{
         create_destination, new_destination_config, new_name, updated_destination_config,
         updated_name,
     },
-    integration::images_test::create_default_image,
-    integration::pipelines_test::{new_pipeline_config, updated_pipeline_config},
-    integration::sources_test::create_source,
-    integration::tenants_test::{create_tenant, create_tenant_with_id_and_name},
+    support::mocks::pipelines::{new_pipeline_config, updated_pipeline_config},
+    support::mocks::sources::create_source,
+    support::mocks::tenants::{create_tenant, create_tenant_with_id_and_name},
+    support::test_app::spawn_test_app,
 };
 
+mod support;
+
 #[tokio::test(flavor = "multi_thread")]
 async fn destination_and_pipeline_can_be_created() {
     init_test_tracing();
@@ -75,7 +79,7 @@ async fn destination_and_pipeline_can_be_created() {
 }
 
 #[tokio::test(flavor = "multi_thread")]
-async fn destination_and_pipeline_with_another_tenants_source_cant_be_created() {
+async fn destination_and_pipeline_with_another_tenants_source_cannot_be_created() {
     init_test_tracing();
     // Arrange
     let app = spawn_test_app().await;
@@ -177,7 +181,7 @@ async fn an_existing_destination_and_pipeline_can_be_updated() {
 }
 
 #[tokio::test(flavor = "multi_thread")]
-async fn destination_and_pipeline_with_another_tenants_source_cant_be_updated() {
+async fn destination_and_pipeline_with_another_tenants_source_cannot_be_updated() {
     init_test_tracing();
     // Arrange
     let app = spawn_test_app().await;
@@ -236,7 +240,7 @@ async fn destination_and_pipeline_with_another_tenants_source_cant_be_updated()
 }
 
 #[tokio::test(flavor = "multi_thread")]
-async fn destination_and_pipeline_with_another_tenants_destination_cant_be_updated() {
+async fn destination_and_pipeline_with_another_tenants_destination_cannot_be_updated() {
     init_test_tracing();
     // Arrange
     let app = spawn_test_app().await;
@@ -292,7 +296,7 @@ async fn destination_and_pipeline_with_another_tenants_destination_cant_be_updat
 }
 
 #[tokio::test(flavor = "multi_thread")]
-async fn destination_and_pipeline_with_another_tenants_pipeline_cant_be_updated() {
+async fn destination_and_pipeline_with_another_tenants_pipeline_cannot_be_updated() {
     init_test_tracing();
     // Arrange
     let app = spawn_test_app().await;
@@ -369,7 +373,7 @@ async fn destination_and_pipeline_with_another_tenants_pipeline_cant_be_updated(
 }
 
 #[tokio::test(flavor = "multi_thread")]
-async fn duplicate_destination_pipeline_with_same_source_cant_be_created() {
+async fn duplicate_destination_pipeline_with_same_source_cannot_be_created() {
     init_test_tracing();
     // Arrange
     let app = spawn_test_app().await;

etl-api/tests/integration/destinations_pipelines_test.rs renamed to etl-api/tests/destinations_pipelines.rs

@@ -1,9 +1,11 @@
 use etl_telemetry::tracing::init_test_tracing;
 
-use crate::common::test_app::spawn_test_app;
+use crate::support::test_app::spawn_test_app;
+
+mod support;
 
 #[tokio::test(flavor = "multi_thread")]
-async fn health_check_works() {
+async fn health_check_returns_200() {
     init_test_tracing();
     // Arrange
     let app = spawn_test_app().await;

etl-api/tests/integration/health_check_test.rs renamed to etl-api/tests/health_check.rs

@@ -5,21 +5,10 @@ use etl_api::routes::images::{
 use etl_telemetry::tracing::init_test_tracing;
 use reqwest::StatusCode;
 
-use crate::common::test_app::{TestApp, spawn_test_app};
+use crate::support::mocks::create_image_with_name;
+use crate::support::test_app::spawn_test_app;
 
-pub async fn create_default_image(app: &TestApp) -> i64 {
-    create_image_with_name(app, "some/image".to_string(), true).await
-}
-
-pub async fn create_image_with_name(app: &TestApp, name: String, is_default: bool) -> i64 {
-    let image = CreateImageRequest { name, is_default };
-    let response = app.create_image(&image).await;
-    let response: CreateImageResponse = response
-        .json()
-        .await
-        .expect("failed to deserialize response");
-    response.id
-}
+mod support;
 
 #[tokio::test(flavor = "multi_thread")]
 async fn image_can_be_created() {
@@ -77,7 +66,7 @@ async fn an_existing_image_can_be_read() {
 }
 
 #[tokio::test(flavor = "multi_thread")]
-async fn a_non_existing_image_cant_be_read() {
+async fn non_existing_image_cannot_be_read() {
     init_test_tracing();
     // Arrange
     let app = spawn_test_app().await;
@@ -130,7 +119,7 @@ async fn an_existing_image_can_be_updated() {
 }
 
 #[tokio::test(flavor = "multi_thread")]
-async fn a_non_existing_source_cant_be_updated() {
+async fn non_existing_image_cannot_be_updated() {
     init_test_tracing();
     // Arrange
     let app = spawn_test_app().await;
@@ -177,7 +166,7 @@ async fn an_existing_image_can_be_deleted() {
 }
 
 #[tokio::test(flavor = "multi_thread")]
-async fn a_non_existing_image_cant_be_deleted() {
+async fn non_existing_image_cannot_be_deleted() {
     init_test_tracing();
     // Arrange
     let app = spawn_test_app().await;

etl-api/tests/integration/images_test.rs renamed to etl-api/tests/images.rs

@@ -1,9 +1,11 @@
 use etl_telemetry::tracing::init_test_tracing;
 
-use crate::common::test_app::spawn_test_app;
+use crate::support::test_app::spawn_test_app;
+
+mod support;
 
 #[tokio::test(flavor = "multi_thread")]
-async fn metrics_endpoint_works() {
+async fn metrics_endpoint_returns_200() {
     init_test_tracing();
     // Arrange
     let app = spawn_test_app().await;

etl-api/tests/integration/mod.rs

@@ -7,11 +7,13 @@ use etl_telemetry::tracing::init_test_tracing;
 use reqwest::StatusCode;
 use sqlx::PgPool;
 
-use crate::common::database::create_test_source_database;
-use crate::common::test_app::{TestApp, spawn_test_app};
-use crate::integration::destination_test::create_destination;
-use crate::integration::images_test::create_default_image;
-use crate::integration::tenants_test::create_tenant;
+use crate::support::database::create_test_source_database;
+use crate::support::mocks::create_default_image;
+use crate::support::mocks::destinations::create_destination;
+use crate::support::mocks::tenants::create_tenant;
+use crate::support::test_app::{TestApp, spawn_test_app};
+
+mod support;
 
 async fn create_pipeline_with_unmigrated_source_db(
     app: &TestApp,
@@ -25,7 +27,7 @@ async fn create_pipeline_with_unmigrated_source_db(
     let req = CreatePipelineRequest {
         source_id,
         destination_id,
-        config: crate::integration::pipelines_test::new_pipeline_config(),
+        config: crate::support::mocks::pipelines::new_pipeline_config(),
     };
 
     let response = app.create_pipeline(tenant_id, &req).await;