Ignore test that is failing for external reasons

Signed-off-by: Ryan Levick <[email protected]>

Author: rylev

.github/actions/spin-ci-dependencies/action.yml

@@ -72,6 +72,12 @@ inputs:
     required: false
     type: string
 
+  docker: 
+    description: 'setup docker'
+    required: false
+    default: 'false'
+    type: bool
+
 runs:
   using: "composite"
   steps:

.github/workflows/build.yml

@@ -48,7 +48,7 @@ jobs:
         if: failure()
         uses: andymckay/[email protected]
 
-  ## This is separated out to remove e2e-tests dependency on windows/mac builds
+  ## This is separated out to remove full integration tests dependencies on windows/mac builds
   build-rust-ubuntu:
     name: Build Spin Ubuntu
     runs-on: ubuntu-latest
@@ -157,11 +157,15 @@ jobs:
           rust-cache: true
           nomad: true
 
-      - name: Cargo Unit and Integration Tests
+      - name: Run Unit Tests
         run: |
-          make test-unit test-integration
-        env:
-          CARGO_INCREMENTAL: 0
+          make test-unit
+
+      - name: Run Integration Tests
+        run: |
+          make test-integration
+        # Only run integration tests on macOS as they will be run on ubuntu separately
+        if: ${{ matrix.runner == 'macos-latest' }}
 
   test-go:
     name: Test Spin SDK - Go
@@ -193,7 +197,7 @@ jobs:
       - name: "Test Go SDK"
         run: make test-sdk-go
 
-  e2e-tests:
+  all-integration-tests:
     # run on a larger runner for more SSD/resource access
     runs-on: ubuntu-22.04-4core-spin
     if: ${{ github.repository_owner == 'fermyon' }}
@@ -225,6 +229,6 @@ jobs:
           mv grain-linux-x64 grain
           echo "$PWD" >> $GITHUB_PATH
 
-      - name: Run e2e tests
+      - name: Run Full Integration Tests
         run: |
-          make test-spin-up
+          make test-integration-full

Cargo.lock

@@ -108,8 +108,8 @@ wit-component = "0.19.0"
 
 [features]
 default = ["llm"]
-all-tests = ["e2e-tests"]
-e2e-tests = []
+all-tests = ["extern-dependencies-tests"]
+extern-dependencies-tests = []
 llm = ["spin-trigger-http/llm"]
 llm-metal = ["llm", "spin-trigger-http/llm-metal"]
 llm-cublas = ["llm", "spin-trigger-http/llm-cublas"]

Cargo.toml

@@ -3,44 +3,6 @@ CERT_NAME ?= local
 SPIN_DOC_NAME ?= new-doc.md
 export PATH := target/debug:target/release:$(HOME)/.cargo/bin:$(PATH)
 
-ARCH = $(shell uname -p)
-
-## dependencies for e2e-tests
-E2E_BUILD_SPIN                  ?= false
-E2E_FETCH_SPIN                  ?= true
-E2E_TESTS_DOCKERFILE            ?= e2e-tests.Dockerfile
-MYSQL_IMAGE                     ?= mysql:8.0.22
-REDIS_IMAGE                     ?= redis:7.0.8-alpine3.17
-POSTGRES_IMAGE                  ?= postgres:14.7-alpine
-REGISTRY_IMAGE                  ?= registry:2
-E2E_SPIN_RELEASE_VOLUME_MOUNT   ?=
-E2E_SPIN_DEBUG_VOLUME_MOUNT     ?=
-
-## overrides for aarch64
-ifneq ($(ARCH),x86_64)
-	MYSQL_IMAGE             = arm64v8/mysql:8.0.32
-	REDIS_IMAGE             = arm64v8/redis:6.0-alpine3.17
-	POSTGRES_IMAGE          = arm64v8/postgres:14.7
-	REGISTRY_IMAGE          = arm64v8/registry:2
-	E2E_TESTS_DOCKERFILE    = e2e-tests-aarch64.Dockerfile
-endif
-
-ifneq (,$(wildcard $(shell pwd)/target/release/spin))
-	E2E_SPIN_RELEASE_VOLUME_MOUNT = -v $(shell pwd)/target/release/spin:/from-host/target/release/spin
-endif
-
-ifneq (,$(wildcard $(shell pwd)/target/debug/spin))
-	E2E_SPIN_DEBUG_VOLUME_MOUNT = -v $(shell pwd)/target/debug/spin:/from-host/target/debug/spin
-endif
-
-## Reset volume mounts for e2e-tests if Darwin because the
-## spin binaries built on macOS won't run in the docker container
-ifeq ($(shell uname -s),Darwin)
-	E2E_SPIN_RELEASE_VOLUME_MOUNT = 
-	E2E_SPIN_DEBUG_VOLUME_MOUNT =
-	E2E_BUILD_SPIN = true
-endif
-
 ## overrides for Windows
 ifeq ($(OS),Windows_NT)
 	LOG_LEVEL_VAR = 
@@ -87,20 +49,21 @@ update-cargo-locks:
 
 .PHONY: test-unit
 test-unit:
-	$(LOG_LEVEL_VAR) cargo test --all --no-fail-fast -- --skip integration_tests --skip spinup_tests --skip cloud_tests --nocapture
+	$(LOG_LEVEL_VAR) cargo test --all --no-fail-fast -- --skip integration_tests --nocapture
 
 .PHONY: test-crate
 test-crate:
-	$(LOG_LEVEL_VAR) cargo test -p $(crate) --no-fail-fast -- --skip integration_tests --skip spinup_tests --skip cloud_tests --nocapture
+	$(LOG_LEVEL_VAR) cargo test -p $(crate) --no-fail-fast -- --skip integration_tests --nocapture
 
+# Run the integration tests without the tests that use some sort of assumed external depedency (e.g., Docker, a language toolchain, etc.)
 .PHONY: test-integration
 test-integration:
-	cargo test -F e2e-tests -- runtime_tests --nocapture; \
-	$(LOG_LEVEL_VAR) cargo test --test integration --no-fail-fast -- --skip spinup_tests --skip cloud_tests --nocapture
+	cargo test --release integration_tests --no-default-features --no-fail-fast -- --nocapture
 
-.PHONY: test-spin-up
-test-spin-up:
-	cargo test --release spinup_tests --no-default-features --features e2e-tests --no-fail-fast -- --nocapture
+# Run all of the integration tests including those that use some sort of assumed external depedency (e.g., Docker, a language toolchain, etc.)
+.PHONY: test-integration-full
+test-integration-full:
+	cargo test --release integration_tests --no-default-features --features extern-dependencies-tests --no-fail-fast -- --nocapture
 
 .PHONY: test-sdk-go
 test-sdk-go:

Makefile

@@ -1,8 +1,8 @@
 #![cfg(test)]
 
-// Module for unit-testing the built-in templates when a full e2e test would be overkill.
+// Module for unit-testing the built-in templates when a full integration test would be overkill.
 // If your test involves invoking the Spin CLI, or builds or runs an application, use
-// an e2e test.
+// an integration test.
 
 use std::{collections::HashMap, path::PathBuf};
 

crates/templates/src/test_built_ins/mod.rs

@@ -12,3 +12,4 @@ test = false
 [dependencies]
 heck = "0.4.0"
 quote = "1.0.32"
+syn = "2.0"

crates/test-codegen-macro/Cargo.toml

@@ -4,7 +4,9 @@ use std::{env, path::PathBuf};
 
 /// This macro generates the `#[test]` functions for the runtime tests.
 #[proc_macro]
-pub fn codegen_runtime_tests(_input: TokenStream) -> TokenStream {
+pub fn codegen_runtime_tests(input: TokenStream) -> TokenStream {
+    let input = syn::parse_macro_input!(input);
+    let ignores = ignores(input);
     let mut tests = Vec::new();
     let tests_path =
         PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/runtime-tests/tests");
@@ -20,9 +22,14 @@ pub fn codegen_runtime_tests(_input: TokenStream) -> TokenStream {
             let requires_services = entry.path().join("services").exists();
 
             let name = test.file_stem().unwrap().to_str().unwrap();
+            let ignore = if ignores.contains(&name.to_string()) {
+                quote::quote!(#[ignore])
+            } else {
+                quote::quote!()
+            };
             let ident = quote::format_ident!("{}", name.to_snake_case());
             let feature_attribute = if requires_services {
-                quote::quote!(#[cfg(feature = "e2e-tests")])
+                quote::quote!(#[cfg(feature = "extern-dependencies-tests")])
             } else {
                 quote::quote!()
             };
@@ -35,6 +42,7 @@ pub fn codegen_runtime_tests(_input: TokenStream) -> TokenStream {
             // ```
             tests.push(quote::quote! {
                 #[test]
+                #ignore
                 #feature_attribute
                 fn #ident() {
                     run(::std::path::PathBuf::from(#tests_path_string).join(#name))
@@ -44,3 +52,26 @@ pub fn codegen_runtime_tests(_input: TokenStream) -> TokenStream {
     }
     (quote::quote!(#(#tests)*)).into()
 }
+
+fn ignores(input: syn::FieldValue) -> Vec<String> {
+    let syn::Member::Named(n) = input.member else {
+        panic!("codegen_runtime_tests!() requires a named field");
+    };
+    if n != "ignore" {
+        panic!("codegen_runtime_tests!() only supports the `ignore` field");
+    }
+    let syn::Expr::Array(a) = input.expr else {
+        panic!("codegen_runtime_tests!() requires an array of strings");
+    };
+    a.elems
+        .iter()
+        .map(|e| {
+            if let syn::Expr::Lit(l) = e {
+                if let syn::Lit::Str(s) = &l.lit {
+                    return s.value();
+                }
+            }
+            panic!("codegen_runtime_tests!() requires an array of strings");
+        })
+        .collect()
+}

crates/test-codegen-macro/src/lib.rs

@@ -23,17 +23,16 @@ The runtime tests are handled through the `runtime-tests` support crate. See the
 You can run runtime tests like so:
 
 ```bash
-cargo test runtime_tests -F e2e-tests
+cargo test runtime_tests -F extern-dependencies-tests
 ```
 
 # Integration tests
 
 Integration tests are meant to test anything that cannot be tested through some other testing mechanism usually because the scenario under test is complicated and involves the interaction between many different subsystems.  Historically, integration tests have been a landing pad for experimentation around testing that have eventually been turned into their own class of tests. 
 
-Currently, integration tests are split between two different modules that will soon be combined into one: `integration_tests` and `spinup_tests`.
-
 You can run integration tests like so:
 ```bash
 make test-integration
-make test-spin-up
 ```
+
+Note that this also runs the runtime tests as well.