Merge pull request #2247 from fermyon/move-over

Move wasi-http integration tests over to new testing framework

Author: rylev

.github/actions/spin-ci-dependencies/action.yml

@@ -72,6 +72,12 @@ inputs:
     required: false
     type: string
 
+  docker: 
+    description: 'setup docker'
+    required: false
+    default: 'false'
+    type: bool
+
 runs:
   using: "composite"
   steps:

.github/workflows/build.yml

@@ -48,7 +48,7 @@ jobs:
         if: failure()
         uses: andymckay/[email protected]
 
-  ## This is separated out to remove e2e-tests dependency on windows/mac builds
+  ## This is separated out to remove full integration tests dependencies on windows/mac builds
   build-rust-ubuntu:
     name: Build Spin Ubuntu
     runs-on: ubuntu-latest
@@ -157,11 +157,15 @@ jobs:
           rust-cache: true
           nomad: true
 
-      - name: Cargo Unit and Integration Tests
+      - name: Run Unit Tests
         run: |
-          make test-unit test-integration
-        env:
-          CARGO_INCREMENTAL: 0
+          make test-unit
+
+      - name: Run Integration Tests
+        run: |
+          make test-integration
+        # Only run integration tests on macOS as they will be run on ubuntu separately
+        if: ${{ matrix.runner == 'macos-latest' }}
 
   test-go:
     name: Test Spin SDK - Go
@@ -193,7 +197,7 @@ jobs:
       - name: "Test Go SDK"
         run: make test-sdk-go
 
-  e2e-tests:
+  all-integration-tests:
     # run on a larger runner for more SSD/resource access
     runs-on: ubuntu-22.04-4core-spin
     if: ${{ github.repository_owner == 'fermyon' }}
@@ -225,6 +229,6 @@ jobs:
           mv grain-linux-x64 grain
           echo "$PWD" >> $GITHUB_PATH
 
-      - name: Run e2e tests
+      - name: Run Full Integration Tests
         run: |
-          make test-spin-up
+          make test-integration-full

Cargo.lock

@@ -108,8 +108,8 @@ wit-component = "0.19.0"
 
 [features]
 default = ["llm"]
-all-tests = ["e2e-tests"]
-e2e-tests = []
+all-tests = ["extern-dependencies-tests"]
+extern-dependencies-tests = []
 llm = ["spin-trigger-http/llm"]
 llm-metal = ["llm", "spin-trigger-http/llm-metal"]
 llm-cublas = ["llm", "spin-trigger-http/llm-cublas"]

Cargo.toml

@@ -3,44 +3,6 @@ CERT_NAME ?= local
 SPIN_DOC_NAME ?= new-doc.md
 export PATH := target/debug:target/release:$(HOME)/.cargo/bin:$(PATH)
 
-ARCH = $(shell uname -p)
-
-## dependencies for e2e-tests
-E2E_BUILD_SPIN                  ?= false
-E2E_FETCH_SPIN                  ?= true
-E2E_TESTS_DOCKERFILE            ?= e2e-tests.Dockerfile
-MYSQL_IMAGE                     ?= mysql:8.0.22
-REDIS_IMAGE                     ?= redis:7.0.8-alpine3.17
-POSTGRES_IMAGE                  ?= postgres:14.7-alpine
-REGISTRY_IMAGE                  ?= registry:2
-E2E_SPIN_RELEASE_VOLUME_MOUNT   ?=
-E2E_SPIN_DEBUG_VOLUME_MOUNT     ?=
-
-## overrides for aarch64
-ifneq ($(ARCH),x86_64)
-	MYSQL_IMAGE             = arm64v8/mysql:8.0.32
-	REDIS_IMAGE             = arm64v8/redis:6.0-alpine3.17
-	POSTGRES_IMAGE          = arm64v8/postgres:14.7
-	REGISTRY_IMAGE          = arm64v8/registry:2
-	E2E_TESTS_DOCKERFILE    = e2e-tests-aarch64.Dockerfile
-endif
-
-ifneq (,$(wildcard $(shell pwd)/target/release/spin))
-	E2E_SPIN_RELEASE_VOLUME_MOUNT = -v $(shell pwd)/target/release/spin:/from-host/target/release/spin
-endif
-
-ifneq (,$(wildcard $(shell pwd)/target/debug/spin))
-	E2E_SPIN_DEBUG_VOLUME_MOUNT = -v $(shell pwd)/target/debug/spin:/from-host/target/debug/spin
-endif
-
-## Reset volume mounts for e2e-tests if Darwin because the
-## spin binaries built on macOS won't run in the docker container
-ifeq ($(shell uname -s),Darwin)
-	E2E_SPIN_RELEASE_VOLUME_MOUNT = 
-	E2E_SPIN_DEBUG_VOLUME_MOUNT =
-	E2E_BUILD_SPIN = true
-endif
-
 ## overrides for Windows
 ifeq ($(OS),Windows_NT)
 	LOG_LEVEL_VAR = 
@@ -87,20 +49,21 @@ update-cargo-locks:
 
 .PHONY: test-unit
 test-unit:
-	$(LOG_LEVEL_VAR) cargo test --all --no-fail-fast -- --skip integration_tests --skip spinup_tests --skip cloud_tests --nocapture
+	$(LOG_LEVEL_VAR) cargo test --all --no-fail-fast -- --skip integration_tests --nocapture
 
 .PHONY: test-crate
 test-crate:
-	$(LOG_LEVEL_VAR) cargo test -p $(crate) --no-fail-fast -- --skip integration_tests --skip spinup_tests --skip cloud_tests --nocapture
+	$(LOG_LEVEL_VAR) cargo test -p $(crate) --no-fail-fast -- --skip integration_tests --nocapture
 
+# Run the integration tests without the tests that use some sort of assumed external depedency (e.g., Docker, a language toolchain, etc.)
 .PHONY: test-integration
 test-integration:
-	cargo test -F e2e-tests -- runtime_tests --nocapture; \
-	$(LOG_LEVEL_VAR) cargo test --test integration --no-fail-fast -- --skip spinup_tests --skip cloud_tests --nocapture
+	cargo test --release integration_tests --no-default-features --no-fail-fast -- --nocapture
 
-.PHONY: test-spin-up
-test-spin-up:
-	cargo test --release spinup_tests --no-default-features --features e2e-tests --no-fail-fast -- --nocapture
+# Run all of the integration tests including those that use some sort of assumed external depedency (e.g., Docker, a language toolchain, etc.)
+.PHONY: test-integration-full
+test-integration-full:
+	cargo test --release integration_tests --no-default-features --features extern-dependencies-tests --no-fail-fast -- --nocapture
 
 .PHONY: test-sdk-go
 test-sdk-go:

Makefile

@@ -8,8 +8,6 @@ use std::{
 use cargo_target_dep::build_target_dep;
 
 const TIMER_TRIGGER_INTEGRATION_TEST: &str = "examples/spin-timer/app-example";
-const WASI_HTTP_INTEGRATION_TEST: &str = "examples/wasi-http-rust-streaming-outgoing-body";
-const OUTBOUND_HTTP_POST_INTEGRATION_TEST: &str = "examples/http-rust-outbound-post";
 
 fn main() {
     // Extract environment information to be passed to plugins.
@@ -84,8 +82,6 @@ error: the `wasm32-wasi` target is not installed
     build_wasm_test_program("timer_app_example.wasm", "examples/spin-timer/app-example");
 
     cargo_build(TIMER_TRIGGER_INTEGRATION_TEST);
-    cargo_build(WASI_HTTP_INTEGRATION_TEST);
-    cargo_build(OUTBOUND_HTTP_POST_INTEGRATION_TEST);
 }
 
 fn build_wasm_test_program(name: &'static str, root: &'static str) {

build.rs

@@ -1,8 +1,8 @@
 #![cfg(test)]
 
-// Module for unit-testing the built-in templates when a full e2e test would be overkill.
+// Module for unit-testing the built-in templates when a full integration test would be overkill.
 // If your test involves invoking the Spin CLI, or builds or runs an application, use
-// an e2e test.
+// an integration test.
 
 use std::{collections::HashMap, path::PathBuf};
 

crates/templates/src/test_built_ins/mod.rs

@@ -12,3 +12,4 @@ test = false
 [dependencies]
 heck = "0.4.0"
 quote = "1.0.32"
+syn = "2.0"

crates/test-codegen-macro/Cargo.toml

@@ -4,7 +4,9 @@ use std::{env, path::PathBuf};
 
 /// This macro generates the `#[test]` functions for the runtime tests.
 #[proc_macro]
-pub fn codegen_runtime_tests(_input: TokenStream) -> TokenStream {
+pub fn codegen_runtime_tests(input: TokenStream) -> TokenStream {
+    let input = syn::parse_macro_input!(input);
+    let ignores = ignores(input);
     let mut tests = Vec::new();
     let tests_path =
         PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/runtime-tests/tests");
@@ -20,9 +22,14 @@ pub fn codegen_runtime_tests(_input: TokenStream) -> TokenStream {
             let requires_services = entry.path().join("services").exists();
 
             let name = test.file_stem().unwrap().to_str().unwrap();
+            let ignore = if ignores.contains(&name.to_string()) {
+                quote::quote!(#[ignore])
+            } else {
+                quote::quote!()
+            };
             let ident = quote::format_ident!("{}", name.to_snake_case());
             let feature_attribute = if requires_services {
-                quote::quote!(#[cfg(feature = "e2e-tests")])
+                quote::quote!(#[cfg(feature = "extern-dependencies-tests")])
             } else {
                 quote::quote!()
             };
@@ -35,6 +42,7 @@ pub fn codegen_runtime_tests(_input: TokenStream) -> TokenStream {
             // ```
             tests.push(quote::quote! {
                 #[test]
+                #ignore
                 #feature_attribute
                 fn #ident() {
                     run(::std::path::PathBuf::from(#tests_path_string).join(#name))
@@ -44,3 +52,26 @@ pub fn codegen_runtime_tests(_input: TokenStream) -> TokenStream {
     }
     (quote::quote!(#(#tests)*)).into()
 }
+
+fn ignores(input: syn::FieldValue) -> Vec<String> {
+    let syn::Member::Named(n) = input.member else {
+        panic!("codegen_runtime_tests!() requires a named field");
+    };
+    if n != "ignore" {
+        panic!("codegen_runtime_tests!() only supports the `ignore` field");
+    }
+    let syn::Expr::Array(a) = input.expr else {
+        panic!("codegen_runtime_tests!() requires an array of strings");
+    };
+    a.elems
+        .iter()
+        .map(|e| {
+            if let syn::Expr::Lit(l) = e {
+                if let syn::Lit::Str(s) = &l.lit {
+                    return s.value();
+                }
+            }
+            panic!("codegen_runtime_tests!() requires an array of strings");
+        })
+        .collect()
+}