From 765a74e2e730ac47b720fa1bc598554f5e292ba7 Mon Sep 17 00:00:00 2001 From: Kate Goldenring Date: Mon, 28 Oct 2024 20:22:45 -0700 Subject: [PATCH 1/7] Update Spin and Go versions for actions Signed-off-by: Kate Goldenring --- .github/workflows/ci.yaml | 2 +- images/spin/Dockerfile | 6 +++--- scripts/setup-linux.sh | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 646e6ec9..57795776 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -46,7 +46,7 @@ jobs: - uses: azure/setup-kubectl@v4 - uses: fermyon/actions/spin/setup@v1 with: - version: "v2.4.2" + version: "v2.7.0" - name: Setup build env run: | diff --git a/images/spin/Dockerfile b/images/spin/Dockerfile index a631b8bf..a1bbbac9 100644 --- a/images/spin/Dockerfile +++ b/images/spin/Dockerfile @@ -1,12 +1,12 @@ -FROM --platform=${BUILDPLATFORM} rust:1.71 AS build +FROM --platform=${BUILDPLATFORM} rust:1.79 AS build WORKDIR /opt/build COPY . . RUN rustup target add wasm32-wasi && cargo build --target wasm32-wasi --release -FROM --platform=linux/amd64 golang:1.21.3-bullseye AS build-go +FROM --platform=linux/amd64 golang:1.23.2-bullseye AS build-go WORKDIR /opt/build COPY . . -RUN curl -LO https://github.com/tinygo-org/tinygo/releases/download/v0.30.0/tinygo_0.30.0_amd64.deb && dpkg -i tinygo_0.30.0_amd64.deb +RUN curl -LO https://github.com/tinygo-org/tinygo/releases/download/v0.34.0/tinygo_0.34.0_amd64.deb && dpkg -i tinygo_0.34.0_amd64.deb RUN cd go-hello && tinygo build -target=wasi -o spin_go_hello.wasm main.go FROM scratch diff --git a/scripts/setup-linux.sh b/scripts/setup-linux.sh index 6d2b3150..14a0479c 100755 --- a/scripts/setup-linux.sh +++ b/scripts/setup-linux.sh @@ -8,6 +8,6 @@ sudo rustup target add wasm32-wasi && sudo rustup target add wasm32-unknown-unkn ## setup tinygo. required for building test spin app echo "setting up tinygo" -wget https://github.com/tinygo-org/tinygo/releases/download/v0.30.0/tinygo_0.30.0_amd64.deb -sudo dpkg -i tinygo_0.30.0_amd64.deb -rm tinygo_0.30.0_amd64.deb +wget https://github.com/tinygo-org/tinygo/releases/download/v0.34.0/tinygo_0.34.0_amd64.deb +sudo dpkg -i tinygo_0.34.0_amd64.deb +rm tinygo_0.34.0_amd64.deb From afa8e8d21ddf9bdb469aa00fc29f2afafc2aa778 Mon Sep 17 00:00:00 2001 From: Kate Goldenring Date: Thu, 31 Oct 2024 16:54:21 -0700 Subject: [PATCH 2/7] Use Mosquitto test broker instead of EMQX Signed-off-by: Kate Goldenring --- .gitignore | 4 ++- scripts/deploy-workloads.sh | 14 ---------- tests/src/integration_test.rs | 28 +++++-------------- tests/workloads-common/mqtt-broker.yaml | 25 ----------------- .../workloads.yaml | 5 ++-- .../workloads.yaml | 5 ++-- 6 files changed, 14 insertions(+), 67 deletions(-) delete mode 100644 tests/workloads-common/mqtt-broker.yaml diff --git a/.gitignore b/.gitignore index c048ab6d..968563e4 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,6 @@ target/ .tmp -test/* \ No newline at end of file +test/* + +.vscode/* \ No newline at end of file diff --git a/scripts/deploy-workloads.sh b/scripts/deploy-workloads.sh index d7bf3c44..5eaf4edf 100755 --- a/scripts/deploy-workloads.sh +++ b/scripts/deploy-workloads.sh @@ -9,28 +9,14 @@ if ! command -v kubectl &> /dev/null; then sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl; fi -update_mqtt_workload_with_broker_cluster_ip() { - local dir=$1 - echo "Waiting for emqx pod to be ready" - kubectl wait --for=condition=ready --timeout=20s pod/emqx - # The MQTT trigger cannot do DNS resolution, so we need to use the IP address of the MQTT broker - # Replace "EMQX_CLUSTER_IP" with the actual ClusterIP of the EMQX service - local cluster_ip=$(kubectl get svc emqx -o jsonpath='{.spec.clusterIP}') - sed -i "s/EMQX_CLUSTER_IP/$cluster_ip/g" $dir/workloads.yaml - echo "Updated workloads.yaml with ClusterIP: $cluster_ip" -} - - # apply the workloads echo ">>> apply workloads" kubectl apply -f tests/workloads-common if [ "$1" == "workloads-pushed-using-spin-registry-push" ]; then - update_mqtt_workload_with_broker_cluster_ip "tests/workloads-pushed-using-spin-registry-push" echo "deploying spin apps pushed to registry using 'spin registry push' command" kubectl apply -f tests/workloads-pushed-using-spin-registry-push else - update_mqtt_workload_with_broker_cluster_ip "tests/workloads-pushed-using-docker-build-push" echo "deploying spin apps pushed to registry using 'docker build && k3d image import' command" kubectl apply -f tests/workloads-pushed-using-docker-build-push fi diff --git a/tests/src/integration_test.rs b/tests/src/integration_test.rs index fe017c89..1d2f3a71 100644 --- a/tests/src/integration_test.rs +++ b/tests/src/integration_test.rs @@ -135,16 +135,16 @@ mod test { anyhow::bail!("kubectl is not installed"); } - // Port forward the emqx mqtt broker - let forward_port = port_forward_emqx(mqtt_port).await?; - - // Publish a message to the emqx broker - let mut mqttoptions = rumqttc::MqttOptions::new("123", "127.0.0.1", forward_port); + // Publish a message to the MQTT broker + let mut mqttoptions = rumqttc::MqttOptions::new("123", "test.mosquitto.org", mqtt_port); mqttoptions.set_keep_alive(std::time::Duration::from_secs(1)); let (client, mut eventloop) = rumqttc::AsyncClient::new(mqttoptions, 10); client - .subscribe("hello", rumqttc::QoS::AtMostOnce) + .subscribe( + "containerd-shim-spin/mqtt-test-17h24d", + rumqttc::QoS::AtMostOnce, + ) .await .unwrap(); @@ -153,7 +153,7 @@ mod test { for _i in 0..iterations { client .publish( - "hello", + "containerd-shim-spin/mqtt-test-17h24d", rumqttc::QoS::AtLeastOnce, false, message.as_bytes(), @@ -227,20 +227,6 @@ mod test { Ok(port) } - async fn port_forward_emqx(emqx_port: u16) -> Result { - let port = get_random_port()?; - - println!(" >>> kubectl portforward emqx {}:{} ", port, emqx_port); - - Command::new("kubectl") - .arg("port-forward") - .arg("emqx") - .arg(format!("{}:{}", port, emqx_port)) - .spawn()?; - tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; - Ok(port) - } - async fn get_logs_by_label(label: &str) -> Result { let output = Command::new("kubectl") .arg("logs") diff --git a/tests/workloads-common/mqtt-broker.yaml b/tests/workloads-common/mqtt-broker.yaml deleted file mode 100644 index 4688bc46..00000000 --- a/tests/workloads-common/mqtt-broker.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: emqx - labels: - app: emqx -spec: - containers: - - name: emqx - image: emqx/emqx - ports: - - containerPort: 1883 ---- -apiVersion: v1 -kind: Service -metadata: - name: emqx -spec: - selector: - app: emqx - ports: - - protocol: TCP - port: 1883 - targetPort: 1883 - type: ClusterIP \ No newline at end of file diff --git a/tests/workloads-pushed-using-docker-build-push/workloads.yaml b/tests/workloads-pushed-using-docker-build-push/workloads.yaml index 11f5e4df..db071b87 100644 --- a/tests/workloads-pushed-using-docker-build-push/workloads.yaml +++ b/tests/workloads-pushed-using-docker-build-push/workloads.yaml @@ -284,11 +284,10 @@ spec: - containerPort: 80 env: - name: SPIN_VARIABLE_MQTT_TOPIC - value: hello + value: containerd-shim-spin/mqtt-test-17h24d # The MQTT trigger cannot do DNS resolution, so we need to use the IP address of the MQTT broker - # Substitute `EMQX_CLUSTER_IP` with the result of `kubectl get svc emqx -n default -o jsonpath='{.spec.clusterIP}'` - name: SPIN_VARIABLE_MQTT_BROKER_URI - value: "mqtt://EMQX_CLUSTER_IP:1883" + value: "mqtt://test.mosquitto.org" --- apiVersion: v1 kind: Service diff --git a/tests/workloads-pushed-using-spin-registry-push/workloads.yaml b/tests/workloads-pushed-using-spin-registry-push/workloads.yaml index 8dc0d6ff..47647b89 100644 --- a/tests/workloads-pushed-using-spin-registry-push/workloads.yaml +++ b/tests/workloads-pushed-using-spin-registry-push/workloads.yaml @@ -284,11 +284,10 @@ spec: - containerPort: 80 env: - name: SPIN_VARIABLE_MQTT_TOPIC - value: hello + value: containerd-shim-spin/mqtt-test-17h24d # The MQTT trigger cannot do DNS resolution, so we need to use the IP address of the MQTT broker - # Substitute `EMQX_CLUSTER_IP` with the result of `kubectl get svc emqx -n default -o jsonpath='{.spec.clusterIP}'` - name: SPIN_VARIABLE_MQTT_BROKER_URI - value: "mqtt://EMQX_CLUSTER_IP:1883" + value: "mqtt://test.mosquitto.org" --- apiVersion: v1 kind: Service From 2c187f191ca331615dae4b6b4f0ab1160c3d2de4 Mon Sep 17 00:00:00 2001 From: Kate Goldenring Date: Fri, 1 Nov 2024 09:40:02 -0700 Subject: [PATCH 3/7] Wait for common workloads to be ready before deploying test apps Signed-off-by: Kate Goldenring --- scripts/deploy-workloads.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/deploy-workloads.sh b/scripts/deploy-workloads.sh index 5eaf4edf..edbcb72e 100755 --- a/scripts/deploy-workloads.sh +++ b/scripts/deploy-workloads.sh @@ -12,6 +12,8 @@ fi # apply the workloads echo ">>> apply workloads" kubectl apply -f tests/workloads-common +# wait for all the pods to be ready +kubectl wait --for=condition=ready --timeout=120s pod --all if [ "$1" == "workloads-pushed-using-spin-registry-push" ]; then echo "deploying spin apps pushed to registry using 'spin registry push' command" From 3ed2e3fda2d22052bcd198429198631a2026522e Mon Sep 17 00:00:00 2001 From: Kate Goldenring Date: Fri, 1 Nov 2024 10:53:54 -0700 Subject: [PATCH 4/7] Only use one agent in K3s test cluster Signed-off-by: Kate Goldenring --- scripts/up.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/up.sh b/scripts/up.sh index 51777f76..087d9e2f 100755 --- a/scripts/up.sh +++ b/scripts/up.sh @@ -13,7 +13,7 @@ IMAGES=("spin-hello-world" "spin-keyvalue" "spin-outbound-redis" "spin-multi-tri docker build -t k3d-shim-test "$dockerfile_path" k3d cluster create "$cluster_name" \ - --image k3d-shim-test --api-port 6551 -p '8082:80@loadbalancer' --agents 2 \ + --image k3d-shim-test --api-port 6551 -p '8082:80@loadbalancer' --agents 1 \ --registry-create test-registry:0.0.0.0:5000 kubectl wait --for=condition=ready node --all --timeout=120s From 60a18eac9e6062e9ea71e95a9f835a53f6dd4418 Mon Sep 17 00:00:00 2001 From: Kate Goldenring Date: Mon, 4 Nov 2024 16:58:24 -0800 Subject: [PATCH 5/7] Modify kubelet eviction threshold for k3d cluster Signed-off-by: Kate Goldenring --- scripts/up.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/up.sh b/scripts/up.sh index 087d9e2f..3ffd0479 100755 --- a/scripts/up.sh +++ b/scripts/up.sh @@ -13,8 +13,10 @@ IMAGES=("spin-hello-world" "spin-keyvalue" "spin-outbound-redis" "spin-multi-tri docker build -t k3d-shim-test "$dockerfile_path" k3d cluster create "$cluster_name" \ - --image k3d-shim-test --api-port 6551 -p '8082:80@loadbalancer' --agents 1 \ - --registry-create test-registry:0.0.0.0:5000 + --image k3d-shim-test --api-port 6551 -p '8082:80@loadbalancer' --agents 2 \ + --registry-create test-registry:0.0.0.0:5000 \ + --k3s-arg '--kubelet-arg=eviction-hard=imagefs.available<1%,nodefs.available<1%@agent:*' \ + --k3s-arg '--kubelet-arg=eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%@agent:*' kubectl wait --for=condition=ready node --all --timeout=120s From 5da21ba7ec7ae5355a0a81b76757b1bc0eb21f42 Mon Sep 17 00:00:00 2001 From: Kate Goldenring Date: Mon, 4 Nov 2024 22:13:05 -0800 Subject: [PATCH 6/7] Try freeing up disk space before running tests Signed-off-by: Kate Goldenring --- .github/workflows/ci.yaml | 6 ++++-- Makefile | 6 +++++- scripts/free-disk.sh | 11 +++++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) create mode 100755 scripts/free-disk.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 57795776..76080931 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -63,7 +63,7 @@ jobs: run: make install-k3d - name: run integration tests - run: BIN_DIR="./bin" make integration-tests + run: BIN_DIR="./bin" IS_CI=true make integration-tests - name: run collect debug logs if: failure() @@ -76,7 +76,9 @@ jobs: name: debug-logs path: debug-logs/ retention-days: 5 - + - name: Output runner storage on failure + if: failure() + run: df -h - name: clean up k3d if: always() run: make tests/clean diff --git a/Makefile b/Makefile index 81398cc1..6509d96e 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ else VERBOSE_FLAG := -vvv endif +IS_CI ?= false BIN_DIR ?= UNAME_S := $(shell uname -s) @@ -52,6 +53,9 @@ unit-tests: build integration-tests: prepare-cluster-and-images integration-docker-build-push-tests integration-spin-registry-push-tests echo "Integration tests complete. You may run 'make tests/clean' to clean up the test environment." +free-disk: + ./scripts/free-disk.sh + # integration-tests for workloads pushed using docker build push integration-docker-build-push-tests: ./scripts/run-integration-tests.sh "workloads-pushed-using-docker-build-push" @@ -126,7 +130,7 @@ deploy-workloads-pushed-using-spin-registry-push: pod-terminates-test: ./scripts/pod-terminates-test.sh -prepare-cluster-and-images: check-bins move-bins up pod-status-check +prepare-cluster-and-images: check-bins move-bins up free-disk pod-status-check # clean diff --git a/scripts/free-disk.sh b/scripts/free-disk.sh new file mode 100755 index 00000000..1e9f7207 --- /dev/null +++ b/scripts/free-disk.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +IS_CI=${IS_CI:-false} + +# clean up Rust builds to free space +cargo install cargo-clean-all +cargo clean-all -y +if [ "$IS_CI" = true ]; then + # remove all docker images + docker system prune -af +fi \ No newline at end of file From 3e53c6e6481193b98beecce3ec7eaffa1dc0e43c Mon Sep 17 00:00:00 2001 From: Kate Goldenring Date: Mon, 4 Nov 2024 22:59:19 -0800 Subject: [PATCH 7/7] Reset gitignore Signed-off-by: Kate Goldenring --- .gitignore | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 968563e4..c048ab6d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,4 @@ target/ .tmp -test/* - -.vscode/* \ No newline at end of file +test/* \ No newline at end of file