diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 646e6ec9..76080931 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -46,7 +46,7 @@ jobs: - uses: azure/setup-kubectl@v4 - uses: fermyon/actions/spin/setup@v1 with: - version: "v2.4.2" + version: "v2.7.0" - name: Setup build env run: | @@ -63,7 +63,7 @@ jobs: run: make install-k3d - name: run integration tests - run: BIN_DIR="./bin" make integration-tests + run: BIN_DIR="./bin" IS_CI=true make integration-tests - name: run collect debug logs if: failure() @@ -76,7 +76,9 @@ jobs: name: debug-logs path: debug-logs/ retention-days: 5 - + - name: Output runner storage on failure + if: failure() + run: df -h - name: clean up k3d if: always() run: make tests/clean diff --git a/Makefile b/Makefile index 81398cc1..6509d96e 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ else VERBOSE_FLAG := -vvv endif +IS_CI ?= false BIN_DIR ?= UNAME_S := $(shell uname -s) @@ -52,6 +53,9 @@ unit-tests: build integration-tests: prepare-cluster-and-images integration-docker-build-push-tests integration-spin-registry-push-tests echo "Integration tests complete. You may run 'make tests/clean' to clean up the test environment." +free-disk: + ./scripts/free-disk.sh + # integration-tests for workloads pushed using docker build push integration-docker-build-push-tests: ./scripts/run-integration-tests.sh "workloads-pushed-using-docker-build-push" @@ -126,7 +130,7 @@ deploy-workloads-pushed-using-spin-registry-push: pod-terminates-test: ./scripts/pod-terminates-test.sh -prepare-cluster-and-images: check-bins move-bins up pod-status-check +prepare-cluster-and-images: check-bins move-bins up free-disk pod-status-check # clean diff --git a/images/spin/Dockerfile b/images/spin/Dockerfile index a631b8bf..a1bbbac9 100644 --- a/images/spin/Dockerfile +++ b/images/spin/Dockerfile @@ -1,12 +1,12 @@ -FROM --platform=${BUILDPLATFORM} rust:1.71 AS build +FROM --platform=${BUILDPLATFORM} rust:1.79 AS build WORKDIR /opt/build COPY . . RUN rustup target add wasm32-wasi && cargo build --target wasm32-wasi --release -FROM --platform=linux/amd64 golang:1.21.3-bullseye AS build-go +FROM --platform=linux/amd64 golang:1.23.2-bullseye AS build-go WORKDIR /opt/build COPY . . -RUN curl -LO https://github.com/tinygo-org/tinygo/releases/download/v0.30.0/tinygo_0.30.0_amd64.deb && dpkg -i tinygo_0.30.0_amd64.deb +RUN curl -LO https://github.com/tinygo-org/tinygo/releases/download/v0.34.0/tinygo_0.34.0_amd64.deb && dpkg -i tinygo_0.34.0_amd64.deb RUN cd go-hello && tinygo build -target=wasi -o spin_go_hello.wasm main.go FROM scratch diff --git a/scripts/deploy-workloads.sh b/scripts/deploy-workloads.sh index d7bf3c44..edbcb72e 100755 --- a/scripts/deploy-workloads.sh +++ b/scripts/deploy-workloads.sh @@ -9,28 +9,16 @@ if ! command -v kubectl &> /dev/null; then sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl; fi -update_mqtt_workload_with_broker_cluster_ip() { - local dir=$1 - echo "Waiting for emqx pod to be ready" - kubectl wait --for=condition=ready --timeout=20s pod/emqx - # The MQTT trigger cannot do DNS resolution, so we need to use the IP address of the MQTT broker - # Replace "EMQX_CLUSTER_IP" with the actual ClusterIP of the EMQX service - local cluster_ip=$(kubectl get svc emqx -o jsonpath='{.spec.clusterIP}') - sed -i "s/EMQX_CLUSTER_IP/$cluster_ip/g" $dir/workloads.yaml - echo "Updated workloads.yaml with ClusterIP: $cluster_ip" -} - - # apply the workloads echo ">>> apply workloads" kubectl apply -f tests/workloads-common +# wait for all the pods to be ready +kubectl wait --for=condition=ready --timeout=120s pod --all if [ "$1" == "workloads-pushed-using-spin-registry-push" ]; then - update_mqtt_workload_with_broker_cluster_ip "tests/workloads-pushed-using-spin-registry-push" echo "deploying spin apps pushed to registry using 'spin registry push' command" kubectl apply -f tests/workloads-pushed-using-spin-registry-push else - update_mqtt_workload_with_broker_cluster_ip "tests/workloads-pushed-using-docker-build-push" echo "deploying spin apps pushed to registry using 'docker build && k3d image import' command" kubectl apply -f tests/workloads-pushed-using-docker-build-push fi diff --git a/scripts/free-disk.sh b/scripts/free-disk.sh new file mode 100755 index 00000000..1e9f7207 --- /dev/null +++ b/scripts/free-disk.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +IS_CI=${IS_CI:-false} + +# clean up Rust builds to free space +cargo install cargo-clean-all +cargo clean-all -y +if [ "$IS_CI" = true ]; then + # remove all docker images + docker system prune -af +fi \ No newline at end of file diff --git a/scripts/setup-linux.sh b/scripts/setup-linux.sh index 6d2b3150..14a0479c 100755 --- a/scripts/setup-linux.sh +++ b/scripts/setup-linux.sh @@ -8,6 +8,6 @@ sudo rustup target add wasm32-wasi && sudo rustup target add wasm32-unknown-unkn ## setup tinygo. required for building test spin app echo "setting up tinygo" -wget https://github.com/tinygo-org/tinygo/releases/download/v0.30.0/tinygo_0.30.0_amd64.deb -sudo dpkg -i tinygo_0.30.0_amd64.deb -rm tinygo_0.30.0_amd64.deb +wget https://github.com/tinygo-org/tinygo/releases/download/v0.34.0/tinygo_0.34.0_amd64.deb +sudo dpkg -i tinygo_0.34.0_amd64.deb +rm tinygo_0.34.0_amd64.deb diff --git a/scripts/up.sh b/scripts/up.sh index 51777f76..3ffd0479 100755 --- a/scripts/up.sh +++ b/scripts/up.sh @@ -14,7 +14,9 @@ docker build -t k3d-shim-test "$dockerfile_path" k3d cluster create "$cluster_name" \ --image k3d-shim-test --api-port 6551 -p '8082:80@loadbalancer' --agents 2 \ - --registry-create test-registry:0.0.0.0:5000 + --registry-create test-registry:0.0.0.0:5000 \ + --k3s-arg '--kubelet-arg=eviction-hard=imagefs.available<1%,nodefs.available<1%@agent:*' \ + --k3s-arg '--kubelet-arg=eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%@agent:*' kubectl wait --for=condition=ready node --all --timeout=120s diff --git a/tests/src/integration_test.rs b/tests/src/integration_test.rs index fe017c89..1d2f3a71 100644 --- a/tests/src/integration_test.rs +++ b/tests/src/integration_test.rs @@ -135,16 +135,16 @@ mod test { anyhow::bail!("kubectl is not installed"); } - // Port forward the emqx mqtt broker - let forward_port = port_forward_emqx(mqtt_port).await?; - - // Publish a message to the emqx broker - let mut mqttoptions = rumqttc::MqttOptions::new("123", "127.0.0.1", forward_port); + // Publish a message to the MQTT broker + let mut mqttoptions = rumqttc::MqttOptions::new("123", "test.mosquitto.org", mqtt_port); mqttoptions.set_keep_alive(std::time::Duration::from_secs(1)); let (client, mut eventloop) = rumqttc::AsyncClient::new(mqttoptions, 10); client - .subscribe("hello", rumqttc::QoS::AtMostOnce) + .subscribe( + "containerd-shim-spin/mqtt-test-17h24d", + rumqttc::QoS::AtMostOnce, + ) .await .unwrap(); @@ -153,7 +153,7 @@ mod test { for _i in 0..iterations { client .publish( - "hello", + "containerd-shim-spin/mqtt-test-17h24d", rumqttc::QoS::AtLeastOnce, false, message.as_bytes(), @@ -227,20 +227,6 @@ mod test { Ok(port) } - async fn port_forward_emqx(emqx_port: u16) -> Result { - let port = get_random_port()?; - - println!(" >>> kubectl portforward emqx {}:{} ", port, emqx_port); - - Command::new("kubectl") - .arg("port-forward") - .arg("emqx") - .arg(format!("{}:{}", port, emqx_port)) - .spawn()?; - tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; - Ok(port) - } - async fn get_logs_by_label(label: &str) -> Result { let output = Command::new("kubectl") .arg("logs") diff --git a/tests/workloads-common/mqtt-broker.yaml b/tests/workloads-common/mqtt-broker.yaml deleted file mode 100644 index 4688bc46..00000000 --- a/tests/workloads-common/mqtt-broker.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: emqx - labels: - app: emqx -spec: - containers: - - name: emqx - image: emqx/emqx - ports: - - containerPort: 1883 ---- -apiVersion: v1 -kind: Service -metadata: - name: emqx -spec: - selector: - app: emqx - ports: - - protocol: TCP - port: 1883 - targetPort: 1883 - type: ClusterIP \ No newline at end of file diff --git a/tests/workloads-pushed-using-docker-build-push/workloads.yaml b/tests/workloads-pushed-using-docker-build-push/workloads.yaml index 11f5e4df..db071b87 100644 --- a/tests/workloads-pushed-using-docker-build-push/workloads.yaml +++ b/tests/workloads-pushed-using-docker-build-push/workloads.yaml @@ -284,11 +284,10 @@ spec: - containerPort: 80 env: - name: SPIN_VARIABLE_MQTT_TOPIC - value: hello + value: containerd-shim-spin/mqtt-test-17h24d # The MQTT trigger cannot do DNS resolution, so we need to use the IP address of the MQTT broker - # Substitute `EMQX_CLUSTER_IP` with the result of `kubectl get svc emqx -n default -o jsonpath='{.spec.clusterIP}'` - name: SPIN_VARIABLE_MQTT_BROKER_URI - value: "mqtt://EMQX_CLUSTER_IP:1883" + value: "mqtt://test.mosquitto.org" --- apiVersion: v1 kind: Service diff --git a/tests/workloads-pushed-using-spin-registry-push/workloads.yaml b/tests/workloads-pushed-using-spin-registry-push/workloads.yaml index 8dc0d6ff..47647b89 100644 --- a/tests/workloads-pushed-using-spin-registry-push/workloads.yaml +++ b/tests/workloads-pushed-using-spin-registry-push/workloads.yaml @@ -284,11 +284,10 @@ spec: - containerPort: 80 env: - name: SPIN_VARIABLE_MQTT_TOPIC - value: hello + value: containerd-shim-spin/mqtt-test-17h24d # The MQTT trigger cannot do DNS resolution, so we need to use the IP address of the MQTT broker - # Substitute `EMQX_CLUSTER_IP` with the result of `kubectl get svc emqx -n default -o jsonpath='{.spec.clusterIP}'` - name: SPIN_VARIABLE_MQTT_BROKER_URI - value: "mqtt://EMQX_CLUSTER_IP:1883" + value: "mqtt://test.mosquitto.org" --- apiVersion: v1 kind: Service