Use worker aggregator in CI

Author: mendess

.github/workflows/daphneci.yml

@@ -80,3 +80,15 @@ jobs:
         env:
           HPKE_SIGNING_KEY: ${{ steps.hpke_signing_key.outputs.hpke_signing_key }}
           E2E_TEST_HPKE_SIGNING_CERTIFICATE: ${{ steps.hpke_signing_cert.outputs.hpke_signing_cert }}
+
+  e2e-worker-aggregator:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checking out
+        uses: actions/checkout@v3
+
+      - name: Run integration tests
+        uses: hoverkraft-tech/compose-action@v2.0.1
+        with:
+          compose-file: "./crates/daphne-worker-test/docker/docker-compose-e2e.yaml"
+          up-flags: "--build --abort-on-container-exit --exit-code-from test"

Makefile

@@ -45,6 +45,12 @@ e2e: /tmp/private-key /tmp/certificate
 		--abort-on-container-exit \
 		--exit-code-from test
 
+e2e-worker:
+	docker compose -f ./crates/daphne-worker-test/docker/docker-compose-e2e.yaml up \
+		--build \
+		--abort-on-container-exit \
+		--exit-code-from test
+
 build_interop:
 	docker build . -f ./interop/Dockerfile.interop_helper --tag daphne-interop
 

crates/daphne-worker-test/docker/aggregator.Dockerfile

@@ -0,0 +1,37 @@
+# Copyright (c) 2024 Cloudflare, Inc. All rights reserved.
+# SPDX-License-Identifier: BSD-3-Clause
+
+FROM rust:1.83-bookworm AS builder
+RUN apt update && apt install -y capnproto clang cmake
+
+# Pre-install worker-build and Rust's wasm32 target to speed up our custom build command
+RUN rustup target add wasm32-unknown-unknown
+RUN cargo install --git https://github.com/cloudflare/workers-rs
+
+# Build the storage proxy.
+WORKDIR /tmp/dap_test
+COPY Cargo.toml Cargo.lock ./
+COPY crates/daphne-worker-test ./crates/daphne-worker-test
+COPY crates/daphne-worker ./crates/daphne-worker
+COPY crates/daphne-service-utils ./crates/daphne-service-utils
+COPY crates/daphne ./crates/daphne
+WORKDIR /tmp/dap_test/crates/daphne-worker-test
+RUN worker-build --dev
+
+FROM node:bookworm AS leader
+RUN npm install -g wrangler@3.60.1 && npm cache clean --force
+COPY --from=builder /tmp/dap_test/crates/daphne-worker-test/build/ /build
+COPY crates/daphne-worker-test/wrangler.aggregator.toml /
+# this container doesn't need worker-build but the wrangler.toml requires it, so we just fake it
+RUN ln -s /usr/bin/true /usr/bin/worker-build
+
+ENTRYPOINT ["wrangler", "dev", "--config", "wrangler.aggregator.toml", "-e", "leader", "--port", "8787"]
+
+FROM node:bookworm AS helper
+RUN npm install -g wrangler@3.60.1 && npm cache clean --force
+COPY --from=builder /tmp/dap_test/crates/daphne-worker-test/build/ /build
+COPY crates/daphne-worker-test/wrangler.aggregator.toml /
+# this container doesn't need worker-build but the wrangler.toml requires it, so we just fake it
+RUN ln -s /usr/bin/true /usr/bin/worker-build
+
+ENTRYPOINT ["wrangler", "dev", "--config", "wrangler.aggregator.toml", "-e", "helper", "--port", "8788"]

crates/daphne-worker-test/docker/docker-compose-e2e.yaml

@@ -0,0 +1,43 @@
+# Copyright (c) 2024 Cloudflare, Inc. All rights reserved.
+# SPDX-License-Identifier: BSD-3-Clause
+---
+version: "3.9"
+
+networks:
+  dap_network:
+    driver: bridge
+
+services:
+  leader:
+    networks:
+      - dap_network
+    ports:
+      - "8787"
+    build:
+      context: ../../..
+      dockerfile: crates/daphne-worker-test/docker/aggregator.Dockerfile
+      target: leader
+    environment:
+      - RUST_LOG=info
+  helper:
+    networks:
+      - dap_network
+    ports:
+      - "8788"
+    build:
+      context: ../../..
+      dockerfile: crates/daphne-worker-test/docker/aggregator.Dockerfile
+      target: helper
+    environment:
+      - RUST_LOG=info
+  test:
+    networks:
+      - dap_network
+    build:
+      context: ../../..
+      dockerfile: crates/daphne-worker-test/docker/runtests.Dockerfile
+    depends_on:
+      - leader
+      - helper
+    environment:
+      - "E2E_TEST_HPKE_SIGNING_CERTIFICATE=-----BEGIN CERTIFICATE-----\nMIICCTCCAa+gAwIBAgIUBECNyioI8d+hgXsgmVI+TcRD8wUwCgYIKoZIzj0EAwIw\nWjELMAkGA1UEBhMCUFQxDjAMBgNVBAcMBUJyYWdhMRcwFQYDVQQKDA5DbG91ZGZs\nYXJlIExkYTEiMCAGA1UEAwwZaGVscGVyLmRhcC5jbG91ZGZsYXJlLmNvbTAeFw0y\nNTAxMDYxMTAwNDdaFw0yNjAxMDYxMTAwNDdaMFoxCzAJBgNVBAYTAlBUMQ4wDAYD\nVQQHDAVCcmFnYTEXMBUGA1UECgwOQ2xvdWRmbGFyZSBMZGExIjAgBgNVBAMMGWhl\nbHBlci5kYXAuY2xvdWRmbGFyZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNC\nAASheYdDsJLsG4UG95bs2qlVr1QQcK6+k6emAJSDAlr7bIGjHUoLwUdIQK818g/N\ngVL0vig90b4uGTS7KdKJ9o4Ko1MwUTAdBgNVHQ4EFgQUeOUaahWphjiaQotYoRfb\nVBdby+wwHwYDVR0jBBgwFoAUeOUaahWphjiaQotYoRfbVBdby+wwDwYDVR0TAQH/\nBAUwAwEB/zAKBggqhkjOPQQDAgNIADBFAiEAl0pg+5iQC3yskSbZrz8gyEgAaKx2\niyrASYsFh2gdfkICIAgkOlAOHsUHlhh0zRt9m283dLR0/ZYVoEhII8ZMkb1/\n-----END CERTIFICATE-----"

crates/daphne-worker-test/wrangler.aggregator.toml

@@ -34,6 +34,11 @@ DAP_DURABLE_AGGREGATE_STORE_GC_AFTER_SECS = "30"
 TASKPROV_SECRETS_ENABLED = "true"
 TASKPROV_SECRETS_VDAF_VERIFY_KEY_INIT = "b029a72fa327931a5cb643dcadcaafa098fcbfac07d990cb9e7c9a8675fafb18"
 TASKPROV_SECRETS_PEER_AUTH_EXPECT_LEADER_TOKEN = "I-am-the-leader"
+SIGNING_KEY = """-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIPg1ObFKii4YgTwltHaC/vgD6pwg5EtvW1YoyHVB5HfKoAoGCCqGSM49
+AwEHoUQDQgAEoXmHQ7CS7BuFBveW7NqpVa9UEHCuvpOnpgCUgwJa+2yBox1KC8FH
+SECvNfIPzYFS9L4oPdG+Lhk0uynSifaOCg==
+-----END EC PRIVATE KEY-----"""
 
 [env.helper.vars.SERVICE_CONFIG]
 env = "oxy"