Use HTTP server for TUF conformance testing

Signed-off-by: Aaron Lew <[email protected]>

Author: aaronlew02

.github/workflows/tuf-conformance.yml

@@ -1,4 +1,6 @@
 name: TUF Conformance Tests
+permissions:
+  contents: read
 
 on:
   push:
@@ -37,11 +39,14 @@ jobs:
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4.4.1
 
-      - name: Build tuf cli
-        run: ./gradlew :tuf-cli:build
+      - name: Build tuf cli and server jar
+        run: ./gradlew :tuf-cli:serverShadowJar
 
-      - name: Unpack tuf distribution
-        run: tar -xvf ${{ github.workspace }}/tuf-cli/build/distributions/tuf-cli-*.tar --strip-components 1
+      - name: Start test server in background
+        run: java -jar ${{ github.workspace }}/tuf-cli/build/libs/tuf-cli-server-all.jar &
+
+      - name: Wait for server to be ready
+        run: curl --retry-connrefused --retry 10 --retry-delay 1 --fail http://localhost:8080/
 
       - name: Set up JDK ${{ matrix.java-version }}
         uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
@@ -51,5 +56,5 @@ jobs:
 
       - uses: theupdateframework/tuf-conformance@9bfc222a371e30ad5511eb17449f68f855fb9d8f # v2.3.0
         with:
-          entrypoint: ${{ github.workspace }}/bin/tuf-cli
+          entrypoint: ${{ github.workspace }}/tuf-cli/tuf-cli-server
           artifact-name: test repositories for tuf-cli java ${{ matrix.java-version }}

tuf-cli/build.gradle.kts

@@ -1,6 +1,9 @@
+import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
+
 plugins {
     id("build-logic.java")
     id("application")
+    id("com.gradleup.shadow") version "9.0.0-rc3"
 }
 
 repositories {
@@ -15,6 +18,11 @@ dependencies {
     implementation(platform("com.google.oauth-client:google-oauth-client-bom:1.39.0"))
     implementation("com.google.oauth-client:google-oauth-client")
 
+    implementation("org.eclipse.jetty:jetty-server:11.0.24")
+    implementation("org.eclipse.jetty:jetty-servlet:11.0.24")
+
+    implementation("org.slf4j:slf4j-simple:2.0.17")
+
     annotationProcessor("info.picocli:picocli-codegen:4.7.6")
 }
 
@@ -37,3 +45,20 @@ distributions.main {
 tasks.run.configure {
     workingDir = rootProject.projectDir
 }
+
+tasks.register<ShadowJar>("serverShadowJar") {
+    archiveBaseName.set("tuf-cli-server")
+    archiveClassifier.set("all")
+    archiveVersion.set("")
+
+    mergeServiceFiles()
+
+    from(sourceSets.main.get().output)
+    configurations = listOf(project.configurations.runtimeClasspath.get())
+
+    exclude("META-INF/*.SF", "META-INF/*.DSA", "META-INF/*.RSA")
+
+    manifest {
+        attributes("Main-Class" to "dev.sigstore.tuf.cli.TufConformanceServer")
+    }
+}

tuf-cli/src/main/java/dev/sigstore/tuf/cli/Download.java

@@ -49,6 +49,7 @@ public Integer call() throws Exception {
             .setMetaFetcher(MetaFetcher.newFetcher(HttpFetcher.newFetcher(metadataUrl)))
             .setTargetFetcher(HttpFetcher.newFetcher(targetBaseUrl))
             .setTargetStore(fsStore)
+            .setClock(TestClock.get())
             .build();
     // the java client isn't one shot like other clients, so downloadTarget doesn't call update
     // for the sake of conformance updateMeta here

tuf-cli/src/main/java/dev/sigstore/tuf/cli/Refresh.java

@@ -44,6 +44,7 @@ public Integer call() throws Exception {
                     PassthroughCacheMetaStore.newPassthroughMetaCache(fsStore)))
             .setTrustedRootPath(RootProvider.fromFile(metadataDir.resolve("root.json")))
             .setMetaFetcher(MetaFetcher.newFetcher(HttpFetcher.newFetcher(metadataUrl)))
+            .setClock(TestClock.get())
             .build();
     tuf.updateMeta();
     return 0;

tuf-cli/src/main/java/dev/sigstore/tuf/cli/TestClock.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2025 The Sigstore Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package dev.sigstore.tuf.cli;
+
+import java.time.Clock;
+
+public class TestClock {
+  private static Clock clock = null;
+
+  public static void set(Clock clock) {
+    TestClock.clock = clock;
+  }
+
+  public static Clock get() {
+    return clock == null ? Clock.systemUTC() : clock;
+  }
+
+  public static void reset() {
+    TestClock.clock = null;
+  }
+}

tuf-cli/src/main/java/dev/sigstore/tuf/cli/TufConformanceServer.java

@@ -0,0 +1,143 @@
+/*
+ * Copyright 2025 The Sigstore Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package dev.sigstore.tuf.cli;
+
+import com.google.gson.Gson;
+import jakarta.servlet.ServletException;
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.servlet.http.HttpServletResponse;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
+import java.time.Clock;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.handler.AbstractHandler;
+
+public class TufConformanceServer {
+
+  private static final Gson GSON = new Gson();
+  private static boolean debug = false;
+
+  private static class ExecuteRequest {
+    String[] args;
+    String faketime;
+  }
+
+  public static void main(String[] args) throws Exception {
+    if (args.length > 0 && "--debug".equals(args[0])) {
+      debug = true;
+    }
+    int port = 8080;
+    Server server = new Server(port);
+    server.setHandler(new TufConformanceHandler());
+    server.start();
+    server.join();
+  }
+
+  public static class TufConformanceHandler extends AbstractHandler {
+    @Override
+    public void handle(
+        String target,
+        Request baseRequest,
+        HttpServletRequest request,
+        HttpServletResponse response)
+        throws IOException, ServletException {
+      if ("/".equals(target)) {
+        handleHealthCheck(response);
+      } else if ("/execute".equals(target) && "POST".equals(request.getMethod())) {
+        handleExecute(request, response);
+      }
+      baseRequest.setHandled(true);
+    }
+  }
+
+  private static void handleExecute(HttpServletRequest request, HttpServletResponse response)
+      throws IOException {
+    ExecuteRequest executeRequest;
+    try (InputStream is = request.getInputStream()) {
+      String requestBody = new String(is.readAllBytes(), StandardCharsets.UTF_8);
+      executeRequest = GSON.fromJson(requestBody, ExecuteRequest.class);
+    }
+
+    // Tests should not be run in parallel, to ensure orderly input/output
+    PrintStream originalOut = System.out;
+    PrintStream originalErr = System.err;
+
+    ByteArrayOutputStream outContent = new ByteArrayOutputStream();
+    ByteArrayOutputStream errContent = new ByteArrayOutputStream();
+
+    try (PrintStream outPs = new PrintStream(outContent, true, StandardCharsets.UTF_8);
+        PrintStream errPs = new PrintStream(errContent, true, StandardCharsets.UTF_8)) {
+      if (!debug) {
+        System.setOut(outPs);
+        System.setErr(errPs);
+      }
+
+      try {
+        Instant fakeNow = Instant.ofEpochSecond(Long.parseLong(executeRequest.faketime));
+        TestClock.set(Clock.fixed(fakeNow, ZoneOffset.UTC));
+      } catch (NumberFormatException e) {
+        throw new IllegalArgumentException(
+            "Invalid 'faketime' format. Must be a long representing epoch seconds, but was: '"
+                + executeRequest.faketime
+                + "'",
+            e);
+      }
+
+      List<String> resolvedArgs = new java.util.ArrayList<>();
+      resolvedArgs.addAll(Arrays.asList(executeRequest.args));
+
+      int exitCode =
+          new picocli.CommandLine(new Tuf()).execute(resolvedArgs.toArray(new String[0]));
+
+      Map<String, Object> responseMap =
+          Map.of(
+              "stdout", outContent.toString(StandardCharsets.UTF_8),
+              "stderr", errContent.toString(StandardCharsets.UTF_8),
+              "exitCode", exitCode);
+      String jsonResponse = GSON.toJson(responseMap);
+
+      response.setStatus(HttpServletResponse.SC_OK);
+      response.setContentType("application/json");
+      byte[] responseBytes = jsonResponse.getBytes(StandardCharsets.UTF_8);
+      response.setContentLength(responseBytes.length);
+
+      try (OutputStream os = response.getOutputStream()) {
+        os.write(responseBytes);
+      }
+    } finally {
+      if (!debug) {
+        System.setOut(originalOut);
+        System.setErr(originalErr);
+      }
+      TestClock.reset();
+    }
+  }
+
+  private static void handleHealthCheck(HttpServletResponse response) throws IOException {
+    response.setStatus(HttpServletResponse.SC_OK);
+    response.getWriter().println("OK");
+  }
+}

tuf-cli/tuf-cli-server

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -o pipefail -o errexit -o nounset
+
+CWD=$PWD
+
+ARGS_JSON="["
+for arg in "$@"; do
+  escaped_arg=$(echo -n "$arg" | jq -R -s '.')
+  ARGS_JSON="$ARGS_JSON$escaped_arg,"
+done
+if [[ $ARGS_JSON == *, ]]; then
+  ARGS_JSON="${ARGS_JSON%,}"
+fi
+ARGS_JSON="$ARGS_JSON]"
+
+DATE_VAL="$(date +%s)"
+
+JSON_PAYLOAD=$(jq -nc --arg cwd "$CWD" --argjson args "$ARGS_JSON" --arg faketime "$DATE_VAL" '{"cwd": $cwd, "args": $args, "faketime": $faketime}')
+
+RESPONSE=$(curl -s -X POST --header "Content-Type: application/json" --data-binary "$JSON_PAYLOAD" http://localhost:8080/execute)
+
+STDOUT=$(echo "$RESPONSE" | jq -r .stdout)
+STDERR=$(echo "$RESPONSE" | jq -r .stderr)
+EXIT_CODE=$(echo "$RESPONSE" | jq .exitCode)
+
+echo -n "$STDOUT"
+echo -n "$STDERR" >&2
+
+exit "$EXIT_CODE"

tuf-cli/tuf-cli-server.xfails

@@ -0,0 +1,25 @@
+test_metadata_bytes_match
+test_unusual_role_name[?]
+test_unusual_role_name[#]
+test_unusual_role_name[/delegatedrole]
+test_unusual_role_name[../delegatedrole]
+test_snapshot_rollback[basic]
+test_snapshot_rollback[with
+test_static_repository[tuf-on-ci-0.11]
+test_graph_traversal[basic-delegation]
+test_graph_traversal[single-level-delegations]
+test_graph_traversal[two-level-delegations]
+test_graph_traversal[two-level-test-DFS-order-of-traversal]
+test_graph_traversal[three-level-delegation-test-DFS-order-of-traversal]
+test_graph_traversal[two-level-terminating-ignores-all-but-roles-descendants]
+test_graph_traversal[three-level-terminating-ignores-all-but-roles-descendants]
+test_graph_traversal[two-level-ignores-all-branches-not-matching-paths]
+test_graph_traversal[three-level-ignores-all-branches-not-matching-paths]
+test_graph_traversal[cyclic-graph]
+test_graph_traversal[two-roles-delegating-to-a-third]
+test_graph_traversal[two-roles-delegating-to-a-third-different-paths]
+test_targetfile_search[targetpath matches wildcard]
+test_targetfile_search[targetpath with separators x]
+test_targetfile_search[targetpath with separators y]
+test_targetfile_search[targetpath is not delegated by all roles in the chain]
+test_snapshot_rollback[with hashes]