Distinguish failure types: javac, native-test

Author: vjovanov

.github/workflows/check-new-library-versions-in-batch.yml

@@ -60,7 +60,7 @@ jobs:
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       ISSUE_TITLE_PREFIX: "[Automation] Testing failed for library: "
-      ISSUE_BODY_PREFIX: "First failing version:  "
+      ISSUE_BODY_PREFIX: "Failure kind: "
     strategy:
       fail-fast: false
       matrix:
@@ -93,11 +93,10 @@ jobs:
           readarray -t VERSIONS < <(echo '${{ toJson(matrix.item.versions) }}' | jq -r '.[]')
           FIRST_TESTING_VERSION="${VERSIONS[0]}"
           
-          TITLE="${{ env.ISSUE_TITLE_PREFIX }}$GROUP_ID:$ARTIFACT_ID"
-          BODY="${{ env.ISSUE_BODY_PREFIX }}$GROUP_ID:$ARTIFACT_ID:$FIRST_TESTING_VERSION"
+          TITLE="${{ env.ISSUE_TITLE_PREFIX }}$GROUP_ID:$ARTIFACT_ID:$FIRST_TESTING_VERSION"
           
           ISSUE_NUMBER=$(gh issue list --repo "${{ github.repository }}" --state open --search "$TITLE" --json number,title,body --jq \
-            '.[] | select(.title == "'"$TITLE"'") | select(.body == "'"$BODY"'") | .number')
+            '.[] | select(.title == "'"$TITLE"'") | .number')
           
           if [[ -n "$ISSUE_NUMBER" ]]; then
             echo "There is no progress since last time this version was tested. Skipping further steps."
@@ -140,9 +139,8 @@ jobs:
           cat successful_versions.txt >> $GITHUB_OUTPUT
           echo "EOF" >> $GITHUB_OUTPUT
 
-          # Extract failed version
-          FAILED_VERSION=$(grep "^FAILED:" test_results.txt | sed 's/FAILED://g')
-          echo "failed_version=$FAILED_VERSION" >> $GITHUB_OUTPUT
+          # Extract failure type and failed version
+          awk -F'[][]|:' '/^FAILED/ {print "failure_type="$2; print "failed_version="$4}' test_results.txt >> $GITHUB_OUTPUT
 
       - name: "✔️ New library versions are supported (with locks, waiting and retries)"
         if: steps.runtests.outputs.successful_versions != ''
@@ -200,8 +198,8 @@ jobs:
           
           FAILED_VERSION="${{ steps.runtests.outputs.failed_version }}"
           REPO="${{ github.repository }}"
-          TITLE="${{ env.ISSUE_TITLE_PREFIX }}${{ matrix.item.name }}"
-          BODY="${{ env.ISSUE_BODY_PREFIX }}${{ matrix.item.name }}:$FAILED_VERSION"
+          TITLE="${{ env.ISSUE_TITLE_PREFIX }}${{ matrix.item.name }}:$FAILED_VERSION"
+          BODY="${{ env.ISSUE_BODY_PREFIX }}${{ steps.runtests.outputs.failure_type }}"
 
           ISSUE_NUMBER=$(gh issue list --repo "$REPO" --state open --search "$TITLE" --json number,title --jq '.[] | select(.title == "'"$TITLE"'") | .number')
           if [ -n "$ISSUE_NUMBER" ]; then

.github/workflows/run-consecutive-tests.sh

@@ -20,11 +20,19 @@ VERSIONS_JSON="${VERSIONS_JSON%"${VERSIONS_JSON##*[!\']}"}"
 readarray -t VERSIONS < <(echo "$VERSIONS_JSON" | jq -r '.[]')
 
 for VERSION in "${VERSIONS[@]}"; do
+  # check if javac works for the new version
+  GVM_TCK_LV="$VERSION" ./gradlew clean javac -Pcoordinates="$TEST_COORDINATES"
+  RESULT=$?
+  if [ "$RESULT" -ne 0 ]; then
+    echo "FAILED [javac]:$VERSION"
+    break
+  fi
+
   echo "Running test with GVM_TCK_LV=$VERSION and coordinates=$TEST_COORDINATES"
   GVM_TCK_LV="$VERSION" ./gradlew test -Pcoordinates="$TEST_COORDINATES"
   RESULT=$?
-
   ATTEMPTS=1
+
   # maybe we failed because the test was flaky => try two more times to be sure
   while [ "$RESULT" -ne 0 ] && [ $ATTEMPTS -le 2 ]; do
     echo "Re-running the test with GVM_TCK_LV=$VERSION and coordinates=$TEST_COORDINATES"
@@ -36,7 +44,7 @@ for VERSION in "${VERSIONS[@]}"; do
   if [ "$RESULT" -eq 0 ]; then
     echo "PASSED:$VERSION"
   else
-    echo "FAILED:$VERSION"
+    echo "FAILED [native-test]:$VERSION"
     break
   fi
 done

tests/tck-build-logic/src/main/groovy/org.graalvm.internal.tck-harness.gradle

@@ -19,6 +19,8 @@ import org.graalvm.internal.tck.TestedVersionUpdaterTask
 import org.graalvm.internal.tck.harness.tasks.TestInvocationTask
 import org.graalvm.internal.tck.harness.tasks.CheckstyleInvocationTask
 import org.graalvm.internal.tck.harness.tasks.CleanInvocationTask
+import org.graalvm.internal.tck.harness.tasks.JavacInvocationTask
+import org.graalvm.internal.tck.harness.tasks.JavaTestInvocationTask
 import org.graalvm.internal.tck.updaters.FetchExistingLibrariesWithNewerVersionsTask
 import org.graalvm.internal.tck.updaters.GroupUnsupportedLibraries
 
@@ -43,6 +45,18 @@ Provider<Task> checkstyle = tasks.register("checkstyle") { task ->
     task.setGroup(JavaBasePlugin.VERIFICATION_GROUP)
 }
 
+ // gradle javac -Pcoordinates=<maven-coordinates>
+Provider<Task> javac = tasks.register("javac") { task ->
+    task.setDescription("Compiles sources (javac) for all subprojects")
+    task.setGroup(LifecycleBasePlugin.BUILD_GROUP)
+}
+
+ // gradle javaTest -Pcoordinates=<maven-coordinates>
+Provider<Task> javaTest = tasks.register("javaTest") { task ->
+    task.setDescription("Runs JVM tests (Gradle 'test') for all subprojects")
+    task.setGroup(JavaBasePlugin.VERIFICATION_GROUP)
+}
+
 tasks.named("check").configure {
     dependsOn(checkstyle)
 }
@@ -72,6 +86,22 @@ for (String coordinates in matchingCoordinates) {
     clean.configure {
         dependsOn(cleanTaskName)
     }
+
+    String javacTaskName = generateTaskName("javac", coordinates)
+    if ((!tasks.getNames().contains(javacTaskName))) {
+        tasks.register(javacTaskName, JavacInvocationTask, coordinates)
+    }
+    javac.configure {
+        dependsOn(javacTaskName)
+    }
+
+    String javaTestTaskName = generateTaskName("javaTest", coordinates)
+    if ((!tasks.getNames().contains(javaTestTaskName))) {
+        tasks.register(javaTestTaskName, JavaTestInvocationTask, coordinates)
+    }
+    javaTest.configure {
+        dependsOn(javaTestTaskName)
+    }
 }
 
 // gradle diff -PbaseCommit=<base-commit> -PnewCommit=<new-commit>

tests/tck-build-logic/src/main/groovy/org/graalvm/internal/tck/harness/tasks/JavaTestInvocationTask.groovy

@@ -0,0 +1,35 @@
+/*
+ * Copyright and related rights waived via CC0
+ *
+ * You should have received a copy of the CC0 legalcode along with this
+ * work. If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+
+package org.graalvm.internal.tck.harness.tasks
+
+import org.gradle.api.tasks.Input
+
+import javax.inject.Inject
+/**
+ * Task that is used to run JVM tests (Gradle 'test') on subprojects.
+ */
+@SuppressWarnings("unused")
+abstract class JavaTestInvocationTask extends AbstractSubprojectTask {
+
+    @Inject
+    JavaTestInvocationTask(String coordinates) {
+        super(coordinates)
+    }
+
+    @Override
+    @Input
+    List<String> getCommand() {
+        return [tckExtension.repoRoot.get().asFile.toPath().resolve("gradlew").toString(), "test"]
+    }
+
+    @Override
+    protected String getErrorMessage(int exitCode) {
+        "Java tests failed"
+    }
+
+}

tests/tck-build-logic/src/main/groovy/org/graalvm/internal/tck/harness/tasks/JavacInvocationTask.groovy

@@ -0,0 +1,35 @@
+/*
+ * Copyright and related rights waived via CC0
+ *
+ * You should have received a copy of the CC0 legalcode along with this
+ * work. If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+
+package org.graalvm.internal.tck.harness.tasks
+
+import org.gradle.api.tasks.Input
+
+import javax.inject.Inject
+/**
+ * Task that is used to compile subprojects with javac.
+ */
+@SuppressWarnings("unused")
+abstract class JavacInvocationTask extends AbstractSubprojectTask {
+
+    @Inject
+    JavacInvocationTask(String coordinates) {
+        super(coordinates)
+    }
+
+    @Override
+    @Input
+    List<String> getCommand() {
+        return [tckExtension.repoRoot.get().asFile.toPath().resolve("gradlew").toString(), "compileTestJava"]
+    }
+
+    @Override
+    protected String getErrorMessage(int exitCode) {
+        "Compilation failed"
+    }
+
+}