Enable test retry for daily builds (open-telemetry#13086)

Author: laurit

.github/repository-settings.md

@@ -75,6 +75,7 @@ for [`dependabot/**/**`](https://github.com/open-telemetry/community/blob/main/d
   - Key is associated with [@trask](https://github.com/trask)'s gmail address
 - `SONATYPE_KEY` - owned by [@trask](https://github.com/trask)
 - `SONATYPE_USER` - owned by [@trask](https://github.com/trask)
+- `FLAKY_TEST_REPORTER_ACCESS_KEY` - owned by [@laurit](https://github.com/laurit)
 
 ### Organization secrets
 

.github/workflows/build-common.yml

@@ -290,6 +290,36 @@ jobs:
         if: ${{ !cancelled() && hashFiles('build-scan.txt') != '' }}
         run: cat build-scan.txt
 
+      - name: Get current job url
+        id: jobs
+        if: ${{ !cancelled() }}
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        env:
+          matrix: ${{ toJson(matrix) }}
+        with:
+          result-encoding: string
+          script: |
+            const { data: workflow_run } = await github.rest.actions.listJobsForWorkflowRun({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              run_id: context.runId,
+              per_page: 100
+            });
+            const matrix = JSON.parse(process.env.matrix);
+            const job_name = `common / test${ matrix['test-partition'] } (${ matrix['test-java-version'] }, ${ matrix.vm })`;
+            return workflow_run.jobs.find((job) => job.name === job_name).html_url;
+
+      - name: Flaky test report
+        if: ${{ !cancelled() }}
+        env:
+          FLAKY_TEST_REPORTER_ACCESS_KEY: ${{ secrets.FLAKY_TEST_REPORTER_ACCESS_KEY }}
+          JOB_URL: ${{ steps.jobs.outputs.result }}
+        run: |
+          if [ -s build-scan.txt ]; then
+            export BUILD_SCAN_URL=$(cat build-scan.txt)
+          fi
+          ./gradlew :test-report:reportFlakyTests
+
       - name: Upload deadlock detector artifacts if any
         if: failure()
         uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0

.github/workflows/build-daily-no-build-cache.yml

@@ -10,19 +10,16 @@ jobs:
   common:
     uses: ./.github/workflows/build-common.yml
     with:
-      max-test-retries: 0
       no-build-cache: true
 
   test-latest-deps:
     uses: ./.github/workflows/reusable-test-latest-deps.yml
     with:
-      max-test-retries: 0
       no-build-cache: true
 
   test-indy:
     uses: ./.github/workflows/reusable-test-indy.yml
     with:
-      max-test-retries: 0
       no-build-cache: true
 
   # muzzle is not included here because it doesn't use gradle cache anyway and so is already covered

.github/workflows/build-daily.yml

@@ -9,18 +9,12 @@ on:
 jobs:
   common:
     uses: ./.github/workflows/build-common.yml
-    with:
-      max-test-retries: 0
 
   test-latest-deps:
     uses: ./.github/workflows/reusable-test-latest-deps.yml
-    with:
-      max-test-retries: 0
 
   test-indy:
     uses: ./.github/workflows/reusable-test-indy.yml
-    with:
-      max-test-retries: 0
 
   muzzle:
     uses: ./.github/workflows/reusable-muzzle.yml

.github/workflows/reusable-test-indy.yml

@@ -86,3 +86,33 @@ jobs:
       - name: Build scan
         if: ${{ !cancelled() && hashFiles('build-scan.txt') != '' }}
         run: cat build-scan.txt
+
+      - name: Get current job url
+        id: jobs
+        if: ${{ !cancelled() }}
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        env:
+          matrix: ${{ toJson(matrix) }}
+        with:
+          result-encoding: string
+          script: |
+            const { data: workflow_run } = await github.rest.actions.listJobsForWorkflowRun({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              run_id: context.runId,
+              per_page: 100
+            });
+            const matrix = JSON.parse(process.env.matrix);
+            const job_name = `test-indy / testIndy${ matrix['test-partition'] }`;
+            return workflow_run.jobs.find((job) => job.name === job_name).html_url;
+
+      - name: Flaky test report
+        if: ${{ !cancelled() }}
+        env:
+          FLAKY_TEST_REPORTER_ACCESS_KEY: ${{ secrets.FLAKY_TEST_REPORTER_ACCESS_KEY }}
+          JOB_URL: ${{ steps.jobs.outputs.result }}
+        run: |
+          if [ -s build-scan.txt ]; then
+            export BUILD_SCAN_URL=$(cat build-scan.txt)
+          fi
+          ./gradlew :test-report:reportFlakyTests

.github/workflows/reusable-test-latest-deps.yml

@@ -85,6 +85,36 @@ jobs:
         if: ${{ !cancelled() && hashFiles('build-scan.txt') != '' }}
         run: cat build-scan.txt
 
+      - name: Get current job url
+        id: jobs
+        if: ${{ !cancelled() }}
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        env:
+          matrix: ${{ toJson(matrix) }}
+        with:
+          result-encoding: string
+          script: |
+            const { data: workflow_run } = await github.rest.actions.listJobsForWorkflowRun({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              run_id: context.runId,
+              per_page: 100
+            });
+            const matrix = JSON.parse(process.env.matrix);
+            const job_name = `test-latest-deps / testLatestDeps${ matrix['test-partition'] }`;
+            return workflow_run.jobs.find((job) => job.name === job_name).html_url;
+
+      - name: Flaky test report
+        if: ${{ !cancelled() }}
+        env:
+          FLAKY_TEST_REPORTER_ACCESS_KEY: ${{ secrets.FLAKY_TEST_REPORTER_ACCESS_KEY }}
+          JOB_URL: ${{ steps.jobs.outputs.result }}
+        run: |
+          if [ -s build-scan.txt ]; then
+            export BUILD_SCAN_URL=$(cat build-scan.txt)
+          fi
+          ./gradlew :test-report:reportFlakyTests
+
       - name: Upload deadlock detector artifacts if any
         if: failure()
         uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0

settings.gradle.kts

@@ -54,7 +54,8 @@ develocity {
     termsOfUseUrl.set("https://gradle.com/help/legal-terms-of-use")
     termsOfUseAgree.set("yes")
 
-    if (!gradle.startParameter.taskNames.contains("listTestsInPartition")) {
+    if (!gradle.startParameter.taskNames.contains("listTestsInPartition") &&
+      !gradle.startParameter.taskNames.contains(":test-report:reportFlakyTests")) {
       buildScanPublished {
         File("build-scan.txt").printWriter().use { writer ->
           writer.println(buildScanUri)
@@ -97,6 +98,7 @@ include(":instrumentation-annotations-support-testing")
 
 // misc
 include(":dependencyManagement")
+include(":test-report")
 include(":testing:agent-exporter")
 include(":testing:agent-for-testing")
 include(":testing:armeria-shaded-for-testing")

test-report/build.gradle.kts

@@ -0,0 +1,27 @@
+plugins {
+  id("otel.java-conventions")
+}
+
+dependencies {
+  implementation("com.google.api-client:google-api-client:2.7.1")
+  implementation("com.google.apis:google-api-services-sheets:v4-rev20250106-2.0.0")
+  implementation("com.google.auth:google-auth-library-oauth2-http:1.30.1")
+}
+
+otelJava {
+  minJavaVersionSupported.set(JavaVersion.VERSION_17)
+}
+
+tasks {
+  val reportFlakyTests by registering(JavaExec::class) {
+    dependsOn(classes)
+
+    mainClass.set("io.opentelemetry.instrumentation.testreport.FlakyTestReporter")
+    classpath(sourceSets["main"].runtimeClasspath)
+
+    systemProperty("scanPath", project.rootDir)
+    systemProperty("googleSheetsAccessKey", System.getenv("FLAKY_TEST_REPORTER_ACCESS_KEY"))
+    systemProperty("buildScanUrl", System.getenv("BUILD_SCAN_URL"))
+    systemProperty("jobUrl", System.getenv("JOB_URL"))
+  }
+}