enhance proto generation (#207)

Author: YunchuWang

.github/workflows/build-validation.yml

@@ -20,8 +20,6 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-      with:
-        submodules: true
 
     - name: Set up JDK 11
       uses: actions/setup-java@v2
@@ -58,13 +56,13 @@ jobs:
       run: ./gradlew integrationTest
 
     - name: Archive test report
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: Integration test report
         path: client/build/reports/tests/integrationTest
 
     - name: Upload JAR output
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: Package
         path: client/build/libs
@@ -76,8 +74,6 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-        with:
-          submodules: true
 
       - name: Set up JDK 8
         uses: actions/setup-java@v2
@@ -89,7 +85,7 @@ jobs:
         uses: gradle/gradle-build-action@v2
 
       - name: Publish to local
-        run: ./gradlew publishToMavenLocal -x sign
+        run: ./gradlew publishToMavenLocal
 
       - name: Build azure functions sample
         run: ./gradlew azureFunctionsPackage
@@ -105,7 +101,7 @@ jobs:
           arguments: endToEndTest
 
       - name: Archive test report
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: Integration test report
           path: client/build/reports/tests/endToEndTest
@@ -117,8 +113,6 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-        with:
-          submodules: true
 
       - name: Set up JDK 8
         uses: actions/setup-java@v2
@@ -146,7 +140,7 @@ jobs:
           arguments: sampleTest
 
       - name: Archive test report
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: Integration test report
           path: client/build/reports/tests/endToEndTest

.gitmodules

@@ -1,4 +1,5 @@
 ## placeholder
+* Add automatic proto file download and commit hash tracking during build ([#207](https://github.com/microsoft/durabletask-java/pull/207))
 * Fix infinite loop when use continueasnew after wait external event ([#183](https://github.com/microsoft/durabletask-java/pull/183))
 * Fix the issue "Deserialize Exception got swallowed when use anyOf with external event." ([#185](https://github.com/microsoft/durabletask-java/pull/185))
 

CHANGELOG.md

@@ -10,7 +10,6 @@ pool:
 
 steps:
 - checkout: self
-  submodules: true
 
 - task: Gradle@3
   inputs:

azdevops-pipeline/build-release-artifacts.yml

@@ -17,7 +17,7 @@ def jacksonVersion = '2.15.3'
 // When build on local, you need to set this value to your local jdk11 directory.
 // Java11 is used to compile and run all the tests.
 // Example for Windows:  C:/Program Files/Java/openjdk-11.0.12_7/
-def PATH_TO_TEST_JAVA_RUNTIME = "$System.env.JDK_11"
+def PATH_TO_TEST_JAVA_RUNTIME = System.env.JDK_11 ?: System.getProperty("java.home")
 
 dependencies {
 
@@ -49,20 +49,44 @@ compileTestJava {
     options.forkOptions.executable = "${PATH_TO_TEST_JAVA_RUNTIME}/bin/javac"
 }
 
+task downloadProtoFiles {
+    ext.branch = project.hasProperty('protoBranch') ? project.protoBranch : 'main'
+
+    doLast {
+        def protoDir = file("${rootProject.projectDir}/internal/durabletask-protobuf/protos")
+        protoDir.mkdirs()
+
+        // Download the proto file
+        new URL("https://raw.githubusercontent.com/microsoft/durabletask-protobuf/${ext.branch}/protos/orchestrator_service.proto")
+            .withInputStream { i -> 
+                new File(protoDir, 'orchestrator_service.proto').withOutputStream { it << i }
+            }
+
+        // Get and save the commit hash
+        def commitHashFile = new File("${rootProject.projectDir}/internal/durabletask-protobuf/PROTO_SOURCE_COMMIT_HASH")
+        def commitApiUrl = new URL("https://api.github.com/repos/microsoft/durabletask-protobuf/commits?path=protos/orchestrator_service.proto&sha=${ext.branch}&per_page=1")
+        def connection = commitApiUrl.openConnection()
+        connection.setRequestProperty('Accept', 'application/vnd.github.v3+json')
+        def commitHash = new groovy.json.JsonSlurper().parse(connection.inputStream)[0].sha
+        commitHashFile.text = commitHash
+    }
+}
+
 protobuf {
     protoc { artifact = "com.google.protobuf:protoc:${protocVersion}" }
     plugins {
         grpc { artifact = "io.grpc:protoc-gen-grpc-java:${grpcVersion}" }
     }
     generateProtoTasks {
         all()*.plugins { grpc {} }
+        all()*.dependsOn downloadProtoFiles
     }
 }
 
 sourceSets {
     main {
         proto {
-            srcDir '../submodules/durabletask-protobuf/protos'
+            srcDir '../internal/durabletask-protobuf/protos'
         }
         java {
             srcDirs 'build/generated/source/proto/main/grpc'

client/build.gradle

@@ -11,7 +11,6 @@ jobs:
 
       steps:
         - checkout: self
-          submodules: true
 
         - task: Gradle@3
           inputs:

eng/templates/build.yml

@@ -0,0 +1 @@
+4792f47019ab2b3e9ea979fb4af72427a4144c51

internal/durabletask-protobuf/PROTO_SOURCE_COMMIT_HASH

@@ -0,0 +1,24 @@
+# Durable Task Protobuf Files
+
+This directory contains the protocol buffer definitions used by the Durable Task Framework Java SDK. The files in this directory are automatically downloaded and updated during the build process from the [microsoft/durabletask-protobuf](https://github.com/microsoft/durabletask-protobuf) repository.
+
+## Directory Structure
+
+- `protos/` - Contains the downloaded proto files
+- `PROTO_SOURCE_COMMIT_HASH` - Contains the commit hash of the latest proto file version
+
+## Auto-Update Process
+
+The proto files are automatically downloaded and updated when running Gradle builds. This is handled by the `downloadProtoFiles` task in the `client/build.gradle` file. The task:
+
+1. Downloads the latest version of `orchestrator_service.proto`
+2. Saves the current commit hash for tracking purposes
+3. Updates these files before proto compilation begins
+
+## Manual Update
+
+If you need to manually update the proto files, you can run:
+
+```bash
+./gradlew downloadProtoFiles -PprotoBranch=<branch-name>
+```