From ea096bb739ab51c56e7ad3eaa3a2bebf9ee1ea43 Mon Sep 17 00:00:00 2001 From: aIbrahiim Date: Fri, 7 Nov 2025 13:40:13 +0200 Subject: [PATCH 1/7] Fix buildx image loading for Python container --- .../google-cloud-dataflow-java/build.gradle | 46 ++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index 8729bc2032ca..ca64a4361841 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -372,11 +372,55 @@ def buildAndPushDockerPythonContainer = tasks.create("buildAndPushDockerPythonCo root: "apache", tag: project.sdk_version) doLast { + def imageExists = false + try { + exec { + commandLine "docker", "inspect", "--type=image", "${defaultDockerImageName}" + ignoreExitValue = false + } + imageExists = true + } catch (Exception e) { + println "Image ${defaultDockerImageName} not found locally: ${e.message}" + println "Attempting to load image from buildx cache..." + + def pythonContainerProject = project.project(":sdks:python:container:py${pythonVer.replace('.', '')}") + def dockerBuildDir = pythonContainerProject.tasks.getByName("dockerPrepare").destinationDir + def dockerfile = new File(pythonContainerProject.projectDir, "../Dockerfile") + + def loadResult = exec { + commandLine "sh", "-c", """ + cd ${dockerBuildDir} && \\ + docker buildx build --output type=docker \\ + --tag ${defaultDockerImageName} \\ + --build-arg py_version=${pythonVer} \\ + --build-arg pull_licenses=true \\ + -f ${dockerfile.absolutePath} \\ + . + """ + ignoreExitValue = true + } + + if (loadResult.exitValue != 0) { + println "Failed to load from buildx cache. Attempting to pull from registry..." + def pullResult = exec { + commandLine "docker", "pull", "${defaultDockerImageName}" + ignoreExitValue = true + } + + if (pullResult.exitValue != 0) { + throw new GradleException( + "Docker image ${defaultDockerImageName} not found locally, in buildx cache, or in registry. " + + "Check the Docker build output for errors." + ) + } + } + } + exec { commandLine "docker", "tag", "${defaultDockerImageName}", "${dockerPythonImageName}" } exec { - commandLine "gcloud", "docker", "--", "push", "${dockerPythonImageName}" + commandLine "docker", "push", "${dockerPythonImageName}" } } } From 28f4a84b729ecf668001535ffa8f4e49c4c12c85 Mon Sep 17 00:00:00 2001 From: aIbrahiim Date: Fri, 7 Nov 2025 14:10:42 +0200 Subject: [PATCH 2/7] Added missing auth --- .../beam_PostCommit_XVR_GoUsingJava_Dataflow.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml index c22bec84760c..758ca0508902 100644 --- a/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml @@ -75,11 +75,23 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: default + - name: Authenticate to GCP + uses: google-github-actions/auth@v3 + with: + service_account: ${{ secrets.GCP_SA_EMAIL }} + credentials_json: ${{ secrets.GCP_SA_KEY }} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: GCloud Docker credential helper + - name: Configure Docker auth for GCR + run: | + gcloud --quiet auth configure-docker us.gcr.io + gcloud --quiet auth configure-docker gcr.io + gcloud auth list + - name: Docker login to GCR (explicit) run: | - gcloud auth configure-docker us.gcr.io + gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://us.gcr.io - name: run XVR GoUsingJava Dataflow script env: USER: github-actions From ada02ccf8f3bf711292ed9af94169c0d3c7879bb Mon Sep 17 00:00:00 2001 From: aIbrahiim Date: Fri, 7 Nov 2025 14:39:48 +0200 Subject: [PATCH 3/7] Fix gcloud config dir issue --- .../beam_PostCommit_XVR_GoUsingJava_Dataflow.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml index 758ca0508902..0a0db80735f1 100644 --- a/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml @@ -75,27 +75,39 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: default + - name: Set up writable gcloud config directory + run: | + mkdir -p /tmp/gcloud + echo "CLOUDSDK_CONFIG=/tmp/gcloud" >> $GITHUB_ENV - name: Authenticate to GCP + env: + CLOUDSDK_CONFIG: /tmp/gcloud uses: google-github-actions/auth@v3 with: service_account: ${{ secrets.GCP_SA_EMAIL }} credentials_json: ${{ secrets.GCP_SA_KEY }} - name: Set up Cloud SDK + env: + CLOUDSDK_CONFIG: /tmp/gcloud uses: google-github-actions/setup-gcloud@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Configure Docker auth for GCR + env: + CLOUDSDK_CONFIG: /tmp/gcloud run: | gcloud --quiet auth configure-docker us.gcr.io gcloud --quiet auth configure-docker gcr.io gcloud auth list - name: Docker login to GCR (explicit) + env: + CLOUDSDK_CONFIG: /tmp/gcloud run: | gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://us.gcr.io - name: run XVR GoUsingJava Dataflow script env: USER: github-actions - CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + CLOUDSDK_CONFIG: /tmp/gcloud uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerGoUsingJava From c56f25db05359b52d3e62c29c47153a63cd393df Mon Sep 17 00:00:00 2001 From: aIbrahiim Date: Fri, 7 Nov 2025 15:15:36 +0200 Subject: [PATCH 4/7] Fixed image push --- .../workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml | 7 +++++++ runners/google-cloud-dataflow-java/build.gradle | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml index 0a0db80735f1..f4a132577570 100644 --- a/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml @@ -96,14 +96,21 @@ jobs: env: CLOUDSDK_CONFIG: /tmp/gcloud run: | + export CLOUDSDK_CONFIG=/tmp/gcloud gcloud --quiet auth configure-docker us.gcr.io gcloud --quiet auth configure-docker gcr.io gcloud auth list + cat ~/.docker/config.json | grep -A 5 "us.gcr.io" || echo "Docker config check..." + echo "CLOUDSDK_CONFIG=/tmp/gcloud" >> ~/.docker/config.json.env || true - name: Docker login to GCR (explicit) env: CLOUDSDK_CONFIG: /tmp/gcloud run: | + export CLOUDSDK_CONFIG=/tmp/gcloud gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://us.gcr.io + docker info | grep -i "username" || echo "Docker auth configured" + export CLOUDSDK_CONFIG=/tmp/gcloud + echo "export CLOUDSDK_CONFIG=/tmp/gcloud" >> ~/.bashrc || true - name: run XVR GoUsingJava Dataflow script env: USER: github-actions diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index ca64a4361841..cc8ff04ac574 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -420,7 +420,8 @@ def buildAndPushDockerPythonContainer = tasks.create("buildAndPushDockerPythonCo commandLine "docker", "tag", "${defaultDockerImageName}", "${dockerPythonImageName}" } exec { - commandLine "docker", "push", "${dockerPythonImageName}" + environment "CLOUDSDK_CONFIG", System.getenv("CLOUDSDK_CONFIG") ?: System.getProperty("user.home") + "/.config/gcloud" + commandLine "gcloud", "docker", "--", "push", "${dockerPythonImageName}" } } } From 9cfb002a7b2d1c1f34683fb135d3ef6f31d54c33 Mon Sep 17 00:00:00 2001 From: aIbrahiim Date: Fri, 7 Nov 2025 15:38:28 +0200 Subject: [PATCH 5/7] use docker push command --- runners/google-cloud-dataflow-java/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index cc8ff04ac574..64f5ff653eb3 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -421,7 +421,7 @@ def buildAndPushDockerPythonContainer = tasks.create("buildAndPushDockerPythonCo } exec { environment "CLOUDSDK_CONFIG", System.getenv("CLOUDSDK_CONFIG") ?: System.getProperty("user.home") + "/.config/gcloud" - commandLine "gcloud", "docker", "--", "push", "${dockerPythonImageName}" + commandLine "docker", "push", "${dockerPythonImageName}" } } } From 738b2c8f0606e794f2413f6a2db34d389b4bf8da Mon Sep 17 00:00:00 2001 From: aIbrahiim Date: Sun, 9 Nov 2025 20:47:45 +0200 Subject: [PATCH 6/7] Refactoring --- ...am_PostCommit_XVR_GoUsingJava_Dataflow.yml | 35 ---- .../google-cloud-dataflow-java/build.gradle | 162 ++++++++---------- 2 files changed, 74 insertions(+), 123 deletions(-) diff --git a/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml index f4a132577570..cc52476ca303 100644 --- a/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml @@ -75,46 +75,11 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: default - - name: Set up writable gcloud config directory - run: | - mkdir -p /tmp/gcloud - echo "CLOUDSDK_CONFIG=/tmp/gcloud" >> $GITHUB_ENV - - name: Authenticate to GCP - env: - CLOUDSDK_CONFIG: /tmp/gcloud - uses: google-github-actions/auth@v3 - with: - service_account: ${{ secrets.GCP_SA_EMAIL }} - credentials_json: ${{ secrets.GCP_SA_KEY }} - - name: Set up Cloud SDK - env: - CLOUDSDK_CONFIG: /tmp/gcloud - uses: google-github-actions/setup-gcloud@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Configure Docker auth for GCR - env: - CLOUDSDK_CONFIG: /tmp/gcloud - run: | - export CLOUDSDK_CONFIG=/tmp/gcloud - gcloud --quiet auth configure-docker us.gcr.io - gcloud --quiet auth configure-docker gcr.io - gcloud auth list - cat ~/.docker/config.json | grep -A 5 "us.gcr.io" || echo "Docker config check..." - echo "CLOUDSDK_CONFIG=/tmp/gcloud" >> ~/.docker/config.json.env || true - - name: Docker login to GCR (explicit) - env: - CLOUDSDK_CONFIG: /tmp/gcloud - run: | - export CLOUDSDK_CONFIG=/tmp/gcloud - gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://us.gcr.io - docker info | grep -i "username" || echo "Docker auth configured" - export CLOUDSDK_CONFIG=/tmp/gcloud - echo "export CLOUDSDK_CONFIG=/tmp/gcloud" >> ~/.bashrc || true - name: run XVR GoUsingJava Dataflow script env: USER: github-actions - CLOUDSDK_CONFIG: /tmp/gcloud uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerGoUsingJava diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index 64f5ff653eb3..cf04ddd65e5d 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -212,17 +212,17 @@ def commonLegacyExcludeCategories = [ ] def commonRunnerV2ExcludeCategories = [ - 'org.apache.beam.sdk.testing.UsesExternalService', - 'org.apache.beam.sdk.testing.UsesGaugeMetrics', - 'org.apache.beam.sdk.testing.UsesSetState', - 'org.apache.beam.sdk.testing.UsesMapState', - 'org.apache.beam.sdk.testing.UsesMultimapState', - 'org.apache.beam.sdk.testing.UsesMetricsPusher', - 'org.apache.beam.sdk.testing.UsesOrderedListState', - 'org.apache.beam.sdk.testing.UsesTestStream', - 'org.apache.beam.sdk.testing.UsesTestStreamWithProcessingTime', - 'org.apache.beam.sdk.testing.UsesRequiresTimeSortedInput', - 'org.apache.beam.sdk.testing.UsesBoundedTrieMetrics', // Dataflow QM as of now does not support returning back BoundedTrie in metric result. + 'org.apache.beam.sdk.testing.UsesExternalService', + 'org.apache.beam.sdk.testing.UsesGaugeMetrics', + 'org.apache.beam.sdk.testing.UsesSetState', + 'org.apache.beam.sdk.testing.UsesMapState', + 'org.apache.beam.sdk.testing.UsesMultimapState', + 'org.apache.beam.sdk.testing.UsesMetricsPusher', + 'org.apache.beam.sdk.testing.UsesOrderedListState', + 'org.apache.beam.sdk.testing.UsesTestStream', + 'org.apache.beam.sdk.testing.UsesTestStreamWithProcessingTime', + 'org.apache.beam.sdk.testing.UsesRequiresTimeSortedInput', + 'org.apache.beam.sdk.testing.UsesBoundedTrieMetrics', // Dataflow QM as of now does not support returning back BoundedTrie in metric result. ] def createLegacyWorkerValidatesRunnerTest = { Map args -> @@ -241,7 +241,7 @@ def createLegacyWorkerValidatesRunnerTest = { Map args -> maxParallelForks Integer.MAX_VALUE classpath = configurations.validatesRunner testClassesDirs = files(project(":sdks:java:core").sourceSets.test.output.classesDirs) + - files(project(project.path).sourceSets.test.output.classesDirs) + files(project(project.path).sourceSets.test.output.classesDirs) useJUnit { includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner' commonLegacyExcludeCategories.each { @@ -274,7 +274,7 @@ def createRunnerV2ValidatesRunnerTest = { Map args -> maxParallelForks Integer.MAX_VALUE classpath = configurations.validatesRunner testClassesDirs = files(project(":sdks:java:core").sourceSets.test.output.classesDirs) + - files(project(project.path).sourceSets.test.output.classesDirs) + files(project(project.path).sourceSets.test.output.classesDirs) useJUnit { includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner' commonRunnerV2ExcludeCategories.each { @@ -306,9 +306,9 @@ def buildAndPushDockerJavaContainer = tasks.register("buildAndPushDockerJavaCont dependsOn ":sdks:java:container:${javaVer}:docker" def defaultDockerImageName = containerImageName( - name: "${project.docker_image_default_repo_prefix}${javaVer}_sdk", - root: "apache", - tag: project.sdk_version) + name: "${project.docker_image_default_repo_prefix}${javaVer}_sdk", + root: "apache", + tag: project.sdk_version) doLast { exec { commandLine "docker", "tag", "${defaultDockerImageName}", "${dockerJavaImageName}" @@ -368,60 +368,46 @@ def buildAndPushDockerPythonContainer = tasks.create("buildAndPushDockerPythonCo def pythonVer = project.project(':sdks:python').pythonVersion dependsOn ":sdks:python:container:py"+pythonVer.replace('.', '')+":docker" def defaultDockerImageName = containerImageName( - name: "${project.docker_image_default_repo_prefix}python${pythonVer}_sdk", - root: "apache", - tag: project.sdk_version) - doLast { - def imageExists = false - try { + name: "${project.docker_image_default_repo_prefix}python${pythonVer}_sdk", + root: "apache", + tag: project.sdk_version) + doFirst { + def cloudsdkConfig = System.getenv("CLOUDSDK_CONFIG") + if (cloudsdkConfig == null || !new File(cloudsdkConfig).canWrite()) { + def tmpGcloudDir = new File("/tmp/gcloud") + tmpGcloudDir.mkdirs() + System.setProperty("CLOUDSDK_CONFIG", "/tmp/gcloud") exec { - commandLine "docker", "inspect", "--type=image", "${defaultDockerImageName}" + environment "CLOUDSDK_CONFIG", "/tmp/gcloud" + commandLine "gcloud", "--quiet", "auth", "configure-docker", "us.gcr.io" ignoreExitValue = false } - imageExists = true - } catch (Exception e) { - println "Image ${defaultDockerImageName} not found locally: ${e.message}" - println "Attempting to load image from buildx cache..." - - def pythonContainerProject = project.project(":sdks:python:container:py${pythonVer.replace('.', '')}") - def dockerBuildDir = pythonContainerProject.tasks.getByName("dockerPrepare").destinationDir - def dockerfile = new File(pythonContainerProject.projectDir, "../Dockerfile") - - def loadResult = exec { - commandLine "sh", "-c", """ - cd ${dockerBuildDir} && \\ - docker buildx build --output type=docker \\ - --tag ${defaultDockerImageName} \\ - --build-arg py_version=${pythonVer} \\ - --build-arg pull_licenses=true \\ - -f ${dockerfile.absolutePath} \\ - . - """ - ignoreExitValue = true + exec { + environment "CLOUDSDK_CONFIG", "/tmp/gcloud" + commandLine "gcloud", "--quiet", "auth", "configure-docker", "gcr.io" + ignoreExitValue = false } - - if (loadResult.exitValue != 0) { - println "Failed to load from buildx cache. Attempting to pull from registry..." - def pullResult = exec { - commandLine "docker", "pull", "${defaultDockerImageName}" - ignoreExitValue = true - } - - if (pullResult.exitValue != 0) { - throw new GradleException( - "Docker image ${defaultDockerImageName} not found locally, in buildx cache, or in registry. " + - "Check the Docker build output for errors." - ) - } + } else { + exec { + environment "CLOUDSDK_CONFIG", cloudsdkConfig + commandLine "gcloud", "--quiet", "auth", "configure-docker", "us.gcr.io" + ignoreExitValue = false + } + exec { + environment "CLOUDSDK_CONFIG", cloudsdkConfig + commandLine "gcloud", "--quiet", "auth", "configure-docker", "gcr.io" + ignoreExitValue = false } } - + } + doLast { exec { commandLine "docker", "tag", "${defaultDockerImageName}", "${dockerPythonImageName}" } + def cloudsdkConfig = System.getenv("CLOUDSDK_CONFIG") ?: System.getProperty("CLOUDSDK_CONFIG") ?: "/tmp/gcloud" exec { - environment "CLOUDSDK_CONFIG", System.getenv("CLOUDSDK_CONFIG") ?: System.getProperty("user.home") + "/.config/gcloud" - commandLine "docker", "push", "${dockerPythonImageName}" + environment "CLOUDSDK_CONFIG", cloudsdkConfig + commandLine "gcloud", "docker", "--", "push", "${dockerPythonImageName}" } } } @@ -639,13 +625,13 @@ task googleCloudPlatformLegacyWorkerIntegrationTest(type: Test, dependsOn: copyG group = "Verification" dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" systemProperty "beamTestPipelineOptions", JsonOutput.toJson([ - "--runner=TestDataflowRunner", - "--project=${gcpProject}", - "--region=${gcpRegion}", - "--tempRoot=${dataflowPostCommitTempRoot}", - "--dataflowWorkerJar=${dataflowLegacyWorkerJar}", - "--workerHarnessContainerImage=", - "--firestoreDb=${firestoreDb}", + "--runner=TestDataflowRunner", + "--project=${gcpProject}", + "--region=${gcpRegion}", + "--tempRoot=${dataflowPostCommitTempRoot}", + "--dataflowWorkerJar=${dataflowLegacyWorkerJar}", + "--workerHarnessContainerImage=", + "--firestoreDb=${firestoreDb}", ]) include '**/*IT.class' @@ -675,14 +661,14 @@ task googleCloudPlatformLegacyWorkerKmsIntegrationTest(type: Test) { group = "Verification" dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" systemProperty "beamTestPipelineOptions", JsonOutput.toJson([ - "--runner=TestDataflowRunner", - "--project=${gcpProject}", - "--region=${gcpRegion}", - "--tempRoot=${dataflowPostCommitTempRootKms}", - "--dataflowWorkerJar=${dataflowLegacyWorkerJar}", - "--workerHarnessContainerImage=", - "--dataflowKmsKey=${dataflowKmsKey}", - "--firestoreDb=${firestoreDb}", + "--runner=TestDataflowRunner", + "--project=${gcpProject}", + "--region=${gcpRegion}", + "--tempRoot=${dataflowPostCommitTempRootKms}", + "--dataflowWorkerJar=${dataflowLegacyWorkerJar}", + "--workerHarnessContainerImage=", + "--dataflowKmsKey=${dataflowKmsKey}", + "--firestoreDb=${firestoreDb}", ]) include '**/*IT.class' @@ -777,12 +763,12 @@ task coreSDKJavaLegacyWorkerIntegrationTest(type: Test) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" systemProperty "beamTestPipelineOptions", JsonOutput.toJson([ - "--runner=TestDataflowRunner", - "--project=${gcpProject}", - "--region=${gcpRegion}", - "--tempRoot=${dataflowPostCommitTempRoot}", - "--dataflowWorkerJar=${dataflowLegacyWorkerJar}", - "--workerHarnessContainerImage=", + "--runner=TestDataflowRunner", + "--project=${gcpProject}", + "--region=${gcpRegion}", + "--tempRoot=${dataflowPostCommitTempRoot}", + "--dataflowWorkerJar=${dataflowLegacyWorkerJar}", + "--workerHarnessContainerImage=", ]) include '**/*IT.class' @@ -882,17 +868,17 @@ createJavaExamplesArchetypeValidationTask(type: 'MobileGaming', // Generates :runners:google-cloud-dataflow-java:runMobileGamingJavaDataflowBom createJavaExamplesArchetypeValidationTask(type: 'MobileGaming', - runner: 'DataflowBom', - gcpProject: gcpProject, - gcpRegion: gcpRegion, - gcsBucket: gcsBucket, - bqDataset: bqDataset, - pubsubTopic: pubsubTopic) + runner: 'DataflowBom', + gcpProject: gcpProject, + gcpRegion: gcpRegion, + gcsBucket: gcsBucket, + bqDataset: bqDataset, + pubsubTopic: pubsubTopic) // Standalone task for testing GCS upload, use with -PfilesToStage and -PgcpTempRoot. task GCSUpload(type: JavaExec) { mainClass = 'org.apache.beam.runners.dataflow.util.GCSUploadMain' classpath = sourceSets.test.runtimeClasspath args "--stagingLocation=${dataflowUploadTemp}/staging", - "--filesToStage=${testFilesToStage}" + "--filesToStage=${testFilesToStage}" } From eb363fc54beeb4e0b1b728112267414595b01de2 Mon Sep 17 00:00:00 2001 From: aIbrahiim Date: Mon, 10 Nov 2025 19:09:08 +0200 Subject: [PATCH 7/7] eliminate if conditions duplication --- .../google-cloud-dataflow-java/build.gradle | 38 ++++++++----------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index cf04ddd65e5d..e21d41aa0705 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -374,30 +374,22 @@ def buildAndPushDockerPythonContainer = tasks.create("buildAndPushDockerPythonCo doFirst { def cloudsdkConfig = System.getenv("CLOUDSDK_CONFIG") if (cloudsdkConfig == null || !new File(cloudsdkConfig).canWrite()) { - def tmpGcloudDir = new File("/tmp/gcloud") + cloudsdkConfig = "/tmp/gcloud" + } + if (cloudsdkConfig == "/tmp/gcloud") { + def tmpGcloudDir = new File(cloudsdkConfig) tmpGcloudDir.mkdirs() - System.setProperty("CLOUDSDK_CONFIG", "/tmp/gcloud") - exec { - environment "CLOUDSDK_CONFIG", "/tmp/gcloud" - commandLine "gcloud", "--quiet", "auth", "configure-docker", "us.gcr.io" - ignoreExitValue = false - } - exec { - environment "CLOUDSDK_CONFIG", "/tmp/gcloud" - commandLine "gcloud", "--quiet", "auth", "configure-docker", "gcr.io" - ignoreExitValue = false - } - } else { - exec { - environment "CLOUDSDK_CONFIG", cloudsdkConfig - commandLine "gcloud", "--quiet", "auth", "configure-docker", "us.gcr.io" - ignoreExitValue = false - } - exec { - environment "CLOUDSDK_CONFIG", cloudsdkConfig - commandLine "gcloud", "--quiet", "auth", "configure-docker", "gcr.io" - ignoreExitValue = false - } + System.setProperty("CLOUDSDK_CONFIG", cloudsdkConfig) + } + exec { + environment "CLOUDSDK_CONFIG", cloudsdkConfig + commandLine "gcloud", "--quiet", "auth", "configure-docker", "us.gcr.io" + ignoreExitValue = false + } + exec { + environment "CLOUDSDK_CONFIG", cloudsdkConfig + commandLine "gcloud", "--quiet", "auth", "configure-docker", "gcr.io" + ignoreExitValue = false } } doLast {