From 58a7a078db0abc3eb19d8bf76537c626a61599f4 Mon Sep 17 00:00:00 2001 From: aIbrahiim Date: Thu, 6 Nov 2025 14:46:52 +0200 Subject: [PATCH 1/3] Fix Docker image tagging for buildx by pulling when missing --- ...am_PostCommit_XVR_GoUsingJava_Dataflow.yml | 16 +++++- .../google-cloud-dataflow-java/build.gradle | 56 ++++++++++++++++++- 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml index c22bec84760c..758ca0508902 100644 --- a/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml @@ -75,11 +75,23 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: default + - name: Authenticate to GCP + uses: google-github-actions/auth@v3 + with: + service_account: ${{ secrets.GCP_SA_EMAIL }} + credentials_json: ${{ secrets.GCP_SA_KEY }} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: GCloud Docker credential helper + - name: Configure Docker auth for GCR + run: | + gcloud --quiet auth configure-docker us.gcr.io + gcloud --quiet auth configure-docker gcr.io + gcloud auth list + - name: Docker login to GCR (explicit) run: | - gcloud auth configure-docker us.gcr.io + gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://us.gcr.io - name: run XVR GoUsingJava Dataflow script env: USER: github-actions diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index 8729bc2032ca..9ecc044ef56b 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -314,7 +314,7 @@ def buildAndPushDockerJavaContainer = tasks.register("buildAndPushDockerJavaCont commandLine "docker", "tag", "${defaultDockerImageName}", "${dockerJavaImageName}" } exec { - commandLine "gcloud", "docker", "--", "push", "${dockerJavaImageName}" + commandLine "docker", "push", "${dockerJavaImageName}" } } } @@ -332,7 +332,7 @@ def buildAndPushDistrolessDockerJavaContainer = tasks.register("buildAndPushDist commandLine "docker", "tag", "${defaultDockerImageName}", "${dockerJavaDistrolessImageName}" } exec { - commandLine "gcloud", "docker", "--", "push", "${dockerJavaDistrolessImageName}" + commandLine "docker", "push", "${dockerJavaDistrolessImageName}" } } } @@ -372,11 +372,61 @@ def buildAndPushDockerPythonContainer = tasks.create("buildAndPushDockerPythonCo root: "apache", tag: project.sdk_version) doLast { + // Check if image exists locally + def imageExists = false + try { + exec { + commandLine "docker", "inspect", "--type=image", "${defaultDockerImageName}" + ignoreExitValue = false + } + imageExists = true + } catch (Exception e) { + println "Image ${defaultDockerImageName} not found locally: ${e.message}" + def pythonContainerProject = project.project(":sdks:python:container:py${pythonVer.replace('.', '')}") + def dockerBuildDir = pythonContainerProject.tasks.getByName("dockerPrepare").destinationDir + def dockerfile = new File(pythonContainerProject.projectDir, "../Dockerfile") + + def loadResult = exec { + commandLine "sh", "-c", """ + cd ${dockerBuildDir} && \ + docker buildx build --load \ + --tag ${defaultDockerImageName} \ + --build-arg py_version=${pythonVer} \ + --build-arg pull_licenses=true \ + -f ${dockerfile.absolutePath} \ + . + """ + ignoreExitValue = true + } + + if (loadResult.exitValue != 0) { + println "Failed to load from buildx cache. Attempting to pull from registry..." + def pullResult = exec { + commandLine "docker", "pull", "${defaultDockerImageName}" + ignoreExitValue = true + } + + if (pullResult.exitValue != 0) { + throw new GradleException( + "Docker image ${defaultDockerImageName} not found locally, in buildx cache, or in registry. " + + "Check the Docker build output for errors." + ) + } + } + } + exec { commandLine "docker", "tag", "${defaultDockerImageName}", "${dockerPythonImageName}" } + + println "Verifying Docker authentication for ${dockerPythonImageName}" + exec { + commandLine "docker", "info" + ignoreExitValue = false + } + exec { - commandLine "gcloud", "docker", "--", "push", "${dockerPythonImageName}" + commandLine "docker", "push", "${dockerPythonImageName}" } } } From 03d50c52026101d2e63a90082fd4a281c14271d4 Mon Sep 17 00:00:00 2001 From: aIbrahiim Date: Thu, 6 Nov 2025 17:50:47 +0200 Subject: [PATCH 2/3] Add Docker auth to PreCommit Java workflows --- .../beam_PreCommit_Java_Examples_Dataflow.yml | 19 +++++++++++++++++++ ...reCommit_Java_Examples_Dataflow_Java21.yml | 17 +++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml index 2548caed038e..faab349cbdc2 100644 --- a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml +++ b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml @@ -98,7 +98,26 @@ jobs: uses: ./.github/actions/setup-environment-action with: java-version: default + - name: Authenticate to GCP + uses: google-github-actions/auth@v3 + with: + service_account: ${{ secrets.GCP_SA_EMAIL }} + credentials_json: ${{ secrets.GCP_SA_KEY }} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Configure Docker auth for GCR + run: | + gcloud --quiet auth configure-docker us.gcr.io + gcloud --quiet auth configure-docker gcr.io + gcloud auth list + - name: Docker login to GCR (explicit) + run: | + gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://us.gcr.io - name: run javaExamplesDataflowPrecommit script + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH }} uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :javaExamplesDataflowPrecommit diff --git a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml index aa278822550c..9fe9c2008f64 100644 --- a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml +++ b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml @@ -101,6 +101,23 @@ jobs: java-version: | 21 11 + - name: Authenticate to GCP + uses: google-github-actions/auth@v3 + with: + service_account: ${{ secrets.GCP_SA_EMAIL }} + credentials_json: ${{ secrets.GCP_SA_KEY }} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Configure Docker auth for GCR + run: | + gcloud --quiet auth configure-docker us.gcr.io + gcloud --quiet auth configure-docker gcr.io + gcloud auth list + - name: Docker login to GCR (explicit) + run: | + gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://us.gcr.io - name: Clean uses: ./.github/actions/gradle-command-self-hosted-action with: From a9bd87f238df5aed12e225bcfef68e160ace08b1 Mon Sep 17 00:00:00 2001 From: aIbrahiim Date: Thu, 6 Nov 2025 21:15:57 +0200 Subject: [PATCH 3/3] Fix Docker auth in PreCommit Java workflows --- .github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml | 2 +- .../beam_PreCommit_Java_Examples_Dataflow_Java21.yml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml index faab349cbdc2..e2f3f21264f2 100644 --- a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml +++ b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml @@ -117,7 +117,7 @@ jobs: gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://us.gcr.io - name: run javaExamplesDataflowPrecommit script env: - CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH }} + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :javaExamplesDataflowPrecommit diff --git a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml index 9fe9c2008f64..ad71569b38c4 100644 --- a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml +++ b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml @@ -119,6 +119,8 @@ jobs: run: | gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://us.gcr.io - name: Clean + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :clean @@ -127,6 +129,8 @@ jobs: -PdisableCheckStyle=true \ -PskipCheckerFramework \ - name: Build and Test + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:google-cloud-dataflow-java:examples:preCommit :runners:google-cloud-dataflow-java:examplesJavaDistrolessRunnerV2PreCommit