diff --git a/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml index c22bec84760c..758ca0508902 100644 --- a/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml @@ -75,11 +75,23 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: default + - name: Authenticate to GCP + uses: google-github-actions/auth@v3 + with: + service_account: ${{ secrets.GCP_SA_EMAIL }} + credentials_json: ${{ secrets.GCP_SA_KEY }} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: GCloud Docker credential helper + - name: Configure Docker auth for GCR + run: | + gcloud --quiet auth configure-docker us.gcr.io + gcloud --quiet auth configure-docker gcr.io + gcloud auth list + - name: Docker login to GCR (explicit) run: | - gcloud auth configure-docker us.gcr.io + gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://us.gcr.io - name: run XVR GoUsingJava Dataflow script env: USER: github-actions diff --git a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml index 2548caed038e..e2f3f21264f2 100644 --- a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml +++ b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml @@ -98,7 +98,26 @@ jobs: uses: ./.github/actions/setup-environment-action with: java-version: default + - name: Authenticate to GCP + uses: google-github-actions/auth@v3 + with: + service_account: ${{ secrets.GCP_SA_EMAIL }} + credentials_json: ${{ secrets.GCP_SA_KEY }} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Configure Docker auth for GCR + run: | + gcloud --quiet auth configure-docker us.gcr.io + gcloud --quiet auth configure-docker gcr.io + gcloud auth list + - name: Docker login to GCR (explicit) + run: | + gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://us.gcr.io - name: run javaExamplesDataflowPrecommit script + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :javaExamplesDataflowPrecommit diff --git a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml index aa278822550c..ad71569b38c4 100644 --- a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml +++ b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml @@ -101,7 +101,26 @@ jobs: java-version: | 21 11 + - name: Authenticate to GCP + uses: google-github-actions/auth@v3 + with: + service_account: ${{ secrets.GCP_SA_EMAIL }} + credentials_json: ${{ secrets.GCP_SA_KEY }} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Configure Docker auth for GCR + run: | + gcloud --quiet auth configure-docker us.gcr.io + gcloud --quiet auth configure-docker gcr.io + gcloud auth list + - name: Docker login to GCR (explicit) + run: | + gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://us.gcr.io - name: Clean + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :clean @@ -110,6 +129,8 @@ jobs: -PdisableCheckStyle=true \ -PskipCheckerFramework \ - name: Build and Test + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:google-cloud-dataflow-java:examples:preCommit :runners:google-cloud-dataflow-java:examplesJavaDistrolessRunnerV2PreCommit diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index 8729bc2032ca..9ecc044ef56b 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -314,7 +314,7 @@ def buildAndPushDockerJavaContainer = tasks.register("buildAndPushDockerJavaCont commandLine "docker", "tag", "${defaultDockerImageName}", "${dockerJavaImageName}" } exec { - commandLine "gcloud", "docker", "--", "push", "${dockerJavaImageName}" + commandLine "docker", "push", "${dockerJavaImageName}" } } } @@ -332,7 +332,7 @@ def buildAndPushDistrolessDockerJavaContainer = tasks.register("buildAndPushDist commandLine "docker", "tag", "${defaultDockerImageName}", "${dockerJavaDistrolessImageName}" } exec { - commandLine "gcloud", "docker", "--", "push", "${dockerJavaDistrolessImageName}" + commandLine "docker", "push", "${dockerJavaDistrolessImageName}" } } } @@ -372,11 +372,61 @@ def buildAndPushDockerPythonContainer = tasks.create("buildAndPushDockerPythonCo root: "apache", tag: project.sdk_version) doLast { + // Check if image exists locally + def imageExists = false + try { + exec { + commandLine "docker", "inspect", "--type=image", "${defaultDockerImageName}" + ignoreExitValue = false + } + imageExists = true + } catch (Exception e) { + println "Image ${defaultDockerImageName} not found locally: ${e.message}" + def pythonContainerProject = project.project(":sdks:python:container:py${pythonVer.replace('.', '')}") + def dockerBuildDir = pythonContainerProject.tasks.getByName("dockerPrepare").destinationDir + def dockerfile = new File(pythonContainerProject.projectDir, "../Dockerfile") + + def loadResult = exec { + commandLine "sh", "-c", """ + cd ${dockerBuildDir} && \ + docker buildx build --load \ + --tag ${defaultDockerImageName} \ + --build-arg py_version=${pythonVer} \ + --build-arg pull_licenses=true \ + -f ${dockerfile.absolutePath} \ + . + """ + ignoreExitValue = true + } + + if (loadResult.exitValue != 0) { + println "Failed to load from buildx cache. Attempting to pull from registry..." + def pullResult = exec { + commandLine "docker", "pull", "${defaultDockerImageName}" + ignoreExitValue = true + } + + if (pullResult.exitValue != 0) { + throw new GradleException( + "Docker image ${defaultDockerImageName} not found locally, in buildx cache, or in registry. " + + "Check the Docker build output for errors." + ) + } + } + } + exec { commandLine "docker", "tag", "${defaultDockerImageName}", "${dockerPythonImageName}" } + + println "Verifying Docker authentication for ${dockerPythonImageName}" + exec { + commandLine "docker", "info" + ignoreExitValue = false + } + exec { - commandLine "gcloud", "docker", "--", "push", "${dockerPythonImageName}" + commandLine "docker", "push", "${dockerPythonImageName}" } } }