Merge pull request #33088 from shunping/fix-postcommit-python

Revert PR #32960

Author: Abacn

.github/trigger_files/beam_PostCommit_Python.json

@@ -1,5 +1,5 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run.",
-  "modification": 3
+  "modification": 4
 }
 

sdks/python/container/Dockerfile

@@ -103,33 +103,9 @@ RUN if [ "$pull_licenses" = "true" ] ; then \
       python /tmp/license_scripts/pull_licenses_py.py ; \
     fi
 
-FROM beam as base
+FROM beam
 ARG pull_licenses
 COPY --from=third_party_licenses /opt/apache/beam/third_party_licenses /opt/apache/beam/third_party_licenses
 RUN if [ "$pull_licenses" != "true" ] ; then \
       rm -rf /opt/apache/beam/third_party_licenses ; \
     fi
-
-ARG TARGETARCH
-FROM gcr.io/distroless/python3-debian12:latest-${TARGETARCH} as distroless
-ARG py_version
-
-# Contains header files needed by the Python interpreter.
-COPY --from=base /usr/local/include /usr/local/include
-
-# Contains the Python interpreter executables.
-COPY --from=base /usr/local/bin /usr/local/bin
-
-# Contains the Python library dependencies.
-COPY --from=base /usr/local/lib /usr/local/lib
-
-# Python standard library modules.
-COPY --from=base /usr/lib/python${py_version} /usr/lib/python${py_version}
-
-# Contains the boot entrypoint and related files such as licenses.
-COPY --from=base /opt /opt
-
-ENV PATH "$PATH:/usr/local/bin"
-
-# Despite the ENTRYPOINT set above, need to reset since deriving the layer derives from a different image.
-ENTRYPOINT ["/opt/apache/beam/boot"]

sdks/python/container/common.gradle

@@ -71,16 +71,10 @@ def copyLauncherDependencies = tasks.register("copyLauncherDependencies", Copy)
 }
 
 def pushContainers = project.rootProject.hasProperty(["isRelease"]) || project.rootProject.hasProperty("push-containers")
-def baseBuildTarget = 'base'
-def buildTarget = project.findProperty('container-build-target') ?: 'base'
-var imageName = project.docker_image_default_repo_prefix + "python${project.ext.pythonVersion}_sdk"
-if (buildTarget != baseBuildTarget) {
-  imageName += "_${buildTarget}"
-}
 
 docker {
   name containerImageName(
-          name: imageName,
+          name: project.docker_image_default_repo_prefix + "python${project.ext.pythonVersion}_sdk",
           root: project.rootProject.hasProperty(["docker-repository-root"]) ?
                   project.rootProject["docker-repository-root"] :
                   project.docker_image_default_repo_root,
@@ -96,7 +90,6 @@ docker {
   platform(*project.containerPlatforms())
   load project.useBuildx() && !pushContainers
   push pushContainers
-  target buildTarget
 }
 
 dockerPrepare.dependsOn copyLauncherDependencies

sdks/python/test-suites/dataflow/build.gradle

@@ -60,12 +60,6 @@ task validatesContainerTests {
   }
 }
 
-task validatesDistrolessContainerTests {
-  getVersionsAsList('distroless_python_versions').each {
-    dependsOn.add(":sdks:python:test-suites:dataflow:py${getVersionSuffix(it)}:validatesDistrolessContainer")
-  }
-}
-
 task examplesPostCommit {
   getVersionsAsList('dataflow_examples_postcommit_py_versions').each {
     dependsOn.add(":sdks:python:test-suites:dataflow:py${getVersionSuffix(it)}:examples")

sdks/python/test-suites/dataflow/common.gradle

@@ -380,51 +380,6 @@ task validatesContainer() {
   }
 }
 
-/**
- * Validates the distroless (https://github.com/GoogleContainerTools/distroless) variant of the Python SDK container
- * image (sdks/python/container/Dockerfile).
- * To test a single version of Python:
- * ./gradlew :sdks:python:test-suites:dataflow:py311:validatesDistrolessContainer
- * See https://cwiki.apache.org/confluence/display/BEAM/Python+Tips#PythonTips-VirtualEnvironmentSetup
- * for more information on setting up different Python versions.
- */
-task validatesDistrolessContainer() {
-  def pyversion = "${project.ext.pythonVersion.replace('.', '')}"
-  def buildTarget = 'distroless'
-  def repository = "us.gcr.io/apache-beam-testing/${System.getenv('USER')}"
-  def tag = java.time.Instant.now().getEpochSecond()
-  def imageURL = "${repository}/beam_python${project.ext.pythonVersion}_sdk_${buildTarget}:${tag}"
-  project.rootProject.ext['docker-repository-root'] = repository
-  project.rootProject.ext['container-build-target'] = buildTarget
-  project.rootProject.ext['docker-tag'] = tag
-  if (project.rootProject.hasProperty('dry-run')) {
-    println "Running in dry run mode: imageURL: ${imageURL}, pyversion: ${pyversion}, buildTarget: ${buildTarget}, repository: ${repository}, tag: ${tag}, envdir: ${envdir}"
-    return
-  }
-  dependsOn 'initializeForDataflowJob'
-  dependsOn ":sdks:python:container:py${pyversion}:docker"
-  dependsOn ":sdks:python:container:py${pyversion}:dockerPush"
-  def testTarget = "apache_beam/examples/wordcount_it_test.py::WordCountIT::test_wordcount_it"
-  def argMap = [
-          "output": "gs://temp-storage-for-end-to-end-tests/py-it-cloud/output",
-          "project": "apache-beam-testing",
-          "region": "us-central1",
-          "runner": "TestDataflowRunner",
-          "sdk_container_image": "${imageURL}",
-          "sdk_location": "container",
-          "staging_location": "gs://temp-storage-for-end-to-end-tests/staging-it",
-          "temp_location": "gs://temp-storage-for-end-to-end-tests/temp-it",
-  ]
-  def cmdArgs = mapToArgString(argMap)
-  doLast {
-    exec {
-      workingDir = "${rootDir}/sdks/python"
-      executable 'sh'
-      args '-c', ". ${envdir}/bin/activate && pytest ${testTarget} --test-pipeline-options=\"${cmdArgs}\""
-    }
-  }
-}
-
 task validatesContainerARM() {
   def pyversion = "${project.ext.pythonVersion.replace('.', '')}"
   dependsOn 'initializeForDataflowJob'

sdks/python/test-suites/gradle.properties

@@ -54,6 +54,3 @@ prism_examples_postcommit_py_versions=3.9,3.12
 
 # cross language postcommit python test suites
 cross_language_validates_py_versions=3.9,3.12
-
-# Python versions to support distroless variants
-distroless_python_versions=3.9,3.10,3.11,3.12