Skip to content

Commit 1bf5629

Browse files
damccormdamccorm
authored
Fix publishing of ml/distroless images (#36548)
* Fix publishing of ml/distroless images * ml snapshots * Fix license script issue
1 parent af748d0 commit 1bf5629

File tree

5 files changed

+30
-21
lines changed

5 files changed

+30
-21
lines changed

‎.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,11 @@ jobs:
7373
- "python:container:distroless:py311:docker"
7474
- "python:container:distroless:py312:docker"
7575
- "python:container:distroless:py313:docker"
76+
- "python:container:ml:py39:docker"
77+
- "python:container:ml:py310:docker"
78+
- "python:container:ml:py311:docker"
79+
- "python:container:ml:py312:docker"
80+
- "python:container:ml:py313:docker"
7681
- "java:expansion-service:container:docker"
7782
steps:
7883
- uses: actions/checkout@v4

‎.github/workflows/build_release_candidate.yml‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -259,14 +259,14 @@ jobs:
259259
# Any task which is skipped from a broader task must be explicitly included in this list to avoid accidentally missing new
260260
# tasks as they are added.
261261
images_to_publish: [
262-
{"gradle_task": ":pushAllRunnersDockerImages", "skip_flags": ""},
263-
{"gradle_task": ":sdks:python:container:push39", "skip_flags": ""},
264-
{"gradle_task": ":sdks:python:container:push310", "skip_flags": ""},
265-
{"gradle_task": ":sdks:python:container:push311", "skip_flags": ""},
266-
{"gradle_task": ":sdks:python:container:push312", "skip_flags": ""},
267-
{"gradle_task": ":sdks:python:container:pushAll", "skip_flags": "-Pskip-python-39-images -Pskip-python-310-images -Pskip-python-311-images -Pskip-python-312-images"},
268-
{"gradle_task": ":pushAllSdkDockerImages", "skip_flags": "-Pskip-python-images"},
269-
{"gradle_task": ":pushAllDockerImages", "skip_flags": "-Pskip-runner-images -Pskip-sdk-images"}
262+
{"gradle_task": ":pushAllRunnersDockerImages", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"},
263+
{"gradle_task": ":sdks:python:container:push39", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"},
264+
{"gradle_task": ":sdks:python:container:push310", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"},
265+
{"gradle_task": ":sdks:python:container:push311", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"},
266+
{"gradle_task": ":sdks:python:container:push312", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"},
267+
{"gradle_task": ":sdks:python:container:pushAll", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless -Pskip-python-39-images -Pskip-python-310-images -Pskip-python-311-images -Pskip-python-312-images"},
268+
{"gradle_task": ":pushAllSdkDockerImages", "include_skip_flags": "-Pskip-python-images"},
269+
{"gradle_task": ":pushAllDockerImages", "include_skip_flags": "-Pskip-runner-images -Pskip-sdk-images"}
270270
]
271271
steps:
272272
- name: Checkout
@@ -298,7 +298,7 @@ jobs:
298298
username: ${{ secrets.DOCKERHUB_USER }}
299299
password: ${{ secrets.DOCKERHUB_TOKEN }}
300300
- name: Push docker images
301-
run: ./gradlew ${{ matrix.images_to_publish.gradle_task }} -PisRelease -Pdocker-pull-licenses -Pprune-images ${{ matrix.images_to_publish.skip_flags }} -Pdocker-tag=${{ github.event.inputs.RELEASE }}rc${{ github.event.inputs.RC }} --no-daemon --no-parallel
301+
run: ./gradlew ${{ matrix.images_to_publish.gradle_task }} -PisRelease -Pdocker-pull-licenses -Pprune-images ${{ matrix.images_to_publish.include_skip_flags }} -Pdocker-tag=${{ github.event.inputs.RELEASE }}rc${{ github.event.inputs.RC }} --no-daemon --no-parallel
302302

303303
beam_site_pr:
304304
if: ${{ fromJson(github.event.inputs.STAGE).beam_site_pr == 'yes'}}

‎.github/workflows/republish_released_docker_containers.yml‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -46,14 +46,14 @@ jobs:
4646
# Any task which is skipped from a broader task must be explicitly included in this list to avoid accidentally missing new
4747
# tasks as they are added.
4848
images_to_publish: [
49-
{"gradle_task": ":pushAllRunnersDockerImages", "skip_flags": ""},
50-
{"gradle_task": ":sdks:python:container:push39", "skip_flags": ""},
51-
{"gradle_task": ":sdks:python:container:push310", "skip_flags": ""},
52-
{"gradle_task": ":sdks:python:container:push311", "skip_flags": ""},
53-
{"gradle_task": ":sdks:python:container:push312", "skip_flags": ""},
54-
{"gradle_task": ":sdks:python:container:pushAll", "skip_flags": "-Pskip-python-39-images -Pskip-python-310-images -Pskip-python-311-images -Pskip-python-312-images"},
55-
{"gradle_task": ":pushAllSdkDockerImages", "skip_flags": "-Pskip-python-images"},
56-
{"gradle_task": ":pushAllDockerImages", "skip_flags": "-Pskip-runner-images -Pskip-sdk-images"}
49+
{"gradle_task": ":pushAllRunnersDockerImages", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"},
50+
{"gradle_task": ":sdks:python:container:push39", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"},
51+
{"gradle_task": ":sdks:python:container:push310", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"},
52+
{"gradle_task": ":sdks:python:container:push311", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"},
53+
{"gradle_task": ":sdks:python:container:push312", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"},
54+
{"gradle_task": ":sdks:python:container:pushAll", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless -Pskip-python-39-images -Pskip-python-310-images -Pskip-python-311-images -Pskip-python-312-images"},
55+
{"gradle_task": ":pushAllSdkDockerImages", "include_skip_flags": "-Pskip-python-images"},
56+
{"gradle_task": ":pushAllDockerImages", "include_skip_flags": "-Pskip-runner-images -Pskip-sdk-images"}
5757
]
5858
steps:
5959
- name: Checkout
@@ -94,7 +94,7 @@ jobs:
9494
./gradlew ${{ matrix.images_to_publish.gradle_task }} \
9595
-PisRelease \
9696
-Pdocker-pull-licenses \
97-
-Pprune-images ${{ matrix.images_to_publish.skip_flags }} \
97+
-Pprune-images ${{ matrix.images_to_publish.include_skip_flags }} \
9898
-Pdocker-repository-root=gcr.io/apache-beam-testing/updated_released_container_images \
9999
-Pdocker-tag-list=${{ env.release }},${{ github.sha }},$(date +'%Y-%m-%d') \
100100
--no-daemon \

‎sdks/python/container/build.gradle‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,12 @@ for(int i=min_python_version; i<=max_python_version; ++i) {
5757
}
5858
}
5959
dependsOn ':sdks:python:container:py' + cur + ':docker'
60+
if (project.hasProperty("include-ml")) {
61+
dependsOn ':sdks:python:container:ml:push3' + cur
62+
}
63+
if (project.hasProperty("include-distroless")) {
64+
dependsOn ':sdks:python:container:distroless:push3' + cur
65+
}
6066

6167
doLast {
6268
if (project.hasProperty("prune-images")) {
@@ -70,8 +76,6 @@ for(int i=min_python_version; i<=max_python_version; ++i) {
7076
}
7177

7278
tasks.register("pushAll") {
73-
dependsOn ':sdks:python:container:distroless:pushAll'
74-
dependsOn ':sdks:python:container:ml:pushAll'
7579
for(int ver=min_python_version; ver<=max_python_version; ++ver) {
7680
if (!project.hasProperty("skip-python-3" + ver + "-images")) {
7781
dependsOn ':sdks:python:container:push3' + ver

‎sdks/python/container/ml/common.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ def copyDockerfileDependencies = tasks.register("copyDockerfileDependencies", Co
6767
}
6868

6969
def copyLicenseScripts = tasks.register("copyLicenseScripts", Copy){
70-
from ("../license_scripts")
70+
from ("../../license_scripts")
7171
into "build/target/license_scripts"
7272
}
7373

0 commit comments

Comments
 (0)