Add the workflow to test the latest Beam release (#17)

* added the workflow to test the latest Beam

* add the version

* fix the venv

* fix the requests

* fix the pip

* add the packaging

* change to test-latest-env

Author: liferoad

.github/workflows/docker_test.yml

@@ -0,0 +1,72 @@
+#  Copyright 2023 Google LLC
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+
+#       https://www.apache.org/licenses/LICENSE-2.0
+
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+name: Build and push Docker image to GCP Artifact Registry with the latest Beam
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: false
+
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: false
+          docker-images: true
+          swap-storage: true
+      - name: Checkout
+        uses: actions/checkout@v3
+      - id: "auth"
+        name: Authenticate to Google Cloud
+        uses: google-github-actions/[email protected]
+        with:
+          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
+          token_format: access_token
+      - name: Docker login
+        uses: "docker/login-action@v1"
+        with:
+          registry: "us-docker.pkg.dev"
+          username: "oauth2accesstoken"
+          password: "${{ steps.auth.outputs.access_token }}"
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.8"
+      - name: Init env with the test Beam and docker URI
+        run: |
+          cp tests/sample.env.tf .env
+          make init-venv
+          ./venv/bin/pip install requests packaging
+          make test-latest-env
+          sed -i '/CUSTOM_CONTAINER_IMAGE=/d' .env
+          echo -e "\n" >> .env
+          echo "CUSTOM_CONTAINER_IMAGE=us-docker.pkg.dev/apache-beam-testing/dataflow-ml-starter/tf_gpu:test-beam" >> .env
+          echo '${{ steps.auth.outputs.access_token }}' | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev
+          make init
+      - name: Build and push Docker image
+        run: |
+          make docker
+      - name: Test Docker image
+        run: |
+          make run-df-gpu

Makefile

@@ -48,6 +48,11 @@ help: ## Print this help
 	@echo
 	@$(PYTHON) -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST)
 
+test-latest-env: ## Replace the Beam vesion with the latest version (including release candidates)
+	$(eval LATEST_VERSION=$(shell ./venv/bin/python3 scripts/get_beam_version.py))
+	@echo $(LATEST_VERSION)
+	@sed 's/BEAM_VERSION=.*/BEAM_VERSION=$(LATEST_VERSION)/g' .env > .env.new && mv .env.new .env
+
 init-venv: ## Create virtual environment in venv folder
 	@$(PYTHON) -m venv venv
 

scripts/get_beam_version.py

@@ -0,0 +1,28 @@
+#  Copyright 2023 Google LLC
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+
+#       https://www.apache.org/licenses/LICENSE-2.0
+
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+# third party libraries
+import requests
+from packaging.version import Version
+
+
+def beam_versions(package_name, limit_releases=10):
+    url = f"https://pypi.org/pypi/{package_name}/json"
+    data = requests.get(url).json()
+    versions = list(data["releases"].keys())
+    versions.sort(key=Version, reverse=True)
+    return versions[:limit_releases]
+
+
+print("\n".join(beam_versions("apache-beam", 1)))