Building docker images using GitHub actions (#13)

liferoad · web-flow · commit 4f29dc0aea79 · 2023-07-18T11:34:45.000-04:00
* add directrunner tests

* fix the torch check

* split tests

* fix test

* fix sed

* fix path

* fix echo

* test cat

* fix newlines

* fix code coverage

* add creds to docker.yml

* polished sample env

* add workflow_dispatch

* add workflow_dispatch to pr

* update license

---------

Co-authored-by: xqhu &lt;xqhu@google.com&gt;
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -0,0 +1,48 @@
+#  Copyright 2023 Google LLC
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+
+#       https://www.apache.org/licenses/LICENSE-2.0
+
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+name: Build and push Docker image to GCP Artifact Registry
+
+on:
+    workflow_dispatch:
+    push:
+      branches:
+        - main
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@v1.1.1
+        with:
+          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
+          token_format: access_token
+      - name: Docker login -
+        uses: 'docker/login-action@v1'
+        with:
+            registry: 'us-docker.pkg.dev'
+            username: 'oauth2accesstoken'
+            password: '${{ steps.auth.outputs.access_token }}'
+      - name: Init env
+        run: |
+              cp tests/sample.env.tf .env
+              echo '${{ steps.auth.outputs.access_token }}' | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev
+              make init
+      - name: Build and push Docker image
+        run: |
+          make docker
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -1,9 +1,23 @@
+#  Copyright 2023 Google LLC
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+
+#       https://www.apache.org/licenses/LICENSE-2.0
+
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
 name: Run basic tests with Python 3.8
 
-on: [push, pull_request]
+on: [push, pull_request, workflow_dispatch]
 
 jobs:
-  build:
+  tests:
 
     runs-on: ubuntu-latest
 
@@ -20,3 +34,20 @@ jobs:
       - name: Run local tests
         run: |
             make test
+      - name: Run DirectRunner with TF
+        run: |
+            # tf model
+            make run-direct
+            test -f beam-output/beam_test_out.txt && echo "DirectRunner ran successfully!" || $(error "Cannot find beam-output/beam_test_out.txt!")
+      - name: Run DirectRunner with PyTorch
+        run: |
+            # torch model
+            sed -i '/TF_MODEL_URI=/d' .env
+            echo -e "\n" >> .env
+            echo "MODEL_STATE_DICT_PATH=gs://apache-beam-ml/models/torchvision.models.mobilenet_v2.pth" >> .env
+            echo -e "\n" >> .env
+            echo "MODEL_NAME=mobilenet_v2" >> .env
+            make run-direct
+            test -f beam-output/beam_test_out.txt && echo "DirectRunner ran successfully!" || $(error "Cannot find beam-output/beam_test_out.txt!")
+            # restore .env
+            cp tests/sample.env.tf .env
diff --git a/Makefile b/Makefile
@@ -80,9 +80,10 @@ clean: clean-lite ## Remove virtual environment, downloaded models, etc
 	@echo "run 'make init'"
 
 test: lint ## Run tests
-	@PYTHONPATH="./:./src" ./venv/bin/pytest -s -vv --cov-config=.coveragerc --cov-report html:htmlcov_v1 --cov-fail-under=50 tests/
+	@PYTHONPATH="./:./src" ./venv/bin/pytest -s -vv --cov=src --cov-fail-under=50 tests/
 
 run-direct: ## Run a local test with DirectRunner
+	@rm -f beam-output/beam_test_out.txt
 ifeq ($(MODEL_ENV), "TORCH")
 	time ./venv/bin/python3 -m src.run \
 	--input data/openimage_10.txt \
diff --git a/scripts/check-beam.sh b/scripts/check-beam.sh
@@ -1,5 +1,19 @@
 #!/bin/bash
 
+#  Copyright 2023 Google LLC
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+
+#       https://www.apache.org/licenses/LICENSE-2.0
+
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
 # Import environment variables from .env file.
 source .env
 
diff --git a/scripts/check-pipeline.sh b/scripts/check-pipeline.sh
@@ -1,5 +1,19 @@
 #!/bin/bash
 
+#  Copyright 2023 Google LLC
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+
+#       https://www.apache.org/licenses/LICENSE-2.0
+
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
 # Import environment variables from .env file.
 source .env
 
diff --git a/scripts/check-tf-on-gpu.sh b/scripts/check-tf-on-gpu.sh
@@ -1,5 +1,19 @@
 #!/bin/bash
 
+#  Copyright 2023 Google LLC
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+
+#       https://www.apache.org/licenses/LICENSE-2.0
+
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
 # Import environment variables from .env file.
 source .env
 
diff --git a/scripts/check-torch-on-gpu.sh b/scripts/check-torch-on-gpu.sh
@@ -1,5 +1,19 @@
 #!/bin/bash
 
+#  Copyright 2023 Google LLC
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+
+#       https://www.apache.org/licenses/LICENSE-2.0
+
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
 # Import environment variables from .env file.
 source .env
 
diff --git a/scripts/create-gpu-vm.sh b/scripts/create-gpu-vm.sh
@@ -1,5 +1,19 @@
 #!/bin/bash
 
+#  Copyright 2023 Google LLC
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+
+#       https://www.apache.org/licenses/LICENSE-2.0
+
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
 # Import environment variables from .env file.
 source .env
 
diff --git a/tests/sample.env.tf b/tests/sample.env.tf
@@ -19,7 +19,7 @@ VM_NAME=beam-ml-starter-gpu
 ################################################################################
 STAGING_LOCATION=gs://temp-storage-for-perf-tests/loadtests
 TEMP_LOCATION=gs://temp-storage-for-perf-tests/loadtests
-CUSTOM_CONTAINER_IMAGE=us-docker.pkg.dev/apache-beam-testing/xqhu/tf_gpu:test
+CUSTOM_CONTAINER_IMAGE=us-docker.pkg.dev/apache-beam-testing/dataflow-ml-starter/tf_gpu:test
 SERVICE_OPTIONS="worker_accelerator=type:nvidia-tesla-t4;count:1;install-nvidia-driver"
 ################################################################################
 ### DATAFLOW JOB MODEL SETTINGS
@@ -30,4 +30,4 @@ TF_MODEL_URI=https://tfhub.dev/google/tf2-preview/mobilenet_v2/classification/4
 ### DATAFLOW JOB INPUT&OUTPUT SETTINGS
 ################################################################################
 INPUT_DATA="gs://apache-beam-ml/testing/inputs/openimage_50k_benchmark.txt"
-OUTPUT_DATA="gs://temp-storage-for-end-to-end-tests/torch/result_gpu_xqhu.txt"
+OUTPUT_DATA="gs://temp-storage-for-end-to-end-tests/temp-storage-for-end-to-end-tests/dataflow-ml-starter/result_gpu.txt"
