ci: use GitHub actions (#765)

reliveyy · web-flow · commit 7fad8108126b · 2020-11-03T10:00:31.000+08:00
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,55 @@
+name: ci
+
+on: push
+
+jobs:
+  build-amd64:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+          architecture: 'x64'
+      - name: Show Python version
+        run: python -c "import sys; print(sys.version)"
+      - name: Build and push
+        run: tools/push -p linux/amd64
+  build-arm64:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+          architecture: 'x64'
+      - name: Show Python version
+        run: python -c "import sys; print(sys.version)"
+      - name: Build and push
+        run: tools/push -p linux/arm64
diff --git a/README.md b/README.md
@@ -11,7 +11,7 @@ The following instructions are geared towards developers, intending to contribut
 
 ### Prerequisites
 
-Python 3.8 (or higher)
+Python 3.7 (or higher)
 
 ### Initial setup
 
diff --git a/tools/build b/tools/build
@@ -1,3 +1,7 @@
 #!/bin/bash
 
-python $(dirname "$0")/helper.py build "$@"
+set -euo pipefail
+
+cd "$(dirname "$0")" || exit 1
+
+python3 helper.py build "$@"
diff --git a/tools/core/docker.py b/tools/core/docker.py
@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Dict, Optional, List, Literal, Union
+from typing import TYPE_CHECKING, Dict, Optional, List, Union
 
 import json
 from urllib.request import urlopen, Request
@@ -17,7 +17,7 @@
     from http.client import HTTPResponse
 
 
-SupportedPlatform = Literal["linux/arm64", "linux/amd64", "linux/386", "linux/ppc64le", "linux/s390s", "linux/arm/v7", "linux/arm/v6"]
+# SupportedPlatform = Literal["linux/arm64", "linux/amd64", "linux/386", "linux/ppc64le", "linux/s390s", "linux/arm/v7", "linux/arm/v6"]
 
 
 class Platform:
diff --git a/tools/core/image.py b/tools/core/image.py
@@ -138,26 +138,29 @@ def repo(self) -> Optional[str]:
         return repo
 
     def _run_command(self, cmd):
+        on_travis = "TRAVIS_BRANCH" in os.environ
+        if on_travis:
+            self._run_command_on_travis(cmd)
+            return
+        print("\033[34m$ %s\033[0m" % cmd, flush=True)
+        exit_code = os.system(cmd)
+        if exit_code != 0:
+            raise RuntimeError("Failed to build (exit_code=%s)" % exit_code)
+
+    def _run_command_on_travis(self, cmd):
         self._logger.info(cmd)
 
         stop = threading.Event()
 
         def f():
             nonlocal stop
             counter = 0
-            on_travis = "TRAVIS_BRANCH" in os.environ
+
             while not stop.is_set():
                 counter = counter + 1
+                print("Still building... ({})".format(counter), flush=True)
+                stop.wait(10)
 
-                if on_travis:
-                    print("Still building... ({})".format(counter), flush=True)
-                    stop.wait(10)
-                    continue
-
-                print(".", end="", flush=True)
-                stop.wait(1)
-            if not on_travis:
-                print()
         threading.Thread(target=f).start()
         try:
             output = execute(cmd)
@@ -183,8 +186,14 @@ def _buildx_build(self, args: List[str], build_dir: str, build_tag: str, platfor
         self._run_command(cmd)
 
     def build(self, platform: Platform, no_cache: bool) -> None:
-        self._logger.info("Build %s:%s (%s)", self.name, self.tag, platform.tag_suffix)
-        print("Build %s:%s (%s)" % (self.name, self.tag, platform.tag_suffix))
+        self._logger.info("Building %s:%s (%s)", self.name, self.tag, platform.tag_suffix)
+
+        print()
+        print("=" * 80)
+        print("Building %s:%s (%s)" % (self.name, self.tag, platform.tag_suffix))
+        print("=" * 80)
+
+        sys.stdout.flush()
 
         source_manager = self.prepare()
 
@@ -257,18 +266,25 @@ def push(self, platform: Platform, no_cache: bool = False, dirty_push: bool = Fa
 
         tag = self.get_build_tag(self.branch, platform)
 
-        print("Push {}".format(tag))
+        print()
+        print("=" * 80)
+        print("Pushing {}".format(tag))
+        print("=" * 80)
+
+        sys.stdout.flush()
 
         cmd = "docker push {}".format(tag)
+        print("\033[34m$ %s\033[0m" % cmd, flush=True)
         output = execute(cmd)
-        self._logger.debug("$ %s\n%s", cmd, output)
+        print("%s" % output.rstrip(), flush=True)
         last_line = output.splitlines()[-1]
         p = re.compile(r"^(.*): digest: (.*) size: (\d+)$")
         m = p.match(last_line)
         assert m
         assert m.group(1) in tag
 
         new_manifest = "{}/{}@{}".format(self.group, self.name, m.group(2))
+        print("New manifest: %s" % new_manifest, flush=True)
 
         # append to manifest list
         os.environ["DOCKER_CLI_EXPERIMENTAL"] = "enabled"
@@ -284,22 +300,28 @@ def push(self, platform: Platform, no_cache: bool = False, dirty_push: bool = Fa
                     tags.append("{}@{}".format(repo, m.digest))
             tags = " ".join(tags)
             cmd = f"docker manifest create {t0} {new_manifest}"
+
             if len(tags) > 0:
                 cmd += " " + tags
-            output = execute(cmd)
-            self._logger.debug("$ %s\n%s", cmd, output)
+            print("\033[34m$ %s\033[0m" % cmd, flush=True)
+            if os.system(cmd) != 0:
+                raise Exception("Failed to create manifest")
 
             cmd = f"docker manifest push -p {t0}"
-            output = execute(cmd)
-            self._logger.debug("$ %s\n%s", cmd, output)
+            print("\033[34m$ %s\033[0m" % cmd, flush=True)
+            if os.system(cmd) != 0:
+                raise Exception("Failed to push manifest")
+
         else:
             cmd = f"docker manifest create {t0} {new_manifest}"
-            output = execute(cmd)
-            self._logger.debug("$ %s\n%s", cmd, output)
+            print("\033[34m$ %s\033[0m" % cmd, flush=True)
+            if os.system(cmd) != 0:
+                raise Exception("Failed to create manifest")
 
             cmd = f"docker manifest push -p {t0}"
-            output = execute(cmd)
-            self._logger.debug("$ %s\n%s", cmd, output)
+            print("\033[34m$ %s\033[0m" % cmd, flush=True)
+            if os.system(cmd) != 0:
+                raise Exception("Failed to push manifest")
 
     def __repr__(self):
         return "<Image name=%r tag=%r branch=%r>" % (self.name, self.tag, self.branch)
diff --git a/tools/core/toolkit.py b/tools/core/toolkit.py
@@ -3,7 +3,8 @@
 import sys
 from datetime import datetime
 from typing import Optional, List
-from subprocess import CalledProcessError
+from subprocess import CalledProcessError, check_output
+import re
 
 from .docker import DockerTemplate, Platform, Platforms
 from .git import GitTemplate
@@ -111,6 +112,22 @@ def _create_context(self, dry_run: bool, platforms: List[Platform]):
             current_platform=self.current_platform,
         )
 
+    def _get_modified_images(self) -> List[str]:
+        cmd = "git diff --name-only $(git merge-base --fork-point origin/master)..HEAD images"
+        print("\033[34m$ %s" % cmd, flush=True)
+        output = check_output(cmd, shell=True)
+        output = output.decode()
+        print("%s" % output.rstrip(), flush=True)
+        lines = output.splitlines()
+        p = re.compile(r"^images/(.+)/.*$")
+        images = set()
+        for line in lines:
+            m = p.match(line)
+            assert m, "mismatch line (%r): %s" % (p, line)
+            image = m.group(1)
+            images.add(image)
+        return list(images)
+
     def build(self,
               images: List[str] = None,
               dry_run: bool = False,
@@ -125,12 +142,15 @@ def build(self,
 
             ctx = self._create_context(dry_run, platforms)
 
-            if images:
-                for i, name in enumerate(images):
-                    if i > 0:
-                        print()
-                    for p in platforms:
-                        Image(ctx, name).build(platform=p, no_cache=no_cache)
+            if not images:
+                images = self._get_modified_images()
+
+            for i, name in enumerate(images):
+                if i > 0:
+                    print()
+                for p in platforms:
+                    Image(ctx, name).build(platform=p, no_cache=no_cache)
+
         except Exception as e:
             p = e
             while p:
@@ -156,12 +176,15 @@ def push(self,
 
             ctx = self._create_context(dry_run, platforms)
 
-            if images:
-                for i, name in enumerate(images):
-                    if i > 0:
-                        print()
-                    for p in platforms:
-                        Image(ctx, name).push(platform=p, no_cache=no_cache, dirty_push=dirty_push)
+            if not images:
+                images = self._get_modified_images()
+
+            for i, name in enumerate(images):
+                if i > 0:
+                    print()
+                for p in platforms:
+                    Image(ctx, name).push(platform=p, no_cache=no_cache, dirty_push=dirty_push)
+
         except Exception as e:
             p = e
             while p:
@@ -172,7 +195,6 @@ def push(self,
                 p = e.__cause__
             raise
 
-
     def test(self):
         os.chdir(self.project_dir)
         sys.exit(os.system("python3.8 -m pytest -s"))
diff --git a/tools/helper.py b/tools/helper.py
@@ -13,14 +13,14 @@ def main():
     build_parser = subparsers.add_parser("build", prog="build")
     build_parser.add_argument("--dry-run", action="store_true")
     build_parser.add_argument("--no-cache", action="store_true")
-    build_parser.add_argument("--platform", action="append")
+    build_parser.add_argument("--platform", "-p", action="append")
     build_parser.add_argument("images", type=str, nargs="*")
 
     push_parser = subparsers.add_parser("push")
     push_parser.add_argument("--dirty-push", action="store_true")
     push_parser.add_argument("--dry-run", action="store_true")
     push_parser.add_argument("--no-cache", action="store_true")
-    push_parser.add_argument("--platform", action="append")
+    push_parser.add_argument("--platform", "-p", action="append")
     push_parser.add_argument("images", type=str, nargs="*")
 
     subparsers.add_parser("test")
diff --git a/tools/push b/tools/push
@@ -1,19 +1,6 @@
 #!/bin/bash
 
-set -m
 set -euo pipefail
-set -x
-
-function print_running() {
-  while true; do
-    echo ">>> RUNNING <<<"
-    sleep 3
-  done
-}
-
-if [[ $(uname -m) == "aarch64" ]]; then
-  print_running &
-fi
 
 cd "$(dirname "$0")" || exit 1
-python helper.py push "$@"
+python3 helper.py push "$@"
