feature/onnx-to-tosa

.buildbot/Jenkinsfile

@@ -49,25 +49,25 @@ def call() {
       DOCKER_DAEMON_SOCKET = 'unix://var/run/docker.sock'
       DOCKER_REGISTRY_TOKEN_ACCESS = 'true'
       /* Settings for docker.io */
+      /*
       DOCKER_REGISTRY_HOST_NAME = ''
       DOCKER_REGISTRY_USER_NAME = 'onnxmlir'
       DOCKER_REGISTRY_LOGIN_NAME = 'onnxmlir'
+      */
       /* Settings for ghcr.io */
-      /*
       DOCKER_REGISTRY_HOST_NAME = 'ghcr.io'
       DOCKER_REGISTRY_USER_NAME = 'onnxmlir'
       DOCKER_REGISTRY_LOGIN_NAME = 'onnxmlir'
-      */
 
       /* Credentials defined in Jenkins */
       JENKINS_REST_API_TOKEN = credentials('Jenkins-REST-API-Token')
       GITHUB_REPO_ACCESS_TOKEN = credentials('jenkins-buildbot-access-token')
       /* Settings for docker.io */
+      /*
       DOCKER_REGISTRY_LOGIN_TOKEN = credentials('DOCKERHUB-ONNXMLIR-TOKEN')
+      */
       /* Settings for ghcr.io */
-      /*
       DOCKER_REGISTRY_LOGIN_TOKEN = credentials('GITHUB-ONNXMLIR-TOKEN')
-      */
 
       /* Environment variables that depend on the arch */
       JENKINS_REST_API_URL = sh(returnStdout: true,

.buildbot/jenkins-build-llvm-project.py

@@ -7,7 +7,7 @@
 LLVM_PROJECT_DOCKERFILE = "docker/Dockerfile.llvm-project"
 LLVM_PROJECT_GITHUB_URL = "https://api.github.com/repos/llvm/llvm-project"
 LLVM_PROJECT_BASE_IMAGE = {
-    "static": "ubuntu:jammy",
+    "static": "ghcr.io/onnxmlir/ubuntu:jammy",
     "shared": "registry.access.redhat.com/ubi8-minimal:latest",
 }
 LLVM_PROJECT_IMAGE = {
@@ -187,7 +187,7 @@ def setup_per_pr_llvm_project(image_type, exp):
             ):
                 if "stream" in line:
                     # Keep track of the latest successful image layer
-                    m = re.match("^\s*---> ([0-9a-f]+)$", line["stream"])
+                    m = re.match(r"^\s*---> ([0-9a-f]+)$", line["stream"])
                     if m:
                         layer_sha256 = m.group(1)
                     print(line["stream"], end="", flush=True)

.buildbot/jenkins-build-onnx-mlir.py

@@ -178,7 +178,7 @@ def build_per_pr_onnx_mlir(image_type, exp):
         ):
             if "stream" in line:
                 # Keep track of the latest successful image layer
-                m = re.match("^\s*---> ([0-9a-f]+)$", line["stream"])
+                m = re.match(r"^\s*---> ([0-9a-f]+)$", line["stream"])
                 if m:
                     layer_sha256 = m.group(1)
                 print(line["stream"], end="", flush=True)

.buildbot/jenkins-watch-llvm-project.py

@@ -336,7 +336,7 @@ def build_watch_image(repo, commit, dockerfile, base_image, image_repo, image_ta
     ):
         if "stream" in line:
             # Keep track of the latest successful image layer
-            m = re.match("^\s*---> ([0-9a-f]+)$", line["stream"])
+            m = re.match(r"^\s*---> ([0-9a-f]+)$", line["stream"])
             if m:
                 layer_sha256 = m.group(1)
             print(line["stream"], end="", flush=True)

.buildbot/jenkins_common.py

@@ -31,10 +31,11 @@
 MEMORY_IN_GB = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") / (1024.0**3)
 NPROC = str(math.ceil(min(max(2, MEMORY_IN_GB / 8), os.cpu_count())))
 
-RETRY_LIMIT = 5
+RETRY_LIMIT = 10
 READ_CHUNK_SIZE = 1024 * 1024
 BASE_BRANCH = "main"
 
+DOCKER_API_TIMEOUT = 3600
 DOCKER_DIST_MANIFEST = "application/vnd.docker.distribution.manifest.v2+json"
 DOCKER_DIST_MANIFEST_LIST = "application/vnd.docker.distribution.manifest.list.v2+json"
 
@@ -47,7 +48,7 @@
 docker_registry_login_token = os.getenv("DOCKER_REGISTRY_LOGIN_TOKEN")
 docker_registry_token_access = os.getenv("DOCKER_REGISTRY_TOKEN_ACCESS")
 docker_rwlock = fasteners.InterProcessReaderWriterLock(docker_pushpull_rwlock)
-docker_api = docker.APIClient(base_url=docker_daemon_socket)
+docker_api = docker.APIClient(base_url=docker_daemon_socket, timeout=DOCKER_API_TIMEOUT)
 
 github_repo_access_token = os.getenv("GITHUB_REPO_ACCESS_TOKEN")
 github_repo_name = os.getenv("GITHUB_REPO_NAME")
@@ -119,12 +120,12 @@ def strtobool(s: str) -> bool:
 def compute_file_sha1(file_name):
     """Compute sha1 of a file."""
 
-    sha1sum = hashlib.sha1()
+    sha3_256sum = hashlib.sha3_256()
     try:
         with open(file_name, "rb") as f:
             for data in iter(lambda: f.read(READ_CHUNK_SIZE), b""):
-                sha1sum.update(data)
-        return sha1sum.hexdigest()
+                sha3_256sum.update(data)
+        return sha3_256sum.hexdigest()
     except:
         return ""
 

.github/workflows/black-format-check.yml

@@ -6,7 +6,8 @@ jobs:
   check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+
       - uses: psf/black@stable
         with:
           options: "--check --exclude third_party"

.github/workflows/macos-amd64-build.yml

@@ -1,6 +1,9 @@
 name: GitHub Action MacOS amd64
 
-on: [push, pull_request]
+on:
+  pull_request:
+  push:
+    branches: [ main, feature/onnx-to-tosa ]
 
 jobs:
   build:
@@ -18,6 +21,12 @@ jobs:
     - name: install tools that are needed for compilation
       run: |
         brew install automake ninja pybind11
+    - name: ccache
+      uses: hendrikmuhs/[email protected]
+      with:
+        # Create symlinks to intercept compiler calls without modifying the cmake invocation
+        create-symlink: true
+        key: ${{ runner.os }}-ccache
     - name: install protobuf
       run: |
         cd ~/work
@@ -46,6 +55,8 @@ jobs:
         python3 -m pip install -v .
     - name: build onnx-mlir
       run: |
+        # Disable stablehlo to ease bumping; we don't need it.
+        export EXTRA_CMAKE_ARGS="-DONNX_MLIR_ENABLE_STABLEHLO=OFF -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache"
         cd ~/work/onnx-mlir
         sh ~/work/onnx-mlir/onnx-mlir/utils/install-onnx-mlir.sh
     - name: build and run docs/doc_example tests
@@ -57,6 +68,9 @@ jobs:
         cd ~/work/onnx-mlir
         sh ~/work/onnx-mlir/onnx-mlir/utils/check-unittest.sh
     - name: run onnx-mlir backend and numerical tests
+      # Those tests are not relevant to the work on the xilinx fork, but take
+      # 40 min. Don't run them on PRs.
+      if: github.event_name != 'pull_request'
       run: |
         cd ~/work/onnx-mlir
         sh ~/work/onnx-mlir/onnx-mlir/utils/check-onnx-backend-numerical.sh

.github/workflows/ubuntu-build-intree.yml

@@ -0,0 +1,86 @@
+name: In-tree build
+
+on:
+  pull_request:
+  push:
+    branches: [ main, feature/onnx-to-tosa ]
+
+concurrency:
+  # Build every push.
+  # Only build the newest PR; cancel older builds of a PR
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  build-intree:
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: recursive
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+
+    - name: install tools that are needed for compilation
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y gcc g++ cmake ninja-build
+    - name: Setup ccache
+      uses: hendrikmuhs/ccache-action@v1
+      with:
+        # A full build seems to take ~ 250 MB. Leave a bit more room
+        # so we don't run out of cache space in the future.
+        max-size: 400M
+        key: ccache-intree
+        create-symlink: true
+
+    - name: install dependencies
+      run: |
+        utils/install-protobuf.sh
+        utils/install-venv.sh
+
+    - name: clone llvm-project
+      run: sh utils/clone-mlir.sh
+
+    - name: build
+      run: |
+        cmake llvm-project/llvm \
+          -Bbuild \
+          -G Ninja \
+          -DCMAKE_BUILD_TYPE=Release \
+          -DLLVM_ENABLE_PROJECTS=mlir \
+          -DLLVM_EXTERNAL_PROJECTS="onnx-mlir" \
+          -DONNX_MLIR_ENABLE_STABLEHLO=OFF \
+          -DONNX_MLIR_ENABLE_PYBIND=OFF \
+          -DLLVM_EXTERNAL_ONNX_MLIR_SOURCE_DIR=. \
+          -DLLVM_TARGETS_TO_BUILD=host \
+          -DLLVM_BUILD_TOOLS=OFF \
+          -DLLVM_BUILD_UTILS=OFF \
+          -DLLVM_BUILD_RUNTIMES=OFF \
+          -DLLVM_ENABLE_ASSERTIONS=ON \
+          -DLLVM_ENABLE_RTTI=ON \
+          -DLLVM_ENABLE_LIBEDIT=OFF \
+          -DLLVM_USE_LINKER=lld \
+          -DCMAKE_C_COMPILER=clang \
+          -DCMAKE_CXX_COMPILER=clang++ \
+          -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+
+        cmake --build build --target onnx-mlir
+
+    - name: run LIT tests
+      run: |
+        export LIT_OPTS=-v
+        cmake --build build --target check-onnx-lit
+
+
+    - name: build and run docs/doc_example tests
+      run: |
+        cd ..
+        sh onnx-mlir/utils/check-doc-example.sh
+
+    - name: build and run unit tests
+      run: |
+        cd ..
+        sh onnx-mlir/utils/check-unittest.sh

.github/workflows/ubuntu-build.yml

@@ -0,0 +1,68 @@
+name: Out-of-tree build
+
+on:
+  push:
+    branches: [ main ]
+
+concurrency:
+  # Build every push to main
+  # Only build the newest PR; cancel older builds of a PR
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: recursive
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+
+    - name: install tools that are needed for compilation
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y gcc g++ cmake ninja-build
+    - name: Setup ccache
+      uses: hendrikmuhs/ccache-action@v1
+      with:
+        # A full build of llvm, clang, lld, and lldb takes about 250MB
+        # of ccache space. There's not much reason to have more than this,
+        # because we usually won't need to save cache entries from older
+        # builds.  Also, there is an overall 10GB cache limit, and each
+        # run creates a new cache entry so we want to ensure that we have
+        # enough cache space for all the tests to run at once and still
+        # fit under the 10 GB limit.
+        max-size: 500M
+        key: ccache
+        create-symlink: true
+
+    - name: install dependencies
+      run: |
+          utils/install-protobuf.sh
+          utils/install-venv.sh
+
+    - name: clone & build MLIR
+      run: |
+        cd ..
+        export EXTRA_CMAKE_ARGS="-DLLVM_USE_LINKER=lld -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache"
+        sh onnx-mlir/utils/clone-mlir.sh
+        sh onnx-mlir/utils/build-mlir.sh
+
+    - name: build onnx-mlir
+      run: |
+        cd ..
+        export EXTRA_CMAKE_ARGS="-DONNX_MLIR_ENABLE_STABLEHLO=OFF -DLLVM_USE_LINKER=lld -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache"
+        bash onnx-mlir/utils/install-onnx-mlir.sh
+
+    - name: build and run docs/doc_example tests
+      run: |
+        cd ..
+        sh onnx-mlir/utils/check-doc-example.sh
+
+    - name: build and run unit tests
+      run: |
+        cd ..
+        sh onnx-mlir/utils/check-unittest.sh