Merge pull request #1 from Kitware/jrp/ci-cd-fixes

CI/CD Updates and Fixed A* Fork

Author: bluemellophone

.github/workflows/codeql-analysis.yaml

@@ -1,10 +1,12 @@
 name: "CodeQL"
 
 on:
-  push:
-    branches: [ main ]
   pull_request:
-    branches: [ main ]
+    branches:
+      - main
+  push:
+    branches:
+      - main
   schedule:
     - cron: '0 0 * * 1'
 
@@ -19,16 +21,16 @@ jobs:
         language: [ 'python' ]
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v6
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v4
       with:
         languages: ${{ matrix.language }}
 
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v4
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v4

.github/workflows/docker-publish.yaml

@@ -25,18 +25,15 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: nschloe/action-cached-lfs-checkout@v1.1.3
+        uses: nschloe/action-cached-lfs-checkout@v1.2.3
         with:
-          exclude: "batbot/*/models/pytorch/"
+          exclude: "examples/example[2-4].wav"
 
-      - uses: docker/setup-qemu-action@v1
+      - uses: docker/setup-qemu-action@v3
         name: Set up QEMU
         id: qemu
-        with:
-          image: tonistiigi/binfmt:latest
-          platforms: all
 
-      - uses: docker/setup-buildx-action@v1
+      - uses: docker/setup-buildx-action@v3
         name: Set up Docker Buildx
         id: buildx
 
@@ -45,35 +42,43 @@ jobs:
 
       # Log into container registries
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
-          username: batbot
-          password: ${{ secrets.BATBOT_DOCKER_HUB_TOKEN }}
+          username: ${{ vars.BATBOT_DOCKERHUB_USERNAME }}
+          password: ${{ secrets.BATBOT_DOCKERHUB_TOKEN }}
 
-      # Push tagged image (version tag + latest) to registries
-      - name: Tagged Docker Hub
-        if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') }}
-        run: |
-          VERSION=$(echo ${GITHUB_REF} | sed 's#.*/v##')
-          echo "IMAGE_TAG=${VERSION}" >> $GITHUB_ENV
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
-      # Push bleeding-edge image (main tag) to registries
-      - name: Bleeding Edge Docker Hub
-        if: github.ref == 'refs/heads/main'
+      # Push bleeding-edge image ("<branch name>" tag) to registries
+      - name: Bleeding Edge Docker Hub (Default Option)
         run: |
-          echo "IMAGE_TAG=main" >> $GITHUB_ENV
+          TAG=$(echo ${GITHUB_REF_NAME} | sed 's/\//-/')
+          echo "IMAGE_TAG=${TAG}" >> $GITHUB_ENV
 
-      # Push nightly image (nightly tag) to registries
+      # Push nightly image ("nightly" tag) to registries
       - name: Nightly Docker Hub
         if: github.event_name == 'schedule'
         run: |
           echo "IMAGE_TAG=nightly" >> $GITHUB_ENV
 
+      # Push tagged image ("<symantic version>" tag) to registries
+      - name: Tagged Docker Hub
+        if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') }}
+        run: |
+          VERSION=$(echo ${GITHUB_REF} | sed 's#.*/v##')
+          echo "IMAGE_TAG=${VERSION}" >> $GITHUB_ENV
+
       # Build images
       - name: Build Batbot
         run: |
           docker buildx build \
               -t kitware/batbot:${{ env.IMAGE_TAG }} \
+              -t ghcr.io/kitware/batbot:${{ env.IMAGE_TAG }} \
               --platform linux/amd64 \
               --push \
               .
@@ -84,6 +89,7 @@ jobs:
         run: |
           docker buildx build \
               -t kitware/batbot:latest \
+              -t ghcr.io/kitware/batbot:latest \
               --platform linux/amd64 \
               --push \
               .

.github/workflows/python-publish.yaml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - main
+      - jrp/ci-cd-fixes
 
 jobs:
 
@@ -14,16 +15,16 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: [3.8]
+        python-version: [3.12]
 
     steps:
       - name: Checkout code
-        uses: nschloe/action-cached-lfs-checkout@v1.1.3
+        uses: nschloe/action-cached-lfs-checkout@v1.2.3
         with:
-          exclude: "batbot/*/models/pytorch/"
+          exclude: "examples/example[2-4].wav"
 
-      - uses: actions/setup-python@v2
-        name: Install Python
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -33,81 +34,93 @@ jobs:
           pip install build
           python -m build --wheel --outdir dist/ .
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v6
         with:
+          name: artifact-wheel-${{ matrix.os }}-${{ matrix.python-version }}
           path: ./dist/*.whl
 
   build_sdist:
     name: Build source distribution
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: nschloe/action-cached-lfs-checkout@v1.1.3
+        uses: nschloe/action-cached-lfs-checkout@v1.2.3
         with:
-          exclude: "batbot/*/models/pytorch/"
+          exclude: "examples/example[2-4].wav"
 
-      - uses: actions/setup-python@v2
-        name: Install Python
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v6
         with:
-          python-version: '3.8'
+          python-version: '3.12'
 
       - name: Build sdist
         run: |
           pip install --upgrade pip
           pip install build
           python -m build --sdist --outdir dist/ .
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v6
         with:
+          name: artifact-sdist
           path: ./dist/*.tar.gz
 
   test_wheel:
     needs: [build_wheels, build_sdist]
     runs-on: ubuntu-latest
-    env:
-      CLASSIFIER_BATCH_SIZE: 16
 
     # test wheel
     if: github.event_name == 'push'
     steps:
-      - uses: actions/setup-python@v2
-        name: Install Python
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v6
         with:
-          python-version: '3.8'
+          python-version: '3.12'
 
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v7
         with:
-          name: artifact
-          path: dist
+          path: artifact
 
       - name: Install wheel
         run: |
           pip install --upgrade pip
           pip install wheel
+          find .
+          mkdir dist
+          cp artifact/*-ubuntu-*/*.whl dist/
+          cp artifact/*/*.tar.gz dist/
           pip install dist/*.whl
 
       - name: Test module
         run: |
-          python -c "import batbot; batbot.fetch(); batbot.example();"
+          python -c "import batbot;"
 
-      - name: Test CLI
-        run: |
-          batbot fetch
-          batbot example
+      # - name: Test CLI
+      #   run: |
+      #     batbot example
 
   upload_pypi:
     needs: [test_wheel]
     runs-on: ubuntu-latest
     # upload to PyPI on every tag starting with 'v'
     if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
+    environment:
+      name: pypi
+      url: https://pypi.org/p/batbot
+    permissions:
+      id-token: write
     steps:
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v7
         with:
-          name: artifact
-          path: dist
+          path: artifact
+
+      - name: Install wheel
+        run: |
+          find .
+          mkdir dist
+          cp artifact/*-ubuntu-*/*.whl dist/
+          cp artifact/*/*.tar.gz dist/
 
-      - uses: pypa/gh-action-pypi-publish@release/v1
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
         with:
-          user: __token__
-          password: ${{ secrets.PYPI_PASSWORD }}
-          # To test: repository_url: https://test.pypi.org/legacy/
+          password: ${{ secrets.BATBOT_PYPI_TOKEN }}

.github/workflows/testing.yaml

@@ -12,51 +12,46 @@ jobs:
       fail-fast: false
       matrix:
         # Use the same Python version used the Dockerfile
-        python-version: [3.9]
+        python-version: [3.12]
 
     env:
       OS: ubuntu-latest
       PYTHON: ${{ matrix.python-version }}
-      CLASSIFIER_BATCH_SIZE: 16
 
     steps:
       # Checkout and env setup
       - name: Checkout code
-        uses: nschloe/action-cached-lfs-checkout@v1.1.3
+        uses: nschloe/action-cached-lfs-checkout@v1.2.3
         with:
-          exclude: "batbot/*/models/pytorch/"
+          exclude: "examples/example[3-4].wav"
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
-          pip install -r requirements.optional.txt
+          pip install -r requirements/runtime.txt
+          pip install -r requirements/optional.txt
+          pip install -e .
 
-      - name: Lint with flake8
+      - name: Check with pre-commit
         run: |
-          # stop the build if there are Python syntax errors or undefined names
-          flake8 . --count --show-source --statistics
-          # exit-zero treats all errors as warnings.
-          flake8 . --count --exit-zero --max-complexity=10 --statistics
+          SKIP=hadolint pre-commit
+          SKIP=hadolint pre-commit run --all-files
 
-      - name: Run tests
+      - name: Run tests and coverage
         run: |
           set -ex
-          pytest --cov=./ --cov-append --random-order-seed=1
-
-      - name: Run coverage
-        run: |
-          coverage xml
+          pytest --cov=batbot --cov-append --cov-report=xml --random-order-seed=1
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v1.2.1
+        continue-on-error: true
+        uses: codecov/codecov-action@v5
         with:
-          token: ${{ secrets.CODECOV_TOKEN }}
+          token: ${{ secrets.BATBOT_CODECOV_TOKEN }}
           files: ./coverage/coverage.xml
           env_vars: OS,PYTHON
           fail_ci_if_error: true

.gitignore

@@ -2,6 +2,8 @@
 output.*.jpg
 *.log*
 
+build/
+dist/
 *.egg-info/
 
 .coverage*
@@ -16,3 +18,4 @@ docs/_build/
 
 example*.jpg
 example*.json
+.vscode/*

Dockerfile

@@ -31,14 +31,11 @@ RUN python3 -m venv /venv
 # hadolint ignore=DL3003,DL3013
 RUN /venv/bin/pip install --no-cache-dir -r requirements/runtime.txt \
  && /venv/bin/pip install --no-cache-dir -r requirements/optional.txt \
- && cd tpl/pyastar2d/ \
- && /venv/bin/pip install --no-cache-dir -e . \
- && cd ../.. \
- && /venv/bin/pip install --no-cache-dir -e . \
- && if [ "$(uname -m)" != "aarch64" ] \
-       ; then \
-       /venv/bin/pip uninstall -y onnxruntime \
-       /venv/bin/pip install --no-cache-dir onnxruntime-gpu \
-       ; fi
+ && /venv/bin/pip install --no-cache-dir -e .
+ # && if [ "$(uname -m)" != "aarch64" ] \
+ #       ; then \
+ #       /venv/bin/pip uninstall -y onnxruntime \
+ #       /venv/bin/pip install --no-cache-dir onnxruntime-gpu \
+ #       ; fi
 
 CMD [".", "/venv/bin/activate", "&&", "exec", "python", "app.py"]

ISSUES.rst

@@ -2,4 +2,9 @@
 Known Issues
 ============
 
-N/A
+TODO:
+- Fix CI/CD docker build
+- Add API documentation, tutorials, and examples
+- Create example notebooks with Google colab
+- Crate discord / discourse community board
+- [BatAI] Upload training scripts with MLFlow support, database export with celery

MANIFEST.in

@@ -1,5 +1,3 @@
-include pyproject.toml
-
 # Include the README and SECURITY documents
 include *.rst