diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml
new file mode 100644
index 000000000..4abaabdf3
--- /dev/null
+++ b/.github/workflows/build-and-release.yaml
@@ -0,0 +1,101 @@
+name: Build and Release
+
+on:
+  release:
+    types: [ created ]
+
+jobs:
+  # Use the reusable workflow for building binaries
+  build:
+    uses: ./.github/workflows/build-template.yaml
+    with:
+      python-version: "3.11"
+      python-versions-matrix: '["3.11"]'
+      # We build on an older Ubuntu as pyinstaller binaries are forward-compatible not backwards-compatible
+      # See https://pyinstaller.org/en/stable/usage.html?highlight=glibc#making-gnu-linux-apps-forward-compatible:~:text=The%20solution%20is%20to%20always%20build%20your%20app%20on%20the%20oldest%20version%20of%20GNU/Linux%20you%20mean%20to%20support.%20It%20should%20continue%20to%20work%20with%20the%20libc%20found%20on%20newer%20versions.
+      # TODO: for similar reasons, we may want to build on older Windows/MacOS versions as well
+      os-matrix: '["ubuntu-22.04", "windows-latest", "macos-latest"]'
+      is-release: true
+    secrets: inherit
+
+  # Latest release check job
+  check-latest:
+    name: Check if Latest Release
+    needs: build
+    runs-on: ubuntu-22.04
+    outputs:
+      IS_LATEST: ${{ steps.check-latest.outputs.release == github.ref_name }}
+    steps:
+    - id: check-latest
+      uses: pozetroninc/github-action-get-latest-release@v0.7.0
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        repository: ${{ github.repository }}
+        excludes: prerelease, draft
+
+  # macOS hash calculation job
+  mac-hash:
+    name: Calculate macOS Hash
+    needs: check-latest
+    runs-on: ubuntu-latest
+    if: needs.check-latest.outputs.IS_LATEST
+    outputs:
+      MAC_BUILD_HASH: ${{ steps.calc-hash.outputs.MAC_BUILD_HASH }}
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v4
+    - name: Download MacOS artifact
+      uses: actions/download-artifact@v4
+      with:
+        name: holmes-macos-latest-${{ github.ref_name }}
+    - name: Calculate hash
+      id: calc-hash
+      run: echo "::set-output name=MAC_BUILD_HASH::$(sha256sum holmes-macos-latest-${{ github.ref_name }}.zip | awk '{print $1}')"
+
+  # Linux hash calculation job
+  linux-hash:
+    name: Calculate Linux Hash
+    needs: check-latest
+    runs-on: ubuntu-latest
+    if: needs.check-latest.outputs.IS_LATEST
+    outputs:
+      LINUX_BUILD_HASH: ${{ steps.calc-hash.outputs.LINUX_BUILD_HASH }}
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v4
+    - name: Download Linux artifact
+      uses: actions/download-artifact@v4
+      with:
+        name: holmes-ubuntu-22.04-${{ github.ref_name }}
+    - name: Calculate hash
+      id: calc-hash
+      run: echo "::set-output name=LINUX_BUILD_HASH::$(sha256sum holmes-ubuntu-22.04-${{ github.ref_name }}.zip | awk '{print $1}')"
+
+  # Homebrew formula update job
+  update-formula:
+    name: Update Homebrew Formula
+    needs: [ mac-hash, linux-hash ]
+    runs-on: ubuntu-latest
+    if: ${{ !github.event.release.prerelease }}
+    env:
+      TAG_NAME: ${{ github.ref_name }}
+    steps:
+    - name: Checkout homebrew-holmesgpt repository
+      uses: actions/checkout@v4
+      with:
+        repository: robusta-dev/homebrew-holmesgpt
+        token: ${{ secrets.MULTIREPO_GITHUB_TOKEN }}
+    - name: Update holmesgpt.rb formula
+      run: |
+        MAC_BUILD_HASH=${{ needs.mac-hash.outputs.MAC_BUILD_HASH }}
+        LINUX_BUILD_HASH=${{ needs.linux-hash.outputs.LINUX_BUILD_HASH }}
+        awk 'NR==6{$0="        url \"https://github.com/robusta-dev/holmesgpt/releases/download/'"$TAG_NAME"'/holmes-macos-latest-'"$TAG_NAME"'.zip\""}1' ./Formula/holmesgpt.rb > temp && mv temp ./Formula/holmesgpt.rb
+        awk 'NR==7{$0="        sha256 \"'$MAC_BUILD_HASH'\""}1' ./Formula/holmesgpt.rb > temp && mv temp ./Formula/holmesgpt.rb
+        awk 'NR==9{$0="        url \"https://github.com/robusta-dev/holmesgpt/releases/download/'"$TAG_NAME"'/holmes-ubuntu-22.04-'"$TAG_NAME"'.zip\""}1' ./Formula/holmesgpt.rb > temp && mv temp ./Formula/holmesgpt.rb
+        awk 'NR==10{$0="        sha256 \"'$LINUX_BUILD_HASH'\""}1' ./Formula/holmesgpt.rb > temp && mv temp ./Formula/holmesgpt.rb
+    - name: Commit and push changes
+      run: |
+        git config --local user.email "action@github.com"
+        git config --local user.name "GitHub Action"
+        git commit -am "Update formula for release ${TAG_NAME}"
+        git push
diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml
index 6ffdfc15c..1854a5dc4 100644
--- a/.github/workflows/build-and-test.yaml
+++ b/.github/workflows/build-and-test.yaml
@@ -2,81 +2,39 @@
 # we test this on each commit/PR to catch build problems early
 name: Build and test HolmesGPT
 
-on: [push, pull_request, workflow_dispatch]
+on: [ push, pull_request, workflow_dispatch ]
 
 jobs:
+  # Pre-commit checks job
   check:
     name: Pre-commit checks
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-      - name: Install Poetry
-        uses: snok/install-poetry@v1
-        with:
-          virtualenvs-create: true
-          virtualenvs-in-project: true
-          virtualenvs-path: .venv
-          installer-parallel: true
-      - name: Install dependencies
-        run: poetry install --no-interaction --no-root
-      - uses: pre-commit/action@v3.0.1
-  build:
-    needs: check
-    strategy:
-      matrix:
-        python-version: ["3.10", "3.11", "3.12"]
-
-    runs-on: ubuntu-latest
+    - uses: actions/checkout@v4
 
-    steps:
-      - uses: actions/checkout@v2
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
 
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        virtualenvs-create: true
+        virtualenvs-in-project: true
+        virtualenvs-path: .venv
+        installer-parallel: true
 
-      - name: Install Python dependencies and build
-        # if you change something here, you must also change it in .github/workflows/build-binaries-and-brew.yaml
-        run: |
-          python -m pip install --upgrade pip setuptools pyinstaller
+    - name: Install dependencies
+      run: poetry install --no-interaction --no-root
 
-          curl -sSL https://install.python-poetry.org | python3 - --version 1.4.0
-          poetry config virtualenvs.create false
-          poetry install --no-root
+    - uses: pre-commit/action@v3.0.1
 
-          sudo apt-get install -y binutils
-          pyinstaller holmes_cli.py \
-            --name holmes \
-            --add-data 'holmes/plugins/runbooks/*:holmes/plugins/runbooks' \
-            --add-data 'holmes/plugins/prompts/*:holmes/plugins/prompts' \
-            --add-data 'holmes/plugins/toolsets/*:holmes/plugins/toolsets' \
-            --add-data 'holmes/plugins/toolsets/coralogix*:holmes/plugins/toolsets/coralogix' \
-            --add-data 'holmes/plugins/toolsets/grafana*:holmes/plugins/toolsets/grafana' \
-            --add-data 'holmes/plugins/toolsets/internet*:holmes/plugins/toolsets/internet' \
-            --add-data 'holmes/plugins/toolsets/opensearch*:holmes/plugins/toolsets/opensearch' \
-            --add-data 'holmes/plugins/toolsets/prometheus*:holmes/plugins/toolsets/prometheus' \
-            --hidden-import=tiktoken_ext.openai_public \
-            --hidden-import=tiktoken_ext \
-            --hiddenimport litellm.llms.tokenizers \
-            --hiddenimport litellm.litellm_core_utils.tokenizers \
-            --collect-data litellm
-          ls dist
-
-      - name: Run tests
-        shell: bash
-        env:
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-        run: |
-          poetry run pytest -m "not llm"
-
-      - name: Test the binary
-        shell: bash
-        run: |
-          dist/holmes/holmes version
-          if [ $? -ne 0 ]; then
-            echo "Binary test failed"
-            exit 1
-          fi
-          echo "Binary test passed"
+  build-and-test:
+    needs: check
+    if: always() && (needs.check.result == 'success' || needs.check.result == 'skipped')
+    uses: ./.github/workflows/build-template.yaml
+    with:
+      python-versions-matrix: '["3.10", "3.11", "3.12"]'
+      os-matrix: '["ubuntu-latest", "windows-latest", "macos-latest"]'
+      is-release: false
+    secrets: inherit
diff --git a/.github/workflows/build-binaries-and-brew.yaml b/.github/workflows/build-binaries-and-brew.yaml
deleted file mode 100644
index 5205d044d..000000000
--- a/.github/workflows/build-binaries-and-brew.yaml
+++ /dev/null
@@ -1,197 +0,0 @@
-name: Build and Release
-
-on:
-  release:
-    types: [created]
-
-jobs:
-  build:
-    strategy:
-      matrix:
-        # We build on an older Ubuntu as pyinstaller binaries are forward-compatible not backwards-compatible
-        # See https://pyinstaller.org/en/stable/usage.html?highlight=glibc#making-gnu-linux-apps-forward-compatible:~:text=The%20solution%20is%20to%20always%20build%20your%20app%20on%20the%20oldest%20version%20of%20GNU/Linux%20you%20mean%20to%20support.%20It%20should%20continue%20to%20work%20with%20the%20libc%20found%20on%20newer%20versions.
-        # TODO: for similar reasons, we may want to build on older Windows/MacOS versions as well
-        os: [ubuntu-22.04, windows-latest, macos-latest]
-
-    runs-on: ${{ matrix.os }}
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.11'
-
-    - name: Install dependencies
-      if: matrix.os != 'windows-latest'
-      run: |
-        python -m pip install --upgrade pip setuptools pyinstaller
-        curl -sSL https://install.python-poetry.org | python3 - --version 1.4.0
-        poetry config virtualenvs.create false
-        poetry install --no-root
-
-    - name: Install dependencies
-      if: matrix.os == 'windows-latest'
-      run: |
-        python -m pip install --upgrade pip setuptools pyinstaller
-        curl -sSL https://install.python-poetry.org | python3 - --version 1.4.0
-        C:\Users\runneradmin\AppData\Roaming\Python\Scripts\poetry config virtualenvs.create false
-        C:\Users\runneradmin\AppData\Roaming\Python\Scripts\poetry install --no-root
-
-    - name: Install dependencies (Linux)
-      if: matrix.os == 'ubuntu-22.04'
-      run: |
-        sudo apt-get install -y binutils
-
-    - name: Update package version (Linux)
-      if: matrix.os == 'ubuntu-22.04'
-      run: sed -i 's/__version__ = .*/__version__ = "${{ github.ref_name }}"/g' holmes/__init__.py
-
-    # mac has BSD style sed command where you specify -i '' and not just -i
-    - name: Update package version (macOS)
-      if: matrix.os == 'macos-latest'
-      run: sed -i '' 's/__version__ = .*/__version__ = "${{ github.ref_name }}"/g' holmes/__init__.py
-
-    # windows has no sed, so we use powershell
-    - name: Update package version (Windows)
-      if: matrix.os == 'windows-latest'
-      run: |
-        $filePath = 'holmes/__init__.py'
-        (Get-Content $filePath) -replace '__version__ = .+', '__version__ = "${{ github.ref_name }}"' | Set-Content $filePath
-      shell: pwsh
-
-    - name: Install unixODBC (macOS)
-      if: matrix.os == 'macos-latest'
-      run: brew install unixodbc
-
-    # if you change something here, you must also change it in .github/workflows/build-and-test.yaml
-    - name: Build with PyInstaller
-      shell: bash
-      # regarding the tiktoken part of the command, see https://github.com/openai/tiktoken/issues/80
-      # regarding the litellm part of the command, see https://github.com/pyinstaller/pyinstaller/issues/8620#issuecomment-2186540504
-      run: |
-       pyinstaller holmes_cli.py \
-            --name holmes \
-            --add-data 'holmes/plugins/runbooks/*:holmes/plugins/runbooks' \
-            --add-data 'holmes/plugins/prompts/*:holmes/plugins/prompts' \
-            --add-data 'holmes/plugins/toolsets/*:holmes/plugins/toolsets' \
-            --add-data 'holmes/plugins/toolsets/coralogix*:holmes/plugins/toolsets/coralogix' \
-            --add-data 'holmes/plugins/toolsets/grafana*:holmes/plugins/toolsets/grafana' \
-            --add-data 'holmes/plugins/toolsets/internet*:holmes/plugins/toolsets/internet' \
-            --add-data 'holmes/plugins/toolsets/opensearch*:holmes/plugins/toolsets/opensearch' \
-            --add-data 'holmes/plugins/toolsets/prometheus*:holmes/plugins/toolsets/prometheus' \
-            --hidden-import=tiktoken_ext.openai_public \
-            --hidden-import=tiktoken_ext \
-            --hiddenimport litellm.llms.tokenizers \
-            --hiddenimport litellm.litellm_core_utils.tokenizers \
-            --collect-data litellm
-        ls dist
-
-    - name: Zip the application (Unix and Mac)
-      if: matrix.os == 'macos-latest' || matrix.os == 'ubuntu-22.04'
-      run: |
-        cd dist
-        zip -r holmes-${{ matrix.os }}-${{ github.ref_name }}.zip holmes
-        mv holmes-${{ matrix.os }}-${{ github.ref_name }}.zip ../
-        cd ..
-
-    - name: Zip the application (Windows)
-      if: matrix.os == 'windows-latest'
-      run: |
-        Set-Location -Path dist
-        Compress-Archive -Path holmes -DestinationPath holmes-${{ matrix.os }}-${{ github.ref_name }}.zip -Force
-        Move-Item -Path holmes-${{ matrix.os }}-${{ github.ref_name }}.zip -Destination ..\
-        Set-Location -Path ..
-
-    - name: Upload Release Asset
-      uses: actions/upload-release-asset@v1.0.2
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ github.event.release.upload_url }}
-        asset_path: ./holmes-${{ matrix.os }}-${{ github.ref_name }}.zip
-        asset_name: holmes-${{ matrix.os }}-${{ github.ref_name }}.zip
-        asset_content_type: application/octet-stream
-
-    - name: Upload build as artifact
-      uses: actions/upload-artifact@v4
-      with:
-        name: holmes-${{ matrix.os }}-${{ github.ref_name }}
-        path: ./holmes-${{ matrix.os }}-${{ github.ref_name }}.zip
-
-  check-latest:
-    needs: build
-    runs-on: ubuntu-22.04
-    outputs:
-      IS_LATEST: ${{ steps.check-latest.outputs.release == github.ref_name }}
-    steps:
-      - id: check-latest
-        uses: pozetroninc/github-action-get-latest-release@v0.7.0
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          repository: ${{ github.repository }}
-          excludes: prerelease, draft
-
-  # Define MacOS hash job
-  mac-hash:
-    needs: check-latest
-    runs-on: ubuntu-latest
-    if: needs.check-latest.outputs.IS_LATEST
-    outputs:
-      MAC_BUILD_HASH: ${{ steps.calc-hash.outputs.MAC_BUILD_HASH }}
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v2
-      - name: Download MacOS artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: holmes-macos-latest-${{ github.ref_name }}
-      - name: Calculate hash
-        id: calc-hash
-        run: echo "::set-output name=MAC_BUILD_HASH::$(sha256sum holmes-macos-latest-${{ github.ref_name }}.zip | awk '{print $1}')"
-
-  # Define Linux hash job
-  linux-hash:
-    needs: check-latest
-    runs-on: ubuntu-latest
-    if: needs.check-latest.outputs.IS_LATEST
-    outputs:
-      LINUX_BUILD_HASH: ${{ steps.calc-hash.outputs.LINUX_BUILD_HASH }}
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v2
-      - name: Download Linux artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: holmes-ubuntu-22.04-${{ github.ref_name }}
-      - name: Calculate hash
-        id: calc-hash
-        run: echo "::set-output name=LINUX_BUILD_HASH::$(sha256sum holmes-ubuntu-22.04-${{ github.ref_name }}.zip | awk '{print $1}')"
-
-  # TODO: update homebrew formula
-  update-formula:
-    needs: [mac-hash, linux-hash]
-    runs-on: ubuntu-latest
-    if: ${{ !github.event.release.prerelease }}
-    steps:
-      - name: Checkout homebrew-holmesgpt repository
-        uses: actions/checkout@v2
-        with:
-          repository: robusta-dev/homebrew-holmesgpt
-          token: ${{ secrets.MULTIREPO_GITHUB_TOKEN }}
-      - name: Update holmesgpt.rb formula
-        run: |
-          MAC_BUILD_HASH=${{ needs.mac-hash.outputs.MAC_BUILD_HASH }}
-          LINUX_BUILD_HASH=${{ needs.linux-hash.outputs.LINUX_BUILD_HASH }}
-          TAG_NAME=${{ github.ref_name }}
-          awk 'NR==6{$0="        url \"https://github.com/robusta-dev/holmesgpt/releases/download/'"$TAG_NAME"'/holmes-macos-latest-'"$TAG_NAME"'.zip\""}1' ./Formula/holmesgpt.rb > temp && mv temp ./Formula/holmesgpt.rb
-          awk 'NR==7{$0="        sha256 \"'$MAC_BUILD_HASH'\""}1' ./Formula/holmesgpt.rb > temp && mv temp ./Formula/holmesgpt.rb
-          awk 'NR==9{$0="        url \"https://github.com/robusta-dev/holmesgpt/releases/download/'"$TAG_NAME"'/holmes-ubuntu-22.04-'"$TAG_NAME"'.zip\""}1' ./Formula/holmesgpt.rb > temp && mv temp ./Formula/holmesgpt.rb
-          awk 'NR==10{$0="        sha256 \"'$LINUX_BUILD_HASH'\""}1' ./Formula/holmesgpt.rb > temp && mv temp ./Formula/holmesgpt.rb
-      - name: Commit and push changes
-        run: |
-          git config --local user.email "action@github.com"
-          git config --local user.name "GitHub Action"
-          git commit -am "Update formula for release ${TAG_NAME}"
-          git push
diff --git a/.github/workflows/build-template.yaml b/.github/workflows/build-template.yaml
new file mode 100644
index 000000000..9a2bd0a26
--- /dev/null
+++ b/.github/workflows/build-template.yaml
@@ -0,0 +1,214 @@
+# Reusable Multi-OS Binary Build Workflow
+# This template provides a reusable workflow for building binaries across multiple operating systems
+# with proper dependency installation and testing
+
+name: Multi-OS Binary Build Template
+
+on:
+  workflow_call:
+    inputs:
+      python-version:
+        description: 'Python version to use'
+        required: false
+        default: '3.11'
+        type: string
+      python-versions-matrix:
+        description: 'Python versions matrix for testing (JSON array)'
+        required: false
+        default: '["3.11"]'
+        type: string
+      poetry-version:
+        description: 'Poetry version to use'
+        required: false
+        default: '1.8.4'
+        type: string
+      os-matrix:
+        description: 'OS matrix for builds (JSON array)'
+        required: false
+        default: '["ubuntu-22.04", "windows-latest", "macos-latest"]'
+        type: string
+      is-release:
+        description: 'Is this a release build'
+        required: false
+        default: false
+        type: boolean
+
+env:
+  PYTHON_VERSION: ${{ inputs.python-version }}
+  POETRY_VERSION: ${{ inputs.poetry-version }}
+
+jobs:
+  # Main build job with matrix strategy for multiple OS
+  build:
+    name: Build (${{ matrix.os }}, Python ${{ matrix.python-version }})
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ${{ fromJson(inputs.os-matrix) }}
+        python-version: ${{ fromJson(inputs.python-versions-matrix) }}
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    # Common dependency installation for Unix-like systems
+    - name: Install base dependencies (Unix)
+      if: matrix.os != 'windows-latest'
+      run: |
+        python -m pip install --upgrade pip setuptools pyinstaller
+        curl -sSL https://install.python-poetry.org | python3 - --version ${{ env.POETRY_VERSION }}
+        poetry config virtualenvs.create false
+        poetry install --no-root
+
+    # Windows-specific dependency installation
+    - name: Install base dependencies (Windows)
+      if: matrix.os == 'windows-latest'
+      run: |
+        python -m pip install --upgrade pip setuptools pyinstaller
+        curl -sSL https://install.python-poetry.org | python3 - --version ${{ env.POETRY_VERSION }}
+        C:\Users\runneradmin\AppData\Roaming\Python\Scripts\poetry config virtualenvs.create false
+        C:\Users\runneradmin\AppData\Roaming\Python\Scripts\poetry install --no-root
+        #NOTE(mainred): Updated GITHUB_PATH works on next steps, not current step somehow
+        echo "C:\Users\runneradmin\AppData\Roaming\Python\Scripts" >> $env:GITHUB_PATH
+
+    # Linux-specific system dependencies
+    - name: Install system dependencies (Linux)
+      if: matrix.os == 'ubuntu-22.04'
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y binutils
+        # Add other Linux-specific packages here as needed
+        # Example: sudo apt-get install -y build-essential libffi-dev
+
+    # macOS-specific system dependencies
+    - name: Install system dependencies (macOS)
+      if: matrix.os == 'macos-latest'
+      run: |
+        brew install unixodbc
+        # Add other macOS-specific packages here as needed
+        # Example: brew install pkg-config libffi
+
+    # Windows-specific system dependencies (if needed)
+    - name: Install system dependencies (Windows)
+      if: matrix.os == 'windows-latest'
+      run: |
+        # Add Windows-specific installations here
+        # Example: choco install visualcpp-build-tools
+        echo "Windows system dependencies installation placeholder"
+
+    # Version update for release builds
+    - name: Update version (Linux)
+      if: contains(matrix.os, 'ubuntu') && inputs.is-release
+      run: sed -i 's/__version__ = .*/__version__ = "${{ github.ref_name }}"/g' holmes/__init__.py
+
+    - name: Update version (macOS)
+      if: matrix.os == 'macos-latest' && inputs.is-release
+      run: sed -i '' 's/__version__ = .*/__version__ = "${{ github.ref_name }}"/g' holmes/__init__.py
+
+    - name: Update version (Windows)
+      if: matrix.os == 'windows-latest' && inputs.is-release
+      run: |
+        $filePath = 'holmes/__init__.py'
+        (Get-Content $filePath) -replace '__version__ = .+', '__version__ = "${{ github.ref_name }}"' | Set-Content $filePath
+
+    # Build step with PyInstaller
+    - name: Build binary with PyInstaller
+      shell: bash
+      # regarding the tiktoken part of the command, see https://github.com/openai/tiktoken/issues/80
+      # regarding the litellm part of the command, see https://github.com/pyinstaller/pyinstaller/issues/8620#issuecomment-2186540504
+      run: |
+        # Customize this PyInstaller command based on your project structure
+        pyinstaller holmes_cli.py \
+          --name holmes \
+          --add-data 'holmes/plugins/runbooks/*:holmes/plugins/runbooks' \
+          --add-data 'holmes/plugins/prompts/*:holmes/plugins/prompts' \
+          --add-data 'holmes/plugins/toolsets/*:holmes/plugins/toolsets' \
+          --add-data 'holmes/plugins/toolsets/coralogix*:holmes/plugins/toolsets/coralogix' \
+          --add-data 'holmes/plugins/toolsets/grafana*:holmes/plugins/toolsets/grafana' \
+          --add-data 'holmes/plugins/toolsets/internet*:holmes/plugins/toolsets/internet' \
+          --add-data 'holmes/plugins/toolsets/opensearch*:holmes/plugins/toolsets/opensearch' \
+          --add-data 'holmes/plugins/toolsets/prometheus*:holmes/plugins/toolsets/prometheus' \
+          --hidden-import=tiktoken_ext.openai_public \
+          --hidden-import=tiktoken_ext \
+          --hiddenimport litellm.llms.tokenizers \
+          --hiddenimport litellm.litellm_core_utils.tokenizers \
+          --collect-data litellm
+        ls dist/
+
+    # Optional: Run tests
+    - name: Run tests
+      if: ${{ !inputs.is-release }}
+      env:
+        # Add your test environment variables here
+        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      run: |
+        poetry run pytest -m "not llm"
+
+    # Test the built binary
+    - name: Test binary (Unix)
+      if: matrix.os != 'windows-latest'
+      run: |
+        dist/holmes/holmes version
+        if [ $? -ne 0 ]; then
+          echo "Binary test failed"
+          exit 1
+        fi
+        echo "Binary test passed"
+
+    - name: Test binary (Windows)
+      if: matrix.os == 'windows-latest'
+      run: |
+        dist/holmes/holmes.exe version
+        if ($LASTEXITCODE -ne 0) {
+          Write-Output "Binary test failed"
+          exit 1
+        }
+        Write-Output "Binary test passed"
+
+    # Package the binary
+    - name: Package binary (Unix)
+      if: (matrix.os == 'macos-latest' || contains(matrix.os, 'ubuntu')) && inputs.is-release
+      run: |
+        cd dist
+        tar -czf holmes-${{ matrix.os }}-${{ github.run_number }}.tar.gz holmes
+        mv holmes-${{ matrix.os }}-${{ github.run_number }}.tar.gz ../
+        cd ..
+
+    - name: Package binary (Windows)
+      if: matrix.os == 'windows-latest' && inputs.is-release
+      run: |
+        Set-Location -Path dist
+        Compress-Archive -Path holmes -DestinationPath holmes-${{ matrix.os }}-${{ github.run_number }}.zip -Force
+        Move-Item -Path holmes-${{ matrix.os }}-${{ github.run_number }}.zip -Destination ..\
+        Set-Location -Path ..
+
+    # Upload artifacts
+    - name: Upload build artifacts
+      if: ${{ inputs.is-release }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: holmes-${{ matrix.os }}-${{ github.run_number }}
+        path: |
+          holmes-${{ matrix.os }}-${{ github.run_number }}.tar.gz
+          holmes-${{ matrix.os }}-${{ github.run_number }}.zip
+        retention-days: 30
+        if-no-files-found: error
+
+    # Release asset upload (only on release events)
+    - name: Upload release asset
+      if: ${{ inputs.is-release }}
+      uses: actions/upload-release-asset@v1.0.2
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: ./holmes-${{ matrix.os }}-${{ github.run_number }}.${{ matrix.os == 'windows-latest' && 'zip' || 'tar.gz' }}
+        asset_name: holmes-${{ matrix.os }}-${{ github.ref_name }}.${{ matrix.os == 'windows-latest' && 'zip' || 'tar.gz' }}
+        asset_content_type: application/octet-stream