cicd.yaml

# .github/workflows/cicd.yaml
name: LumiBot CI/CD

on:
  push:
    branches: [dev, main]
  pull_request:
    branches: [dev, main]

permissions:
  contents: read

env:
  AIOHTTP_NO_EXTENSIONS: 1
  # CRITICAL: Set this to "none" so tests use their explicit data sources.
  # Tests that want ThetaData must explicitly request it.
  # Without this, the default is "ThetaData" which overrides ALL backtests.
  BACKTESTING_DATA_SOURCE: none
  # Progress bars can produce huge logs and slow CI.
  BACKTESTING_SHOW_PROGRESS_BAR: "false"
  POLYGON_API_KEY: ${{ secrets.POLYGON_API_KEY }}
  # Prevent CI hangs if Polygon hits transient rate limits / retry storms.
  POLYGON_MAX_RETRY_ATTEMPTS: "12"
  POLYGON_MAX_RETRY_SLEEP_SECONDS: "120"
  POLYGON_WAIT_SECONDS_RETRY: "10"
  THETADATA_USERNAME: ${{ secrets.THETADATA_USERNAME }}
  THETADATA_PASSWORD: ${{ secrets.THETADATA_PASSWORD }}
  # NOTE (2025-11-28): Data Downloader is a production proxy for ThetaData that allows
  # shared access without requiring a local ThetaTerminal JAR. When these are set,
  # ThetaData tests will use the remote downloader instead of spawning a local process.
  DATADOWNLOADER_BASE_URL: ${{ secrets.DATADOWNLOADER_BASE_URL }}
  DATADOWNLOADER_API_KEY: ${{ secrets.DATADOWNLOADER_API_KEY }}
  ALPACA_TEST_API_KEY: ${{ secrets.ALPACA_TEST_API_KEY }} # Required for alpaca unit tests
  ALPACA_TEST_API_SECRET: ${{ secrets.ALPACA_TEST_API_SECRET }} # Required for alpaca unit tests
  TRADIER_TEST_ACCESS_TOKEN: ${{ secrets.TRADIER_TEST_ACCESS_TOKEN }} # Required for tradier unit tests
  TRADIER_TEST_ACCOUNT_NUMBER: ${{ secrets.TRADIER_TEST_ACCOUNT_NUMBER }} # Required for tradier unit tests

jobs:
  lint:
    name: Lint
    runs-on: ubuntu-latest
    timeout-minutes: 15
    environment: unit-tests
    steps:
      - uses: actions/checkout@v3

      - name: Set up Python 3.10
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"
          cache: pip

      - name: Install dependencies
        run: |
          echo "Set AIOHTTP_NO_EXTENSIONS=$AIOHTTP_NO_EXTENSIONS so that aiohttp doesn't try to install C extensions"
          python -m pip install --upgrade pip
          pip install requests
          pip install -r requirements_dev.txt

      - name: Run Ruff (ThetaData scope)
        run: |
          # NOTE: The repo currently has a large backlog of Ruff violations outside the
          # ThetaData backtesting surface area. To keep CI meaningful without blocking
          # the PR on unrelated legacy lint, lint only the ThetaData-related files and
          # tests this PR touches.
          ruff check --select F,I \
            lumibot/tools/thetadata_helper.py \
            lumibot/tools/thetadata_queue_client.py \
            lumibot/backtesting/thetadata_backtesting_pandas.py \
            lumibot/components/options_helper.py \
            lumibot/strategies/_strategy.py \
            tests/backtest/test_acceptance_backtests_ci.py \
            tests/test_thetadata_day_timestamp_alignment.py \
            tests/test_thetadata_get_last_price_trade_only.py \
            tests/test_options_helper_thetadata_actionable_strikes.py \
            tests/test_thetadata_queue_client.py

  unit-tests:
    name: Unit Tests (shard ${{ matrix.shard }}/6)
    runs-on: ubuntu-latest
    timeout-minutes: 30
    environment: unit-tests
    needs: lint
    strategy:
      fail-fast: false
      matrix:
        shard: [0, 1, 2, 3, 4, 5]

    steps:
      - uses: actions/checkout@v3

      - name: Set up Python 3.10
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"
          cache: pip

      - name: Install dependencies
        run: |
          echo "Set AIOHTTP_NO_EXTENSIONS=$AIOHTTP_NO_EXTENSIONS so that aiohttp doesn't try to install C extensions"
          python -m pip install --upgrade pip
          pip install requests
          pip install -r requirements_dev.txt

      - name: Run unit tests (sharded)
        env:
          SHARD_INDEX: ${{ matrix.shard }}
          SHARD_TOTAL: "6"
        run: |
          set -euo pipefail

          # CI target: keep PR checks fast and deterministic.
          # Downloader + apitests are run separately / opt-in.
          PYTEST_MARKERS='not apitest and not downloader'
          export PYTEST_MARKERS

          echo "Shard ${SHARD_INDEX}/${SHARD_TOTAL} markers=${PYTEST_MARKERS}"

          python - <<'PY'
          import os
          import subprocess
          import sys
          from collections import Counter

          shard_index = int(os.environ["SHARD_INDEX"])
          shard_total = int(os.environ["SHARD_TOTAL"])
          markers = os.environ.get("PYTEST_MARKERS", "not apitest and not downloader")

          collect_cmd = [
              "pytest",
              "tests/",
              "--ignore=tests/backtest/",
              "-m",
              markers,
              "--collect-only",
              "-q",
          ]
          proc = subprocess.run(collect_cmd, capture_output=True, text=True)
          if proc.returncode != 0:
              sys.stdout.write(proc.stdout)
              sys.stderr.write(proc.stderr)
              raise SystemExit(proc.returncode)

          nodeids = [line.strip() for line in proc.stdout.splitlines() if "::" in line]
          counts = Counter(nodeid.split("::", 1)[0] for nodeid in nodeids)
          files = sorted(counts.items(), key=lambda item: (-item[1], item[0]))

          bins = [{"count": 0, "files": []} for _ in range(shard_total)]
          for path, count in files:
              target_bin = min(bins, key=lambda b: (b["count"], len(b["files"])))
              target_bin["files"].append(path)
              target_bin["count"] += count

          selected_files = bins[shard_index]["files"]
          with open("shard_files.txt", "w", encoding="utf-8") as handle:
              handle.write("\n".join(selected_files))
              handle.write("\n")

          print(
              f"selected_files={len(selected_files)} total_tests={bins[shard_index]['count']} "
              f"bins={[b['count'] for b in bins]}"
          )
          PY

          echo "Running $(wc -l shard_files.txt | awk '{print $1}') files"
          timeout 1500 python -m pytest -m "${PYTEST_MARKERS}" --tb=short -q --durations=30 -x $(cat shard_files.txt)

  backtest-tests:
    name: Backtest Tests (shard ${{ matrix.shard }}/4)
    runs-on: ubuntu-latest
    timeout-minutes: 30
    environment: unit-tests
    needs: lint
    strategy:
      fail-fast: false
      matrix:
        shard: [0, 1, 2, 3]

    steps:
      - uses: actions/checkout@v3

      - name: Set up Python 3.10
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"
          cache: pip

      - name: Install dependencies
        run: |
          echo "Set AIOHTTP_NO_EXTENSIONS=$AIOHTTP_NO_EXTENSIONS so that aiohttp doesn't try to install C extensions"
          python -m pip install --upgrade pip
          pip install requests
          pip install -r requirements_dev.txt

      - name: Run backtest tests (sharded)
        env:
          SHARD_INDEX: ${{ matrix.shard }}
          SHARD_TOTAL: "4"
          # Acceptance backtests (in tests/backtest/) require the same ThetaData + S3 secrets used locally.
          # These env var names intentionally match Strategy Library/Demos/.env.
          THETADATA_USERNAME: ${{ secrets.THETADATA_USERNAME }}
          THETADATA_PASSWORD: ${{ secrets.THETADATA_PASSWORD }}
          DATADOWNLOADER_BASE_URL: ${{ secrets.DATADOWNLOADER_BASE_URL }}
          DATADOWNLOADER_API_KEY: ${{ secrets.DATADOWNLOADER_API_KEY }}
          LUMIBOT_CACHE_S3_BUCKET: ${{ secrets.LUMIBOT_CACHE_S3_BUCKET }}
          LUMIBOT_CACHE_S3_PREFIX: ${{ secrets.LUMIBOT_CACHE_S3_PREFIX }}
          LUMIBOT_CACHE_S3_REGION: ${{ secrets.LUMIBOT_CACHE_S3_REGION }}
          LUMIBOT_CACHE_S3_VERSION: ${{ secrets.LUMIBOT_CACHE_S3_VERSION }}
          LUMIBOT_CACHE_S3_ACCESS_KEY_ID: ${{ secrets.LUMIBOT_CACHE_S3_ACCESS_KEY_ID }}
          LUMIBOT_CACHE_S3_SECRET_ACCESS_KEY: ${{ secrets.LUMIBOT_CACHE_S3_SECRET_ACCESS_KEY }}
        run: |
          set -euo pipefail

          PYTEST_MARKERS='not apitest and not downloader'
          export PYTEST_MARKERS

          echo "Shard ${SHARD_INDEX}/${SHARD_TOTAL} markers=${PYTEST_MARKERS}"

          python - <<'PY'
          import os
          import subprocess
          import sys

          shard_index = int(os.environ["SHARD_INDEX"])
          shard_total = int(os.environ["SHARD_TOTAL"])
          markers = os.environ.get("PYTEST_MARKERS", "not apitest and not downloader")

          collect_cmd = [
              "pytest",
              "tests/backtest/",
              "-m",
              markers,
              "--collect-only",
              "-q",
          ]
          proc = subprocess.run(collect_cmd, capture_output=True, text=True)
          if proc.returncode != 0:
              sys.stdout.write(proc.stdout)
              sys.stderr.write(proc.stderr)
              raise SystemExit(proc.returncode)

          nodeids = [line.strip() for line in proc.stdout.splitlines() if "::" in line]
          bins = [[] for _ in range(shard_total)]
          for idx, nodeid in enumerate(nodeids):
              bins[idx % shard_total].append(nodeid)

          selected = bins[shard_index]
          with open("shard_nodeids.txt", "w", encoding="utf-8") as handle:
              handle.write("\n".join(selected))
              handle.write("\n")

          print(f"selected_nodeids={len(selected)} total_nodeids={len(nodeids)}")
          PY

          echo "Running $(wc -l shard_nodeids.txt | awk '{print $1}') nodeids"
          timeout 1500 python -m pytest -m "${PYTEST_MARKERS}" --tb=short -q --durations=30 -x $(cat shard_nodeids.txt)

  LintAndTest:
    name: LintAndTest
    runs-on: ubuntu-latest
    if: always()
    needs: [lint, unit-tests, backtest-tests]
    steps:
      - name: Check results
        run: |
          echo "lint: ${{ needs.lint.result }}"
          echo "unit-tests: ${{ needs.unit-tests.result }}"
          echo "backtest-tests: ${{ needs.backtest-tests.result }}"

          if [ "${{ needs.lint.result }}" != "success" ]; then
            exit 1
          fi
          if [ "${{ needs.unit-tests.result }}" != "success" ]; then
            exit 1
          fi
          if [ "${{ needs.backtest-tests.result }}" != "success" ]; then
            exit 1
          fi