Major Upgrade: Plugin System, Caching, Async Execution, Cloud Connectors & REST API

.github/workflows/ci.yml

@@ -0,0 +1,122 @@
+name: CI
+
+on:
+  push:
+    branches: [master, main, develop]
+  pull_request:
+    branches: [master, main, develop]
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Cache pip dependencies
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt', '**/pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Run linting
+        run: |
+          pip install ruff
+          ruff check ai_data_science_team/ --exit-zero
+
+      - name: Run tests
+        run: |
+          pytest tests/ -v --tb=short -x --ignore=tests/integration/
+        env:
+          PYTHONPATH: ${{ github.workspace }}
+
+      - name: Run tests with coverage
+        if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11'
+        run: |
+          pytest tests/ -v --cov=ai_data_science_team --cov-report=xml --cov-report=term --ignore=tests/integration/
+
+      - name: Upload coverage to Codecov
+        if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11'
+        uses: codecov/codecov-action@v4
+        with:
+          file: ./coverage.xml
+          fail_ci_if_error: false
+
+  integration-tests:
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Run integration tests
+        run: |
+          pytest tests/integration/ -v --tb=short -m "not requires_api"
+        env:
+          PYTHONPATH: ${{ github.workspace }}
+
+  type-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+          pip install mypy
+
+      - name: Run type checking
+        run: |
+          mypy ai_data_science_team/ --ignore-missing-imports --no-error-summary || true
+
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[docs]"
+
+      - name: Build docs
+        run: |
+          echo "Documentation build placeholder"
+          # mkdocs build --strict

.github/workflows/release.yml

@@ -0,0 +1,41 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      id-token: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install build dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build twine
+
+      - name: Build package
+        run: python -m build
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v1
+        with:
+          files: dist/*
+          generate_release_notes: true
+
+      - name: Publish to PyPI
+        if: startsWith(github.ref, 'refs/tags/v')
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          skip-existing: true

.gitignore

@@ -123,13 +123,115 @@ celerybeat.pid
 
 # Environments
 .env
+.env.*
+.env.local
+.env.development
+.env.production
+.env.test
 .venv
 env/
 venv/
 ENV/
 env.bak/
 venv.bak/
 
+# =============================================================================
+# SENSITIVE / PRIVATE INFORMATION - NEVER COMMIT
+# =============================================================================
+
+# API Keys and Secrets
+*.pem
+*.key
+*.p12
+*.pfx
+*.crt
+*.cer
+secrets.json
+secrets.yaml
+secrets.yml
+secrets.toml
+*_secret*
+*_secrets*
+api_key*
+apikey*
+.secrets/
+secrets/
+
+# Cloud Credentials
+credentials.json
+service_account*.json
+gcp-credentials*.json
+aws_credentials*
+.aws/
+.gcp/
+.azure/
+snowflake_credentials*
+bigquery_credentials*
+redshift_credentials*
+
+# Database Connection Strings
+database.ini
+db_config*
+connection_string*
+*.db
+*.sqlite
+*.sqlite3
+
+# SSH Keys
+id_rsa
+id_rsa.pub
+id_ed25519
+id_ed25519.pub
+*.ppk
+known_hosts
+
+# OAuth Tokens
+token.json
+tokens.json
+oauth_token*
+refresh_token*
+access_token*
+.token
+
+# Config files that may contain secrets
+config.local.*
+config.private.*
+*.local.json
+*.local.yaml
+*.local.yml
+*.private.json
+*.private.yaml
+*.private.yml
+local_config*
+private_config*
+
+# Jupyter notebooks with potential secrets
+*.ipynb
+!examples/*.ipynb
+
+# History files that may contain secrets
+.python_history
+.bash_history
+.zsh_history
+.node_repl_history
+.psql_history
+.mysql_history
+
+# Editor backup files
+*~
+*.swp
+*.swo
+*.bak
+*.backup
+*.orig
+
+# OS generated files
+Thumbs.db
+ehthumbs.db
+Desktop.ini
+.Spotlight-V100
+.Trashes
+
 # Spyder project settings
 .spyderproject
 .spyproject
@@ -181,3 +283,63 @@ mlflow_artifacts/
 
 # Conda Environments
 environment_ds4b_301p_dev.yml
+
+# =============================================================================
+# PROJECT SPECIFIC
+# =============================================================================
+
+# Cache directories (from our caching module)
+.cache/
+cache/
+*.cache
+
+# Uploaded/processed data
+data/
+uploads/
+downloads/
+output/
+outputs/
+results/
+
+# Model files
+*.pkl
+*.pickle
+*.joblib
+*.h5
+*.hdf5
+*.onnx
+*.pt
+*.pth
+models/
+checkpoints/
+
+# Large data files
+*.csv
+*.parquet
+*.feather
+*.arrow
+*.xlsx
+*.xls
+!tests/fixtures/*.csv
+!tests/fixtures/*.xlsx
+
+# API Server logs
+api_logs/
+server.log
+uvicorn.log
+
+# User-specific files
+.user/
+user_config*
+my_config*
+
+# Temporary test files
+.tmp/
+tmp/
+temp_*/
+
+# Claude/AI generated files that may contain context
+.claude/
+.anthropic/
+.openai/
+ai_context/

.pre-commit-config.yaml

@@ -0,0 +1,30 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+        args: ['--maxkb=1000']
+      - id: check-merge-conflict
+      - id: detect-private-key
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.2.0
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+
+  - repo: https://github.com/psf/black
+    rev: 24.1.0
+    hooks:
+      - id: black
+        language_version: python3
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.8.0
+    hooks:
+      - id: mypy
+        additional_dependencies: [types-all]
+        args: [--ignore-missing-imports]