diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..666bb11f2 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,99 @@ +# Dependabot configuration for automated dependency updates +# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates + +version: 2 +updates: + # Python backend dependencies (uv/pip) + - package-ecosystem: "pip" + directory: "/surfsense_backend" + schedule: + interval: "weekly" + day: "monday" + open-pull-requests-limit: 5 + groups: + python-minor-patch: + patterns: + - "*" + update-types: + - "minor" + - "patch" + labels: + - "dependencies" + - "python" + commit-message: + prefix: "chore(deps)" + + # Frontend web dependencies (pnpm/npm) + - package-ecosystem: "npm" + directory: "/surfsense_web" + schedule: + interval: "weekly" + day: "monday" + open-pull-requests-limit: 5 + groups: + npm-minor-patch: + patterns: + - "*" + update-types: + - "minor" + - "patch" + labels: + - "dependencies" + - "javascript" + commit-message: + prefix: "chore(deps)" + + # Browser extension dependencies (pnpm/npm) + - package-ecosystem: "npm" + directory: "/surfsense_browser_extension" + schedule: + interval: "weekly" + day: "monday" + open-pull-requests-limit: 5 + groups: + extension-minor-patch: + patterns: + - "*" + update-types: + - "minor" + - "patch" + labels: + - "dependencies" + - "javascript" + - "extension" + commit-message: + prefix: "chore(deps)" + + # GitHub Actions dependencies + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + open-pull-requests-limit: 3 + labels: + - "dependencies" + - "github-actions" + commit-message: + prefix: "chore(ci)" + + # Docker dependencies + - package-ecosystem: "docker" + directory: "/surfsense_backend" + schedule: + interval: "monthly" + labels: + - "dependencies" + - "docker" + commit-message: + prefix: "chore(docker)" + + - package-ecosystem: "docker" + directory: "/surfsense_web" + schedule: + interval: "monthly" + labels: + - "dependencies" + - "docker" + commit-message: + prefix: "chore(docker)" diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a391ba83c..70927ab7b 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -2,45 +2,87 @@ name: Docker Publish on: workflow_dispatch: + inputs: + push_backend: + description: 'Push backend image' + required: false + default: true + type: boolean + push_frontend: + description: 'Push frontend image' + required: false + default: true + type: boolean + release: + types: [published] + push: + branches: [main] + paths: + - 'surfsense_backend/Dockerfile' + - 'surfsense_web/Dockerfile' + - '.github/workflows/docker-publish.yml' + +env: + REGISTRY: ghcr.io jobs: - # build_and_push_backend: - # runs-on: ubuntu-latest - # permissions: - # contents: read - # packages: write - # steps: - # - name: Checkout repository - # uses: actions/checkout@v4 + build_and_push_backend: + name: Build & Push Backend + runs-on: ubuntu-latest + if: | + github.event_name == 'release' || + (github.event_name == 'workflow_dispatch' && inputs.push_backend) + permissions: + contents: read + packages: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 - # - name: Set up QEMU - # uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v3 + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - # - name: Log in to GitHub Container Registry - # uses: docker/login-action@v3 - # with: - # registry: ghcr.io - # username: ${{ github.actor }} - # password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata for backend + id: meta-backend + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/surfsense_backend + tags: | + type=sha,prefix= + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=latest,enable={{is_default_branch}} - # - name: Build and push backend image - # uses: docker/build-push-action@v5 - # with: - # context: ./surfsense_backend - # file: ./surfsense_backend/Dockerfile - # push: true - # tags: ghcr.io/${{ github.repository_owner }}/surfsense_backend:${{ github.sha }} - # platforms: linux/amd64,linux/arm64 - # labels: | - # org.opencontainers.image.source=${{ github.repositoryUrl }} - # org.opencontainers.image.created=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }} - # org.opencontainers.image.revision=${{ github.sha }} + - name: Build and push backend image + uses: docker/build-push-action@v6 + with: + context: ./surfsense_backend + file: ./surfsense_backend/Dockerfile + push: true + tags: ${{ steps.meta-backend.outputs.tags }} + labels: ${{ steps.meta-backend.outputs.labels }} + platforms: linux/amd64,linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max build_and_push_frontend: + name: Build & Push Frontend runs-on: ubuntu-latest + if: | + github.event_name == 'release' || + (github.event_name == 'workflow_dispatch' && inputs.push_frontend) || + github.event_name == 'push' permissions: contents: read packages: write @@ -57,19 +99,30 @@ jobs: - name: Log in to GitHub Container Registry uses: docker/login-action@v3 with: - registry: ghcr.io + registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata for frontend + id: meta-frontend + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/surfsense_web + tags: | + type=sha,prefix= + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=latest,enable={{is_default_branch}} + - name: Build and push frontend image - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./surfsense_web file: ./surfsense_web/Dockerfile push: true - tags: ghcr.io/${{ github.repository_owner }}/surfsense_web:${{ github.sha }} + tags: ${{ steps.meta-frontend.outputs.tags }} + labels: ${{ steps.meta-frontend.outputs.labels }} platforms: linux/amd64,linux/arm64 - labels: | - org.opencontainers.image.source=${{ github.repositoryUrl }} - org.opencontainers.image.created=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }} - org.opencontainers.image.revision=${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..7655727a4 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,155 @@ +name: Tests + +on: + pull_request: + branches: [main, dev] + types: [opened, synchronize, reopened, ready_for_review] + push: + branches: [main, dev] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + backend-tests: + name: Backend Tests + runs-on: ubuntu-latest + if: github.event.pull_request.draft == false + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Check if backend files changed + id: backend-changes + uses: dorny/paths-filter@v3 + with: + filters: | + backend: + - 'surfsense_backend/**' + + - name: Set up Docker Buildx + if: steps.backend-changes.outputs.backend == 'true' + uses: docker/setup-buildx-action@v3 + + - name: Cache Docker layers + if: steps.backend-changes.outputs.backend == 'true' + uses: actions/cache@v4 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ hashFiles('surfsense_backend/Dockerfile', 'surfsense_backend/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Create test environment file + if: steps.backend-changes.outputs.backend == 'true' + run: | + cat > surfsense_backend/.env << 'EOF' + DATABASE_URL=postgresql+asyncpg://postgres:postgres@db:5432/surfsense + CELERY_BROKER_URL=redis://redis:6379/0 + CELERY_RESULT_BACKEND=redis://redis:6379/0 + SECRET_KEY=test-secret-key-for-ci + TESTING=true + EOF + + - name: Build and run tests with Docker Compose + if: steps.backend-changes.outputs.backend == 'true' + run: | + # Start dependencies + docker compose up -d db redis + + # Wait for services to be ready + echo "Waiting for PostgreSQL..." + timeout 60 bash -c 'until docker compose exec -T db pg_isready -U postgres; do sleep 2; done' + + echo "Waiting for Redis..." + timeout 30 bash -c 'until docker compose exec -T redis redis-cli ping | grep -q PONG; do sleep 2; done' + + # Build backend (pytest is already in Dockerfile) + docker compose build backend + + # Run tests (pytest is baked into the image) + docker compose run --rm -e TESTING=true backend pytest tests/ -v --tb=short --cov=app --cov-report=xml + + # Copy coverage report from the container + docker compose run --rm backend cat /app/coverage.xml > surfsense_backend/coverage.xml || true + + - name: Stop Docker Compose services + if: always() && steps.backend-changes.outputs.backend == 'true' + run: docker compose down -v + + - name: Upload coverage reports + if: steps.backend-changes.outputs.backend == 'true' + uses: codecov/codecov-action@v4 + with: + file: surfsense_backend/coverage.xml + flags: backend + fail_ci_if_error: false + continue-on-error: true + + frontend-tests: + name: Frontend Tests + runs-on: ubuntu-latest + if: github.event.pull_request.draft == false + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Check if frontend files changed + id: frontend-changes + uses: dorny/paths-filter@v3 + with: + filters: | + frontend: + - 'surfsense_web/**' + + - name: Setup Node.js + if: steps.frontend-changes.outputs.frontend == 'true' + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install pnpm + if: steps.frontend-changes.outputs.frontend == 'true' + uses: pnpm/action-setup@v4 + with: + version: 9 + + - name: Cache dependencies + if: steps.frontend-changes.outputs.frontend == 'true' + uses: actions/cache@v4 + with: + path: | + ~/.pnpm-store + surfsense_web/node_modules + key: pnpm-deps-${{ hashFiles('surfsense_web/pnpm-lock.yaml') }} + restore-keys: | + pnpm-deps- + + - name: Install dependencies + if: steps.frontend-changes.outputs.frontend == 'true' + working-directory: surfsense_web + run: pnpm install --frozen-lockfile + + - name: Run tests + if: steps.frontend-changes.outputs.frontend == 'true' + working-directory: surfsense_web + run: pnpm test + + test-gate: + name: Test Gate + runs-on: ubuntu-latest + needs: [backend-tests, frontend-tests] + if: always() + + steps: + - name: Check test jobs status + run: | + if [[ "${{ needs.backend-tests.result }}" == "failure" || "${{ needs.frontend-tests.result }}" == "failure" ]]; then + echo "โŒ Tests failed" + exit 1 + else + echo "โœ… All tests passed" + fi diff --git a/.gitignore b/.gitignore index 342c0b258..3289c1c85 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,78 @@ .flashrank_cache* ./surfsense_backend/podcasts/ .env +.env.* +!.env.example node_modules/ -.ruff_cache/ \ No newline at end of file +.ruff_cache/ + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +/lib/ +/lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# Testing & Coverage +.coverage +.coverage.* +htmlcov/ +.tox/ +.nox/ +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +nosetests.xml + +# Virtual environments +venv/ +ENV/ +env/ +.venv/ + +# IDE +.idea/ +.vscode/ +*.swp +*.swo +*.sublime-workspace +*.sublime-project + +# Lock files (package managers handle these) +uv.lock + +# OS files +.DS_Store +Thumbs.db +*~ + +# Logs +*.log +logs/ + +# Temporary files +tmp/ +temp/ +*.tmp +*.temp + +# Editor backup files +*.bak +*.backup \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cc5dde3cc..6d8bf520c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,7 +71,11 @@ For detailed setup instructions, refer to our [Installation Guide](https://www.s SurfSense consists of three main components: - **`surfsense_backend/`** - Python/FastAPI backend service + - `app/` - Main application code + - `tests/` - Test suite (pytest) + - `alembic/` - Database migrations - **`surfsense_web/`** - Next.js web application + - `tests/` - Frontend tests (vitest) - **`surfsense_browser_extension/`** - Browser extension for data collection ## ๐Ÿงช Development Guidelines @@ -98,9 +102,48 @@ refactor: improve error handling in connectors ``` ### Testing + +We use Docker Compose to run tests with all dependencies (PostgreSQL, Redis, etc.). + +#### Running Backend Tests + +```bash +# Start the test dependencies +docker compose up -d db redis + +# Build the backend (pytest is included in the Docker image) +docker compose build backend + +# Run all tests +docker compose run --rm -e TESTING=true backend pytest tests/ -v --tb=short + +# Run tests with coverage +docker compose run --rm -e TESTING=true backend pytest tests/ -v --tb=short --cov=app --cov-report=html + +# Run a specific test file +docker compose run --rm -e TESTING=true backend pytest tests/test_celery_tasks.py -v + +# Run tests matching a pattern +docker compose run --rm -e TESTING=true backend pytest tests/ -v -k "test_slack" + +# Stop services when done +docker compose down -v +``` + +#### Running Frontend Tests + +```bash +cd surfsense_web +pnpm install +pnpm test +``` + +#### Test Guidelines - Write tests for new features and bug fixes - Ensure existing tests pass before submitting - Include integration tests for API endpoints +- Use `pytest-asyncio` for async tests in the backend +- Mock external services and APIs appropriately ### Branch Naming Use descriptive branch names: diff --git a/docker-compose.yml b/docker-compose.yml index 5bf17ec8a..8785d6534 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: "3.8" - services: db: image: ankane/pgvector:latest @@ -39,6 +37,8 @@ services: - "${BACKEND_PORT:-8000}:8000" volumes: - ./surfsense_backend/app:/app/app + - ./surfsense_backend/tests:/app/tests + - ./surfsense_backend/pytest.ini:/app/pytest.ini - shared_temp:/tmp env_file: - ./surfsense_backend/.env @@ -125,6 +125,30 @@ services: depends_on: - backend + # ============================================================================ + # TEST SERVICE + # Use: docker compose --profile test run --rm backend-test + # Or: docker compose --profile test run --rm backend-test pytest tests/ -v + # ============================================================================ + backend-test: + profiles: ["test"] + build: ./surfsense_backend + volumes: + - ./surfsense_backend/app:/app/app + - ./surfsense_backend/tests:/app/tests + - ./surfsense_backend/pytest.ini:/app/pytest.ini + - ./surfsense_backend/pyproject.toml:/app/pyproject.toml + environment: + - DATABASE_URL=postgresql+asyncpg://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-surfsense} + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + - PYTHONPATH=/app + - TESTING=true + depends_on: + - db + - redis + command: ["pytest", "tests/", "-v", "--tb=short"] + volumes: postgres_data: pgadmin_data: diff --git a/surfsense_backend/Dockerfile b/surfsense_backend/Dockerfile index 91a225754..e473d2d53 100644 --- a/surfsense_backend/Dockerfile +++ b/surfsense_backend/Dockerfile @@ -35,9 +35,10 @@ RUN if [ "$(uname -m)" = "x86_64" ]; then \ pip install --no-cache-dir torch torchvision torchaudio; \ fi -# Install python dependencies +# Install python dependencies including test dependencies RUN pip install --no-cache-dir uv && \ - uv pip install --system --no-cache-dir -e . + uv pip install --system --no-cache-dir -e . && \ + uv pip install --system --no-cache-dir pytest pytest-asyncio pytest-cov # Set SSL environment variables dynamically RUN CERTIFI_PATH=$(python -c "import certifi; print(certifi.where())") && \ diff --git a/surfsense_backend/README.md b/surfsense_backend/README.md new file mode 100644 index 000000000..bb0f5a92c --- /dev/null +++ b/surfsense_backend/README.md @@ -0,0 +1,137 @@ +# SurfSense Backend + +Python/FastAPI backend service for SurfSense. + +## Quick Start + +### Prerequisites + +- Python 3.12+ +- PostgreSQL with PGVector extension +- Redis (for Celery task queue) +- Docker & Docker Compose (recommended) + +### Development Setup + +1. **Clone and navigate to backend** + ```bash + cd surfsense_backend + ``` + +2. **Install dependencies with UV** + ```bash + uv sync + ``` + +3. **Set up environment variables** + ```bash + cp .env.example .env + # Edit .env with your configuration + ``` + +4. **Run database migrations** + ```bash + uv run alembic upgrade head + ``` + +5. **Start the development server** + ```bash + uv run uvicorn app.app:app --reload + ``` + +## Testing + +We use pytest for testing with Docker Compose for dependencies. + +### Running Tests + +```bash +# Start dependencies +docker compose up -d db redis + +# Run all tests +docker compose run --rm -e TESTING=true backend pytest tests/ -v --tb=short + +# Run with coverage +docker compose run --rm -e TESTING=true backend pytest tests/ -v --tb=short --cov=app --cov-report=html + +# Run specific test file +docker compose run --rm -e TESTING=true backend pytest tests/test_celery_tasks.py -v + +# Run tests matching a pattern +docker compose run --rm -e TESTING=true backend pytest tests/ -v -k "test_slack" + +# Stop services +docker compose down -v +``` + +### Test Categories + +Tests are organized by markers: +- `@pytest.mark.unit` - Fast, isolated unit tests +- `@pytest.mark.integration` - Tests requiring external services +- `@pytest.mark.slow` - Slow running tests + +### Running Locally (without Docker) + +```bash +# Ensure PostgreSQL and Redis are running locally +export TESTING=true +uv run pytest tests/ -v --tb=short +``` + +## Project Structure + +``` +surfsense_backend/ +โ”œโ”€โ”€ alembic/ # Database migrations +โ”‚ โ”œโ”€โ”€ versions/ # Migration files +โ”‚ โ””โ”€โ”€ env.py # Alembic configuration +โ”œโ”€โ”€ app/ +โ”‚ โ”œโ”€โ”€ __init__.py +โ”‚ โ”œโ”€โ”€ app.py # FastAPI application +โ”‚ โ”œโ”€โ”€ celery_app.py # Celery configuration +โ”‚ โ”œโ”€โ”€ db.py # Database models +โ”‚ โ”œโ”€โ”€ users.py # User authentication +โ”‚ โ”œโ”€โ”€ agents/ # LLM agents (researcher, podcaster) +โ”‚ โ”œโ”€โ”€ config/ # Configuration management +โ”‚ โ”œโ”€โ”€ connectors/ # External service connectors +โ”‚ โ”œโ”€โ”€ prompts/ # LLM prompts +โ”‚ โ”œโ”€โ”€ retriever/ # Search and retrieval logic +โ”‚ โ”œโ”€โ”€ routes/ # API endpoints +โ”‚ โ”œโ”€โ”€ schemas/ # Pydantic models +โ”‚ โ”œโ”€โ”€ services/ # Business logic services +โ”‚ โ”œโ”€โ”€ tasks/ # Celery tasks +โ”‚ โ””โ”€โ”€ utils/ # Utility functions +โ”œโ”€โ”€ tests/ # Test suite +โ”‚ โ”œโ”€โ”€ conftest.py # Pytest fixtures +โ”‚ โ”œโ”€โ”€ test_*.py # Test modules +โ”‚ โ””โ”€โ”€ __init__.py +โ”œโ”€โ”€ Dockerfile # Container configuration +โ”œโ”€โ”€ pyproject.toml # Project dependencies +โ”œโ”€โ”€ pytest.ini # Pytest configuration +โ””โ”€โ”€ alembic.ini # Alembic configuration +``` + +## API Documentation + +When running the development server, API documentation is available at: +- Swagger UI: http://localhost:8000/docs +- ReDoc: http://localhost:8000/redoc + +## Configuration + +Key environment variables: + +| Variable | Description | Default | +|----------|-------------|---------| +| `DATABASE_URL` | PostgreSQL connection string | Required | +| `CELERY_BROKER_URL` | Redis URL for Celery | Required | +| `SECRET_KEY` | Application secret key | Required | +| `TESTING` | Enable test mode | `false` | + +See `.env.example` for full configuration options. + +## Contributing + +See [CONTRIBUTING.md](../CONTRIBUTING.md) for development guidelines.