Add evaluation server and Railway deployment

- FastAPI server for student policy submissions
- Evaluation runner for PufferLib environments
- SQLite leaderboard
- Docker setup with base image for fast deploys
- GitHub Action to build base image on native x86_64

Author: eugenevinitsky

.dockerignore

@@ -0,0 +1,11 @@
+.git
+.venv
+.claude
+__pycache__
+*.pyc
+*.db
+examples/
+resources
+uv.lock
+*.md
+.gitignore

.github/workflows/build-base-image.yml

@@ -0,0 +1,37 @@
+name: Build Base Image
+
+on:
+  workflow_dispatch:  # Manual trigger
+  push:
+    paths:
+      - 'Dockerfile.base'
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository_owner }}/rl-eval-base
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Log in to Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./Dockerfile.base
+          push: true
+          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest

.gitignore

@@ -217,3 +217,5 @@ models/
 *.ckpt
 tensorboard/
 resources
+server/leaderboard.db
+*.db

Dockerfile

@@ -0,0 +1,19 @@
+# Use pre-built base image with all dependencies
+# Build base with: docker build --platform linux/amd64 -t USERNAME/rl-eval-base -f Dockerfile.base .
+# Push with: docker push USERNAME/rl-eval-base
+FROM ghcr.io/eugenevinitsky/rl-eval-base:latest
+
+WORKDIR /app
+
+# Copy application code (this is the only thing that changes between deploys)
+COPY server/ /app/server/
+
+# Create data directory
+RUN mkdir -p /data
+
+ENV DATA_DIR=/data
+ENV PORT=8000
+
+EXPOSE 8000
+
+CMD ["uvicorn", "server.app:app", "--host", "0.0.0.0", "--port", "8000"]

Dockerfile.base

@@ -0,0 +1,26 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    git \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install uv for faster pip operations
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+ENV PATH="/root/.local/bin:$PATH"
+
+# Install PyTorch CPU-only
+RUN uv pip install --system --no-cache \
+    torch --index-url https://download.pytorch.org/whl/cpu
+
+# Install pufferlib and other heavy deps (this is the slow part)
+RUN uv pip install --system --no-cache \
+    numpy gymnasium pufferlib
+
+# Install web deps
+RUN uv pip install --system --no-cache \
+    fastapi uvicorn python-multipart

Procfile

@@ -0,0 +1 @@
+web: uvicorn server.app:app --host 0.0.0.0 --port $PORT

pyproject.toml

@@ -5,5 +5,8 @@ description = "Reinforcement Learning course project using PufferLib"
 readme = "README.md"
 requires-python = ">=3.13"
 dependencies = [
+    "fastapi>=0.128.0",
     "pufferlib>=3.0.0",
+    "python-multipart>=0.0.21",
+    "uvicorn>=0.40.0",
 ]

railway.toml

@@ -0,0 +1,9 @@
+[build]
+builder = "dockerfile"
+dockerfilePath = "Dockerfile"
+
+[deploy]
+healthcheckPath = "/health"
+healthcheckTimeout = 30
+restartPolicyType = "on_failure"
+restartPolicyMaxRetries = 3