feat: WIP multi-step scheduling continuation

.github/workflows/docker-reshuffle-sim.yml

@@ -7,8 +7,13 @@ on:
         description: "Git ref to build (branch, tag, or SHA). Leave empty to build default branch HEAD."
         required: false
         default: ""
-  # Also run on PRs that touch this image, so it can be exercised before it reaches master (where
-  # workflow_dispatch first becomes available). This builds AND pushes :latest + :sha from the PR.
+      push:
+        description: "Push the built image to GCR (uncheck to build only)."
+        type: boolean
+        required: false
+        default: true
+  # Also run on PRs that touch this image to verify it still builds. PR runs are build-only;
+  # pushing happens only on a manual workflow_dispatch run with the push input checked.
   pull_request:
     paths:
       - "tools/reshuffle_sim/**"
@@ -34,12 +39,18 @@ jobs:
           ref: ${{ github.event.inputs.ref }}
           fetch-depth: 1
 
+      # setup-gcloud@v2 does NOT authenticate (the service_account_key/export_default_credentials
+      # inputs were removed in v1+); credentials must come from the auth action, or the docker push
+      # goes out unauthenticated.
+      - name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+
       - name: Setup gcloud
         uses: google-github-actions/setup-gcloud@v2
         with:
           project_id: ${{ secrets.GCP_PROJECT }}
-          service_account_key: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
-          export_default_credentials: true
 
       - name: Configure Docker auth for GCR
         run: gcloud --quiet auth configure-docker gcr.io
@@ -60,6 +71,8 @@ jobs:
             .
 
       - name: Push image (latest + sha)
+        # Push only on a manual dispatch with the push input checked; PR runs are build-only.
+        if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.push == 'true' }}
         run: |
           docker push "${IMAGE}:latest"
           docker push "${IMAGE}:${{ steps.vars.outputs.SHA }}"

Cargo.toml

@@ -9,12 +9,17 @@ edition = "2024"
 [features]
 default = []
 # The MVCC chunk-lifecycle storage / multi-step scheduler (Postgres backend + sim tests).
-mvcc-chunks = ["dep:sqlx", "sqd-assignments/mvcc-chunks"]
+mvcc-chunks = ["dep:sqlx", "dep:bit-vec", "sqd-assignments/mvcc-chunks"]
+# Exposes the Postgres testcontainer harness (`scheduler_storage::test_harness::pg_harness`) to other
+# crates — e.g. reshuffle-sim. Implies the Postgres backend. The crate's own tests get the harness via
+# `cfg(test)` + dev-dependencies, so they don't need this feature.
+pg-testkit = ["mvcc-chunks", "dep:dtor", "dep:testcontainers", "dep:testcontainers-modules"]
 
 [dependencies]
 anyhow = "1.0.98"
 aws-config = { version = "1.6.2", features = ["behavior-version-latest"] }
 aws-sdk-s3 = "1.85.0"
+bit-vec = { version = "0.8", optional = true }
 bytesize = "2.3.1"
 chrono = "0.4.41"
 clap = { version = "4.5.38", features = ["derive", "env"] }
@@ -47,7 +52,13 @@ tokio = { version = "1.45.0", features = ["full"] }
 tracing = "0.1.41"
 tracing-subscriber = { version = "0.3.19", features = ["env-filter", "parking_lot"] }
 secrecy = { version = "0.10.3", features = ["serde"] }
-sqlx = { version = "0.9", features = ["runtime-tokio", "postgres", "migrate", "chrono", "derive"], optional = true }
+sqlx = { version = "0.9", features = ["runtime-tokio", "postgres", "migrate", "chrono", "derive", "tls-native-tls", "bit-vec", "json"], optional = true }
+# Also dev-dependencies (for the crate's own tests); optional here so the `pg-testkit` feature can hand
+# the harness to downstream crates without forcing testcontainers into a plain build.
+dtor = { version = "1.0.5", optional = true }
+testcontainers = { version = "0.27.2", features = ["watchdog"], optional = true }
+testcontainers-modules = { version = "0.15", features = ["postgres"], optional = true }
+rustc-hash = "2"
 
 [dev-dependencies]
 claims = "0.8"

docs/README.md

@@ -47,7 +47,7 @@ built.
 | Two-phase publish (worker assignment confirmed → portal promotion) | ✅ | Gated on the confirmation watermark. |
 | Chunk corrections (1-to-1 same-range swaps) | ✅ | Both backends; property-tested. |
 | Deferred removal / stale mappings (`ideal ∪ stale`) | ✅ | Both backends. |
-| Capacity-aware placement (footprint charge, credit-held-free, skip-don't-spill, essentials-first) | ✅ | `Reconcile` in `multistep_scheduler/mod.rs`. |
+| Capacity-aware placement (footprint charge, credit-held-free, bounded-load spill, essentials-first) | ✅ | `Reconcile` in `multistep_scheduler/mod.rs`. |
 | X% promotion threshold (quorum) | 🧪 | Computed by the sim's worker fleet; the storage core only consumes a precomputed confirmation watermark. No production quorum logic. |
 | Replica-sufficiency **drain gate** (capacity-aware "rule 2") | 📐 | Drains expire on the M-tick timer only. |
 | Replica-sufficiency **promotion gate** (capacity-aware "rule 3") | 📐 | Promotion is gated on assignment confirmation, not on a confirmed-copy count. |