ci: generate DB fixtures on-demand instead of committing binaries

kariy · claude · kariy · commit 4c0b779cc7e3 · 2026-02-09T16:31:44.000-06:00
Remove spawn_and_move.tar.gz and simple.tar.gz from git. These were
built on macOS (16KB page size) and fail on Linux CI (4KB page size)
with MDBX_CORRUPTED. Instead, add a `generate-db-fixtures` CI job
that compiles and runs `generate_migration_db` on the same platform
where tests run. Uses actions/cache keyed on db version + dojo ref
so full generation only happens on cache miss.

Co-Authored-By: Claude Opus 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -86,8 +86,6 @@ jobs:
           if-no-files-found: error
           path: |
             ./tests/snos/snos/build
-            ./tests/fixtures/db/spawn_and_move
-            ./tests/fixtures/db/simple
             ./crates/contracts/build
 
   build-katana-binary:
@@ -159,8 +157,100 @@ jobs:
       - name: Run Clippy
         run: ./scripts/clippy.sh
 
+  generate-db-fixtures:
+    needs: [generate-test-artifacts]
+    runs-on: ubuntu-latest-32-cores
+    if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.draft == false)
+    container:
+      image: ghcr.io/dojoengine/katana-dev:latest
+    env:
+      MLIR_SYS_190_PREFIX: /usr/lib/llvm-19/
+      LLVM_SYS_191_PREFIX: /usr/lib/llvm-19/
+      TABLEGEN_190_PREFIX: /usr/lib/llvm-19/
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+      # Workaround for https://github.com/actions/runner-images/issues/6775
+      - run: git config --global --add safe.directory "*"
+
+      - name: Cache DB fixtures
+        id: cache-db-fixtures
+        uses: actions/cache@v4
+        with:
+          path: |
+            tests/fixtures/db/spawn_and_move
+            tests/fixtures/db/simple
+          key: db-fixtures-${{ hashFiles('crates/storage/db/src/version.rs') }}-dojo-v1.7.0
+
+      - name: Download contract artifacts
+        if: steps.cache-db-fixtures.outputs.cache-hit != 'true'
+        uses: actions/download-artifact@v5
+        with:
+          name: test-artifacts
+
+      - uses: Swatinem/rust-cache@v2
+        if: steps.cache-db-fixtures.outputs.cache-hit != 'true'
+        with:
+          key: ci-${{ github.job }}
+          shared-key: katana-ci-cache
+
+      - name: Checkout Dojo repository
+        if: steps.cache-db-fixtures.outputs.cache-hit != 'true'
+        uses: actions/checkout@v3
+        with:
+          repository: dojoengine/dojo
+          ref: v1.7.0
+          path: dojo
+
+      - name: Read scarb version from Dojo
+        if: steps.cache-db-fixtures.outputs.cache-hit != 'true'
+        id: dojo-scarb
+        run: |
+          SCARB_VERSION=$(grep '^scarb ' dojo/.tool-versions | awk '{print $2}')
+          echo "version=$SCARB_VERSION" >> $GITHUB_OUTPUT
+
+      - uses: software-mansion/setup-scarb@v1
+        if: steps.cache-db-fixtures.outputs.cache-hit != 'true'
+        with:
+          scarb-version: ${{ steps.dojo-scarb.outputs.version }}
+
+      - name: Install sozo
+        if: steps.cache-db-fixtures.outputs.cache-hit != 'true'
+        run: cargo install --path dojo/bin/sozo --locked --force
+
+      - name: Build generate_migration_db
+        if: steps.cache-db-fixtures.outputs.cache-hit != 'true'
+        run: cargo build --bin generate_migration_db --features node -p katana-utils
+
+      - name: Generate spawn-and-move fixture
+        if: steps.cache-db-fixtures.outputs.cache-hit != 'true'
+        run: ./target/debug/generate_migration_db --example spawn-and-move --output /tmp/spawn_and_move.tar.gz
+
+      - name: Generate simple fixture
+        if: steps.cache-db-fixtures.outputs.cache-hit != 'true'
+        run: ./target/debug/generate_migration_db --example simple --output /tmp/simple.tar.gz
+
+      - name: Extract fixtures
+        if: steps.cache-db-fixtures.outputs.cache-hit != 'true'
+        run: |
+          mkdir -p tests/fixtures/db
+          cd tests/fixtures/db && tar -xzf /tmp/spawn_and_move.tar.gz
+          cd tests/fixtures/db && tar -xzf /tmp/simple.tar.gz
+
+      - name: Upload DB fixtures
+        uses: actions/upload-artifact@v4
+        with:
+          name: db-fixtures
+          overwrite: true
+          retention-days: 14
+          if-no-files-found: error
+          path: |
+            ./tests/fixtures/db/spawn_and_move
+            ./tests/fixtures/db/simple
+
   test:
-    needs: [fmt, clippy, generate-test-artifacts, build-katana-binary]
+    needs: [fmt, clippy, generate-test-artifacts, build-katana-binary, generate-db-fixtures]
     runs-on: ubuntu-latest-32-cores
     timeout-minutes: 30
     if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.draft == false)
@@ -188,6 +278,11 @@ jobs:
         with:
           name: test-artifacts
 
+      - name: Download DB fixtures
+        uses: actions/download-artifact@v5
+        with:
+          name: db-fixtures
+
       - name: Download Katana binary
         uses: actions/download-artifact@v5
         with:
diff --git a/.gitignore b/.gitignore
@@ -31,9 +31,10 @@ tests/snos/snos
 snos-env
 tests/fixtures/katana_db
 
-# Ignore all files under tests/fixtures/db except .tar.gz files
+# Ignore generated DB fixture directories; keep only committed tarballs
 tests/fixtures/db/*
-!tests/fixtures/db/*.tar.gz
+!tests/fixtures/db/1_6_0.tar.gz
+!tests/fixtures/db/snos.tar.gz
 
 crates/contracts/build/
 !crates/contracts/build/legacy/
diff --git a/Makefile b/Makefile
@@ -17,10 +17,7 @@ SNOS_DB_DIR := $(DB_FIXTURES_DIR)/snos
 COMPATIBILITY_DB_TAR ?= $(DB_FIXTURES_DIR)/1_6_0.tar.gz
 COMPATIBILITY_DB_DIR ?= $(DB_FIXTURES_DIR)/1_6_0
 
-SPAWN_AND_MOVE_DB_TAR ?= $(DB_FIXTURES_DIR)/spawn_and_move.tar.gz
 SPAWN_AND_MOVE_DB_DIR := $(DB_FIXTURES_DIR)/spawn_and_move
-
-SIMPLE_DB_TAR ?= $(DB_FIXTURES_DIR)/simple.tar.gz
 SIMPLE_DB_DIR := $(DB_FIXTURES_DIR)/simple
 
 CONTRACTS_CRATE := crates/contracts
@@ -38,7 +35,7 @@ SCARB_VERSION := 2.8.4
 
 .DEFAULT_GOAL := usage
 .SILENT: clean
-.PHONY: usage help check-llvm native-deps native-deps-macos native-deps-linux native-deps-windows build-explorer contracts clean deps install-scarb test-artifacts snos-artifacts db-compat-artifacts install-pyenv
+.PHONY: usage help check-llvm native-deps native-deps-macos native-deps-linux native-deps-windows build-explorer contracts clean deps install-scarb test-artifacts snos-artifacts db-compat-artifacts generate-db-fixtures install-pyenv
 
 usage help:
 	@echo "Usage:"
@@ -49,6 +46,7 @@ usage help:
 	@echo "    test-artifacts:            Prepare tests artifacts (including test database)."
 	@echo "    snos-artifacts:            Prepare SNOS tests artifacts."
 	@echo "    db-compat-artifacts:       Prepare database compatibility test artifacts."
+	@echo "    generate-db-fixtures:      Generate spawn-and-move and simple DB fixtures (requires scarb + sozo)."
 	@echo "    native-deps-macos:         Install cairo-native dependencies for macOS."
 	@echo "    native-deps-linux:         Install cairo-native dependencies for Linux."
 	@echo "    native-deps-windows:       Install cairo-native dependencies for Windows."
@@ -74,7 +72,7 @@ snos-artifacts: $(SNOS_OUTPUT)
 db-compat-artifacts: $(COMPATIBILITY_DB_DIR)
 	@echo "Database compatibility test artifacts prepared successfully."
 
-test-artifacts: $(SNOS_DB_DIR) $(SNOS_OUTPUT) $(COMPATIBILITY_DB_DIR) $(SPAWN_AND_MOVE_DB_DIR) $(SIMPLE_DB_DIR) contracts
+test-artifacts: $(SNOS_DB_DIR) $(SNOS_OUTPUT) $(COMPATIBILITY_DB_DIR) contracts
 	@echo "All test artifacts prepared successfully."
 
 build-explorer:
@@ -128,17 +126,19 @@ $(COMPATIBILITY_DB_DIR): $(COMPATIBILITY_DB_TAR)
 		mv katana_db $(notdir $(COMPATIBILITY_DB_DIR)) || { echo "Failed to extract backward compatibility test database\!"; exit 1; }
 	@echo "Backward compatibility database extracted successfully."
 
-$(SPAWN_AND_MOVE_DB_DIR): $(SPAWN_AND_MOVE_DB_TAR)
-	@echo "Extracting spawn-and-move test database..."
-	@cd $(DB_FIXTURES_DIR) && \
-		tar -xzf $(notdir $(SPAWN_AND_MOVE_DB_TAR)) || { echo "Failed to extract spawn-and-move test database\!"; exit 1; }
-	@echo "Spawn-and-move test database extracted successfully."
-
-$(SIMPLE_DB_DIR): $(SIMPLE_DB_TAR)
-	@echo "Extracting simple test database..."
-	@cd $(DB_FIXTURES_DIR) && \
-		tar -xzf $(notdir $(SIMPLE_DB_TAR)) || { echo "Failed to extract simple test database\!"; exit 1; }
-	@echo "Simple test database extracted successfully."
+generate-db-fixtures:
+	@echo "Building generate_migration_db binary..."
+	cargo build --bin generate_migration_db --features node -p katana-utils
+	@echo "Generating spawn-and-move database fixture..."
+	./target/debug/generate_migration_db --example spawn-and-move --output /tmp/spawn_and_move.tar.gz
+	@echo "Generating simple database fixture..."
+	./target/debug/generate_migration_db --example simple --output /tmp/simple.tar.gz
+	@echo "Extracting spawn-and-move fixture..."
+	@mkdir -p $(DB_FIXTURES_DIR)
+	@cd $(DB_FIXTURES_DIR) && tar -xzf /tmp/spawn_and_move.tar.gz
+	@echo "Extracting simple fixture..."
+	@cd $(DB_FIXTURES_DIR) && tar -xzf /tmp/simple.tar.gz
+	@echo "DB fixtures generated successfully."
 
 check-llvm:
 ifndef MLIR_SYS_190_PREFIX
