Merge pull request #790 from oasisprotocol/ptrus/feature/e2e-regression-tests

feat: E2E regression tests

Author: ptrus

.github/workflows/ci-test.yaml

@@ -172,3 +172,49 @@ jobs:
 
       - name: Upload to codecov.io
         uses: codecov/codecov-action@v5.5.1
+
+  test-e2e-regression:
+    strategy:
+      matrix:
+        suite:
+          - mainnet_sapphire_2025_09
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_HOST: 127.0.0.1
+          POSTGRES_PORT: 5432
+          POSTGRES_DB: postgres
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+        ports:
+          - 5432:5432
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v5
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version: "1.25.x"
+
+      - name: Build Go
+        run: make oasis-web3-gateway
+
+      - name: Block access to external Oasis nodes
+        # External nodes should not be available in CI; we run everything from caches.
+        # Enforce unavailability to keep tests confidently reproducible, and to catch
+        # any bugs that might cause CI to accidentally use live external services.
+        run: |
+          sudo bash -c 'echo 0.0.0.0 grpc.oasis.io | tee -a /etc/hosts'
+
+      - name: Run e2e regression tests
+        run: make E2E_REGRESSION_SUITES=${{ matrix.suite }} test-e2e-regression
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: Actual web3 gateway responses (${{ matrix.suite }})
+          path: |
+            tests/e2e_regression/${{ matrix.suite }}/actual
+        if: failure()

.gitignore

@@ -25,3 +25,7 @@ benchmarks/benchmarks
 .vscode
 
 dist/
+
+# E2E regression test artifacts.
+tests/e2e_regression/*/pebble-cache/
+tests/e2e_regression/*/actual/

.golangci.yml

@@ -39,6 +39,7 @@ linters:
             - github.com/spf13/cobra
             - github.com/stretchr/testify
             - github.com/uptrace/bun
+            - github.com/cockroachdb/pebble
             - golang.org/x/crypto
             - google.golang.org/grpc
           deny:

Makefile

@@ -21,6 +21,79 @@ clean:
 test:
 	@$(GO) test ./...
 
+# E2E regression testing.
+E2E_REGRESSION_SUITES := mainnet_sapphire_2025_09
+
+compress-e2e-regression-caches:
+	for suite in $(E2E_REGRESSION_SUITES); do \
+		cache_dir="tests/e2e_regression/$$suite/pebble-cache"; \
+		archive_path="tests/e2e_regression/$$suite/pebble-cache.tzst"; \
+		echo "Compressing $$cache_dir to $$archive_path..."; \
+		tar -cf "$$archive_path" --exclude='*.backup' -C "$$(dirname $$cache_dir)" "$$(basename $$cache_dir)" \
+			--use-compress-program zstd; \
+	done
+
+clean-e2e-regression-caches:
+	for suite in $(E2E_REGRESSION_SUITES); do \
+		cache_dir="tests/e2e_regression/$$suite/pebble-cache"; \
+		archive_path="tests/e2e_regression/$$suite/pebble-cache.tzst"; \
+		echo "Removing $$cache_dir and $$archive_path..."; \
+		rm -rf "$$cache_dir" "$$archive_path"; \
+	done
+
+ensure-e2e-regression-caches:
+	for suite in $(E2E_REGRESSION_SUITES); do \
+		cache_path="tests/e2e_regression/$$suite/pebble-cache"; \
+		archive_path="tests/e2e_regression/$$suite/pebble-cache.tzst"; \
+		if [ "$$FORCE" = "1" ]; then \
+			echo "Forcing extraction of $$cache_path..."; \
+			rm -rf "$$cache_path"; \
+		fi; \
+		if [ ! -d "$$cache_path" ]; then \
+			if [ -f "$$archive_path" ]; then \
+				echo "Extracting $$archive_path to $$cache_path..."; \
+				mkdir -p "$$(dirname $$cache_path)"; \
+				tar -xf "$$archive_path" -C "$$(dirname $$cache_path)" --use-compress-program unzstd; \
+			else \
+				echo "Archive $$archive_path not found, will run with empty cache."; \
+			fi; \
+		else \
+			echo "Cache already exists for $$suite — skipping."; \
+		fi; \
+	done
+
+# Run the e2e regression tests, assuming the environment is set up.
+test-e2e-regression: oasis-web3-gateway ensure-e2e-regression-caches
+	for suite in $(E2E_REGRESSION_SUITES); do ./tests/e2e_regression/run.sh $$suite; done
+
+# Accept the outputs of the e2e regression tests as the new expected outputs.
+accept-e2e-regression:
+	for suite in $(E2E_REGRESSION_SUITES); do ./tests/e2e_regression/accept.sh $$suite; done
+
+# Check actual vs expected results for e2e regression tests.
+check-e2e-regression:
+	for suite in $(E2E_REGRESSION_SUITES); do ./tests/e2e_regression/check.sh $$suite; done
+
+# Run dockerized postgres for local development.
+postgres:
+	@docker ps -a --format '{{.Names}}' | grep -q oasis-web3-gateway-postgres && docker start oasis-web3-gateway-postgres || \
+	docker run \
+		--name oasis-web3-gateway-postgres \
+		-p 5432:5432 \
+		-e POSTGRES_USER=postgres \
+		-e POSTGRES_PASSWORD=postgres \
+		-e POSTGRES_DB=postgres \
+		-d postgres -c log_statement=all
+	@sleep 3  # Allow time for postgres to start accepting connections.
+
+# Attach to the local DB from "make postgres".
+psql:
+	@docker exec -it oasis-web3-gateway-postgres psql -U postgres postgres
+
+# Stop and remove the postgres container.
+shutdown-postgres:
+	@docker rm oasis-web3-gateway-postgres --force
+
 # Format code.
 fmt:
 	@$(ECHO) "$(CYAN)*** Running Go formatters...$(OFF)"
@@ -64,6 +137,9 @@ docker-sapphire-localnet:
 	oasis-web3-gateway \
 	clean \
 	test \
+	compress-e2e-regression-caches clean-e2e-regression-caches ensure-e2e-regression-caches \
+	test-e2e-regression accept-e2e-regression check-e2e-regression \
+	postgres psql shutdown-postgres \
 	fmt \
 	$(lint-targets) lint \
 	release-build \

conf/config.go

@@ -16,15 +16,18 @@ import (
 
 // Config contains the CLI configuration.
 type Config struct {
-	RuntimeID     string `koanf:"runtime_id"`
-	NodeAddress   string `koanf:"node_address"`
+	RuntimeID string `koanf:"runtime_id"`
+
+	NodeAddress   string  `koanf:"node_address"`
+	NodeCachePath *string `koanf:"node_cache_path"`
+
 	EnablePruning bool   `koanf:"enable_pruning"`
 	PruningStep   uint64 `koanf:"pruning_step"`
 	// IndexingStart. Skip indexing before this block number. Use this to avoid trying to index
 	// blocks that the node doesn't have data for, such as by skipping them in checkpoint sync.
-	// For sensible reasons, indexing may actually start at an even later block, such as if
-	// this block is already indexed or the node indicates that it doesn't have this block.
-	IndexingStart   uint64 `koanf:"indexing_start"`
+	IndexingStart uint64 `koanf:"indexing_start"`
+	// IndexingEnd. Stop indexing after reaching this block number.
+	IndexingEnd     uint64 `koanf:"indexing_end"`
 	IndexingDisable bool   `koanf:"indexing_disable"`
 
 	Log      *LogConfig      `koanf:"log"`