Merge pull request #1130 from oasisprotocol/ptrus/feature/embed-migrations

Embed migrations in the Nexus binary

Author: ptrus

.changelog/799.feature.md

@@ -0,0 +1,4 @@
+Embed migrations in the Nexus binary
+
+Migrations are now embedded directly in the Nexus binary, simplifying
+deployment by eliminating the need to distribute migration files separately.

analyzer/block/block_test.go

@@ -23,10 +23,6 @@ import (
 	pgTestUtil "github.com/oasisprotocol/nexus/storage/postgres/testutil"
 )
 
-// Relative path to the migrations directory when running tests in this file.
-// When running go tests, the working directory is always set to the package directory of the test being run.
-const migrationsPath = "file://../../storage/migrations"
-
 const testsTimeout = 10 * time.Second
 
 // Default block based config used in slow-sync tests.
@@ -102,7 +98,7 @@ func setupDB(t *testing.T) *postgres.Client {
 	// Ensure the test database is empty.
 	require.NoError(t, testDB.Wipe(ctx), "testDb.Wipe")
 	// Run DB migrations.
-	require.NoError(t, analyzerCmd.RunMigrations(migrationsPath, os.Getenv("CI_TEST_CONN_STRING")), "failed to run migrations")
+	require.NoError(t, analyzerCmd.RunMigrations(os.Getenv("CI_TEST_CONN_STRING")), "failed to run migrations")
 
 	return testDB
 }

analyzer/item/item_test.go

@@ -21,10 +21,6 @@ import (
 	pgTestUtil "github.com/oasisprotocol/nexus/storage/postgres/testutil"
 )
 
-// Relative path to the migrations directory when running tests in this file.
-// When running go tests, the working directory is always set to the package directory of the test being run.
-const migrationsPath = "file://../../storage/migrations"
-
 const testsTimeout = 10 * time.Second
 
 const (
@@ -113,7 +109,7 @@ func setupDB(t *testing.T) *postgres.Client {
 	// Ensure the test database is empty.
 	require.NoError(t, testDB.Wipe(ctx), "testDb.Wipe")
 	// Run DB migrations.
-	require.NoError(t, analyzerCmd.RunMigrations(migrationsPath, os.Getenv("CI_TEST_CONN_STRING")), "failed to run migrations")
+	require.NoError(t, analyzerCmd.RunMigrations(os.Getenv("CI_TEST_CONN_STRING")), "failed to run migrations")
 	// Initialize test table.
 	batch := &storage.QueryBatch{}
 	batch.Queue(testTableCreate)

analyzer/runtime/runtime_test.go

@@ -36,10 +36,6 @@ import (
 	pgTestUtil "github.com/oasisprotocol/nexus/storage/postgres/testutil"
 )
 
-// Relative path to the migrations directory when running tests in this file.
-// When running go tests, the working directory is always set to the package directory of the test being run.
-const migrationsPath = "file://../../storage/migrations"
-
 const testsTimeout = 10 * time.Second
 
 func setupDB(t *testing.T) *postgres.Client {
@@ -50,7 +46,7 @@ func setupDB(t *testing.T) *postgres.Client {
 	// Ensure the test database is empty.
 	require.NoError(t, testDB.Wipe(ctx), "testDb.Wipe")
 	// Run DB migrations.
-	require.NoError(t, analyzerCmd.RunMigrations(migrationsPath, os.Getenv("CI_TEST_CONN_STRING")), "failed to run migrations")
+	require.NoError(t, analyzerCmd.RunMigrations(os.Getenv("CI_TEST_CONN_STRING")), "failed to run migrations")
 	// Initialize test table.
 	// batch := &storage.QueryBatch{}
 	// batch.Queue(testTableCreate)

cmd/analyzer/analyzer.go

@@ -13,6 +13,7 @@ import (
 	migratePgx "github.com/golang-migrate/migrate/v4/database/pgx/v5" // postgres pgx driver for golang_migrate
 	_ "github.com/golang-migrate/migrate/v4/source/file"              // support file scheme for golang_migrate
 	_ "github.com/golang-migrate/migrate/v4/source/github"            // support github scheme for golang_migrate
+	"github.com/golang-migrate/migrate/v4/source/iofs"
 	"github.com/jackc/pgx/v5/pgxpool"
 	pgxstd "github.com/jackc/pgx/v5/stdlib"
 
@@ -42,6 +43,7 @@ import (
 	"github.com/oasisprotocol/nexus/config"
 	"github.com/oasisprotocol/nexus/log"
 	"github.com/oasisprotocol/nexus/storage"
+	"github.com/oasisprotocol/nexus/storage/migrations"
 	source "github.com/oasisprotocol/nexus/storage/oasis"
 	"github.com/oasisprotocol/nexus/storage/oasis/nodeapi"
 )
@@ -50,8 +52,8 @@ const (
 	moduleName = "analysis_service"
 )
 
-// RunMigrations runs migrations defined in sourceURL against databaseURL.
-func RunMigrations(sourceURL string, databaseURL string) error {
+// RunMigrations runs embedded migrations against databaseURL.
+func RunMigrations(databaseURL string) error {
 	// Go-migrate supports only basic database URL parameters.
 	// We use pgx to parse the database URL the same way as we do in the indexer
 	// client. That way the config options for migrations and clients are compatible.
@@ -73,10 +75,16 @@ func RunMigrations(sourceURL string, databaseURL string) error {
 	}
 	defer driver.Close()
 
-	m, err := migrate.NewWithDatabaseInstance(sourceURL, config.ConnConfig.Database, driver)
+	sourceDriver, err := iofs.New(migrations.FS, ".")
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to create embedded migration source: %w", err)
 	}
+
+	m, err := migrate.NewWithInstance("iofs", sourceDriver, config.ConnConfig.Database, driver)
+	if err != nil {
+		return fmt.Errorf("failed to create migration instance: %w", err)
+	}
+
 	return m.Up()
 }
 
@@ -94,7 +102,7 @@ func Init(ctx context.Context, cfg *config.AnalysisConfig, logger *log.Logger) (
 	}
 
 	logger.Info("checking if migrations need to be applied...")
-	switch err := RunMigrations(cfg.Storage.Migrations, cfg.Storage.Endpoint); {
+	switch err := RunMigrations(cfg.Storage.Endpoint); {
 	case err == migrate.ErrNoChange:
 		logger.Info("no migrations needed to be applied")
 	case err != nil:

cmd/analyzer/analyzer_migrations_test.go

@@ -12,10 +12,6 @@ import (
 	"github.com/oasisprotocol/nexus/tests"
 )
 
-// Relative path to the migrations directory when running tests in this file.
-// When running go tests, the working directory is always set to the package directory of the test being run.
-const migrationsPath = "file://../../storage/migrations"
-
 func TestMigrations(t *testing.T) {
 	tests.SkipIfShort(t)
 	client := testutil.NewTestClient(t)
@@ -27,5 +23,5 @@ func TestMigrations(t *testing.T) {
 	require.NoError(t, client.Wipe(ctx), "failed to wipe database")
 
 	// Run migrations.
-	require.NoError(t, cmdAnalyzer.RunMigrations(migrationsPath, os.Getenv("CI_TEST_CONN_STRING")), "failed to run migrations")
+	require.NoError(t, cmdAnalyzer.RunMigrations(os.Getenv("CI_TEST_CONN_STRING")), "failed to run migrations")
 }

config/config.go

@@ -133,7 +133,7 @@ func (cfg *AnalysisConfig) Validate() error {
 		}
 	}
 
-	return cfg.Storage.Validate(true /* requireMigrations */)
+	return cfg.Storage.Validate()
 }
 
 type AnalyzersList struct {
@@ -679,7 +679,7 @@ func (cfg *ServerConfig) Validate() error {
 		}
 	}
 
-	return cfg.Storage.Validate(false /* requireMigrations */)
+	return cfg.Storage.Validate()
 }
 
 // StorageBackend is a storage backend.
@@ -737,22 +737,16 @@ type StorageConfig struct {
 	// Backend is the storage backend to select.
 	Backend string `koanf:"backend"`
 
-	// Migrations is the directory containing schema migrations.
-	Migrations string `koanf:"migrations"`
-
 	// If true, we'll first delete all tables in the DB to
 	// force a full re-index of the blockchain.
 	WipeStorage bool `koanf:"DANGER__WIPE_STORAGE_ON_STARTUP"`
 }
 
 // Validate validates the storage configuration.
-func (cfg *StorageConfig) Validate(requireMigrations bool) error {
+func (cfg *StorageConfig) Validate() error {
 	if cfg.Endpoint == "" {
 		return fmt.Errorf("malformed storage endpoint '%s'", cfg.Endpoint)
 	}
-	if cfg.Migrations == "" && requireMigrations {
-		return fmt.Errorf("invalid path to migrations '%s'", cfg.Migrations)
-	}
 	var sb StorageBackend
 	return sb.Set(cfg.Backend)
 }

config/config_test.go

@@ -16,9 +16,8 @@ func TestServerConfigJSON(t *testing.T) {
 	cfg := ServerConfig{
 		Endpoint: "localhost:8080",
 		Storage: &StorageConfig{
-			Endpoint:   "postgresql://user:pass@localhost:5432/db",
-			Backend:    "postgres",
-			Migrations: "/path/to/migrations",
+			Endpoint: "postgresql://user:pass@localhost:5432/db",
+			Backend:  "postgres",
 		},
 		Source: &SourceConfig{
 			ChainName: "mainnet",
@@ -46,7 +45,6 @@ server:
   storage:
     endpoint: postgresql://user:pass@localhost:5432/db
     backend: postgres
-    migrations: /path/to/migrations
     DANGER__WIPE_STORAGE_ON_STARTUP: false
   source:
     chain_name: mainnet

docker/nexus/Dockerfile

@@ -28,7 +28,6 @@ RUN apt-get update -q -q && \
   apt-get install --yes apt-transport-https ca-certificates
 
 COPY --from=nexus-builder /code/go/nexus /usr/local/bin/nexus
-COPY --from=nexus-builder /code/go/storage/migrations /storage/migrations/
 COPY --from=nexus-builder /code/go/ /nexus
 COPY --from=openapi-builder /api/spec/v1.html api/spec/v1.html
 

storage/migrations/migrations.go

@@ -0,0 +1,7 @@
+// Package migrations contains embedded database migration files.
+package migrations
+
+import "embed"
+
+//go:embed *.sql
+var FS embed.FS