enhancement(controlplane): versioned migrations support (#220)

Signed-off-by: Miguel Martinez Trivino <[email protected]>

Author: migmartri

.gitattributes

@@ -1,3 +1,6 @@
 app/controlplane/api/gen/frontend/** linguist-generated=true
 app/controlplane/internal/data/ent/** linguist-generated=true
+app/controlplane/internal/data/ent/migrate/** linguist-generated=false
+app/controlplane/internal/data/ent/migrate/** linguist-detectable=true
+app/controlplane/internal/data/ent/schema/* linguist-generated=false
 app/controlplane/internal/data/ent/schema/* linguist-detectable=true

.goreleaser.yml

@@ -65,6 +65,13 @@ dockers:
     image_templates:
       - "ghcr.io/chainloop-dev/chainloop/control-plane:{{ .Tag }}"
       - "ghcr.io/chainloop-dev/chainloop/control-plane:latest"
+  - dockerfile: app/controlplane/Dockerfile.migrations
+    ids:
+      - control-plane
+    extra_files:
+      - app/controlplane/internal/data/ent/migrate/migrations
+    image_templates:
+      - "ghcr.io/chainloop-dev/chainloop/control-plane-migrations:{{ .Tag }}"
   - dockerfile: app/artifact-cas/Dockerfile.goreleaser
     ids:
       - artifact-cas

Makefile

@@ -12,7 +12,7 @@ init:
 	go install github.com/go-kratos/kratos/cmd/protoc-gen-go-http/v2@latest
 	go install github.com/google/wire/cmd/wire@latest
 	go install github.com/vektra/mockery/[email protected]
-	go install entgo.io/ent/cmd/ent@v0.11.4
+	go install ariga.io/atlas/cmd/atlas@v0.12.0
 	go install github.com/bufbuild/buf/cmd/[email protected]
 
 .PHONY: api

app/controlplane/Dockerfile.migrations

@@ -0,0 +1,5 @@
+# Container image built by go-releaser that's used to run migrations against the database during deployment
+# See https://atlasgo.io/guides/deploying/image
+FROM arigaio/atlas@sha256:37b8b163719e2f9baf5c97099e8d0772bc1bd84f392e402afcc3e565d11e074f
+
+COPY app/controlplane/internal/data/ent/migrate/migrations /migrations

app/controlplane/Makefile

@@ -20,9 +20,23 @@ build:
 
 .PHONY: run
 # run
-run:
+run: migration_apply
 	go run ./cmd/... --conf ./configs
 
+local_migrations_dir = file://internal/data/ent/migrate/migrations
+local_db = postgres://postgres:@localhost:5432/controlplane?sslmode=disable
+
+.PHONY: migration_apply
+# run migrations against local db
+migration_apply:
+	atlas migrate status --dir ${local_migrations_dir} --url ${local_db}
+	atlas migrate apply --dir ${local_migrations_dir} --url ${local_db}
+
+.PHONY: migration_new
+# generate new migration if needed from the current ent schema
+migration_new:
+	atlas migrate diff --dir ${local_migrations_dir} --to "ent://internal/data/ent/schema" --dev-url "docker://postgres/15/test?search_path=public"
+
 .PHONY: test
 # All tests, both unit and integration
 test: 

app/controlplane/README.md

@@ -72,13 +72,18 @@ make api
 
 ### Update Application data model
 
-We use [ent](https://entgo.io) as database Object Relational Mapping (ORM).
+We use [ent](https://entgo.io) as database Object Relational Mapping (ORM) and [atlas versioned migrations](https://entgo.io/docs/versioned-migrations) to manage the database schema changes.
 
-The way a change in the data model is as follows
+The way you can make a change in the data model is
 
+**Update the schema**
 - Add a new/update an existing entity via a schema update. Schemas can be found at `internal/data/ent/schema`
 - Generate the code changes associated with that schema change. `make generate`
-- Restarting the control plane will cause the schema change to be automatically migrated.
+- Generate a new versioned migration `make migration_new`. This will create a new migration file at `internal/data/ent/migrate/migrations 
+
+**Apply the schema change in development DB**
+- `make migration_apply` will apply the migration to the development database
+- Restart the control plane
 
 ### Update configuration schema
 

app/controlplane/internal/biz/testhelpers/database.go

@@ -23,11 +23,12 @@ import (
 	"testing"
 	"time"
 
-	// Requuired for the database waitFor strategy
+	// Required for the database waitFor strategy
 	_ "github.com/lib/pq"
 
 	"github.com/chainloop-dev/chainloop/app/controlplane/internal/biz"
 	"github.com/chainloop-dev/chainloop/app/controlplane/internal/conf"
+	"github.com/chainloop-dev/chainloop/app/controlplane/internal/data"
 	"github.com/chainloop-dev/chainloop/app/controlplane/plugins/sdk/v1"
 	"github.com/chainloop-dev/chainloop/internal/credentials"
 	creds "github.com/chainloop-dev/chainloop/internal/credentials/mocks"
@@ -45,8 +46,9 @@ import (
 // NOTE: It connects to a real database
 type TestingUseCases struct {
 	// Misc
-	DB *TestDatabase
-	L  log.Logger
+	DB   *TestDatabase
+	Data *data.Data
+	L    log.Logger
 
 	// Use cases
 	Membership             *biz.MembershipUseCase
@@ -101,6 +103,9 @@ func NewTestingUseCases(t *testing.T, opts ...NewTestingUCOpt) *TestingUseCases
 	}, newArgs.integrations)
 	assert.NoError(t, err)
 
+	// Run DB migrations for testing
+	require.NoError(t, testData.Data.SchemaLoad())
+
 	return testData
 }
 

app/controlplane/internal/biz/testhelpers/wire_gen.go

@@ -25,7 +25,6 @@ import (
 
 	"entgo.io/ent/dialect"
 	entsql "entgo.io/ent/dialect/sql"
-	"entgo.io/ent/dialect/sql/schema"
 
 	"github.com/chainloop-dev/chainloop/app/controlplane/internal/conf"
 	"github.com/chainloop-dev/chainloop/app/controlplane/internal/data/ent"
@@ -35,7 +34,7 @@ import (
 	"github.com/google/wire"
 
 	// Load PGX driver
-	_ "github.com/jackc/pgx/v4/stdlib"
+	_ "github.com/jackc/pgx/v5/stdlib"
 )
 
 // ProviderSet is data providers.
@@ -59,6 +58,13 @@ type Data struct {
 	db *ent.Client
 }
 
+// Load DB schema
+// NOTE: this is different than running migrations
+// this method is used to load the schema into the DB for TESTING!
+func (data *Data) SchemaLoad() error {
+	return data.db.Schema.Create(context.Background())
+}
+
 // NewData .
 func NewData(c *conf.Data, logger log.Logger) (*Data, func(), error) {
 	if logger == nil {
@@ -96,11 +102,8 @@ func initSQLDatabase(c *conf.Data_Database, log *log.Helper) (*ent.Client, error
 	drv := entsql.OpenDB(dialect.Postgres, db)
 	client := ent.NewClient(ent.Driver(drv))
 
-	// Run DB migration
-	if err := client.Schema.Create(context.Background(), schema.WithDropColumn(true)); err != nil {
-		return nil, fmt.Errorf("error performing the schema change: %w", err)
-	}
-
+	// NOTE: We do not run migrations automatically anymore
+	// Instead we leverage atlas cli to run migrations
 	return client, nil
 }