Skip to content
This repository was archived by the owner on Mar 4, 2025. It is now read-only.

Commit d2326b5

Browse files
justincliftjustinclift
committed
database, common: Switch to using migrations instead of manual db changes
This is super overdue. ;) With this commit, we switch to using automatic database migrations instead of needing to manually apply database changes. To manage the migrations, we're using golang-migrate: https://github.com/golang-migrate/migrate New migrations are automatically applied by our daemons, whenever any of them connects to our PostgreSQL backend. New migrations should be created in the database/migrations folder.
1 parent 10b8b63 commit d2326b5

File tree

6 files changed

+140
-59
lines changed

6 files changed

+140
-59
lines changed

‎common/postgresql.go

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ import (
2323
"github.com/smtp2go-oss/smtp2go-go"
2424
gfm "github.com/sqlitebrowser/github_flavored_markdown"
2525
"golang.org/x/crypto/bcrypt"
26+
27+
"github.com/golang-migrate/migrate/v4"
28+
_ "github.com/golang-migrate/migrate/v4/database/pgx/v5"
29+
_ "github.com/golang-migrate/migrate/v4/source/file"
2630
)
2731

2832
var (
@@ -486,6 +490,19 @@ func ConnectPostgreSQL() (err error) {
486490
return fmt.Errorf("Couldn't connect to PostgreSQL server: %v", err)
487491
}
488492

493+
// Apply any outstanding database migrations
494+
m, err := migrate.New(fmt.Sprintf("file://%s/database/migrations", Conf.Web.BaseDir), "pgx5://dbhub@localhost:5432/dbhub")
495+
if err != nil {
496+
return
497+
}
498+
499+
// Bizarrely, migrate throws a "no change" error when there are no migrations to apply. So, we work around it:
500+
// https://github.com/golang-migrate/migrate/issues/485
501+
err = m.Up()
502+
if err != nil && !errors.Is(err, migrate.ErrNoChange) {
503+
return
504+
}
505+
489506
// Log successful connection
490507
log.Printf("Connected to PostgreSQL server: %v:%v", Conf.Pg.Server, uint16(Conf.Pg.Port))
491508
return nil

‎database/README.md

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
# Database schema for the DBHub applications
22

3+
[**NOTE - DO NOT update the schema file, as we're using migrations
4+
to make changes now**]
5+
36
This creates an empty DBHub.io database (no users, SQLite databases,
47
etc), ready for the DBHub.io applications to use.
58

6-
To load this into a fresh PostgreSQL 9.6 database run these commands
9+
To load this into a fresh PostgreSQL database, run these commands
710
from the postgres superuser:
811

912
$ createuser -d dbhub
@@ -12,6 +15,26 @@ from the postgres superuser:
1215

1316
It should finish with no errors.
1417

15-
Note - This schema is created using:
18+
Note - This schema was created using:
1619

1720
$ pg_dump -Os -U dbhub dbhub > dbhub.sql
21+
22+
## Migrations
23+
24+
To install the `migrate` cli locally on your system, use:
25+
26+
$ go install -tags 'pgx5' github.com/golang-migrate/migrate/v4/cmd/migrate@latest
27+
28+
This will build `migrate` with the driver we use (`pgx5`), then install
29+
it to your `$GOBIN` directory. If you don't have `GOBIN` defined, it'll
30+
go into `$HOME/go/bin/` instead.
31+
32+
## Manually running the migrations
33+
34+
In theory (!), you shouldn't need to apply the migrations manually as
35+
that's automatically done by our daemons when any of them start.
36+
37+
But, this is how to apply them in your local (DBHub.io) Docker container
38+
database if needed for some reason:
39+
40+
$ migrate -database pgx5://dbhub@localhost:5432/dbhub -path database/migrations up

‎database/migrations/000001_initial.down.sql

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
BEGIN;
2+
DROP TRIGGER track_applied_migrations ON schema_migrations;
3+
DROP FUNCTION track_applied_migration();
4+
DROP TABLE schema_migrations_history;
5+
COMMIT;

‎database/migrations/000001_initial.up.sql

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
BEGIN;
2+
3+
CREATE TABLE schema_migrations_history (
4+
id SERIAL PRIMARY KEY NOT NULL,
5+
version BIGINT NOT NULL,
6+
applied_at timestamptz NOT NULL DEFAULT NOW()
7+
);
8+
9+
CREATE OR REPLACE FUNCTION track_applied_migration()
10+
RETURNS TRIGGER AS $$
11+
DECLARE _current_version integer;
12+
BEGIN
13+
SELECT COALESCE(MAX(version),0) FROM schema_migrations_history INTO _current_version;
14+
IF new.dirty = 'f' AND new.version > _current_version THEN
15+
INSERT INTO schema_migrations_history(version) VALUES (new.version);
16+
END IF;
17+
RETURN NEW;
18+
END;
19+
$$ language 'plpgsql';
20+
21+
-- TRIGGER
22+
CREATE TRIGGER track_applied_migrations AFTER INSERT ON schema_migrations FOR EACH ROW EXECUTE PROCEDURE track_applied_migration();
23+
24+
COMMIT;

‎go.mod

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ require (
1717
github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b
1818
github.com/bradleypeabody/gorilla-sessions-memcache v0.0.0-20181103040241-659414f458e1
1919
github.com/go-playground/validator/v10 v10.3.0
20+
github.com/golang-migrate/migrate/v4 v4.15.3-0.20230407054901-84009cf2ab46
2021
github.com/gorilla/sessions v1.2.0
2122
github.com/gwenn/gosqlite v0.0.0-20200521090053-24878be1a237
2223
github.com/jackc/pgx/v5 v5.3.1
@@ -27,19 +28,22 @@ require (
2728
github.com/segmentio/ksuid v1.0.3
2829
github.com/smtp2go-oss/smtp2go-go v1.0.2
2930
github.com/sqlitebrowser/github_flavored_markdown v0.0.0-20190120045821-b8cf8f054e47
30-
golang.org/x/crypto v0.6.0
31-
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
31+
golang.org/x/crypto v0.7.0
32+
golang.org/x/oauth2 v0.1.0
3233
)
3334

3435
require (
3536
github.com/aymerick/douceur v0.2.0 // indirect
3637
github.com/go-ini/ini v1.56.0 // indirect
3738
github.com/go-playground/locales v0.13.0 // indirect
3839
github.com/go-playground/universal-translator v0.17.0 // indirect
39-
github.com/golang/protobuf v1.4.2 // indirect
40+
github.com/golang/protobuf v1.5.2 // indirect
4041
github.com/gorilla/css v1.0.0 // indirect
4142
github.com/gorilla/securecookie v1.1.1 // indirect
4243
github.com/gwenn/yacr v0.0.0-20200112083327-bbe82c1f4d60 // indirect
44+
github.com/hashicorp/errwrap v1.1.0 // indirect
45+
github.com/hashicorp/go-multierror v1.1.1 // indirect
46+
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa // indirect
4347
github.com/jackc/pgpassfile v1.0.0 // indirect
4448
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
4549
github.com/jackc/puddle/v2 v2.2.0 // indirect
@@ -56,15 +60,16 @@ require (
5660
github.com/shurcooL/highlight_go v0.0.0-20191220051317-782971ddf21b // indirect
5761
github.com/shurcooL/octicon v0.0.0-20191102190552-cbb32d6a785c // indirect
5862
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
59-
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect
63+
github.com/smartystreets/goconvey v1.8.0 // indirect
6064
github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d // indirect
6165
github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e // indirect
6266
github.com/sqlitebrowser/blackfriday v9.0.0+incompatible // indirect
63-
golang.org/x/net v0.7.0 // indirect
67+
go.uber.org/atomic v1.7.0 // indirect
68+
golang.org/x/net v0.8.0 // indirect
6469
golang.org/x/sync v0.1.0 // indirect
65-
golang.org/x/sys v0.5.0 // indirect
66-
golang.org/x/text v0.7.0 // indirect
67-
google.golang.org/appengine v1.6.6 // indirect
68-
google.golang.org/protobuf v1.23.0 // indirect
69-
gopkg.in/ini.v1 v1.46.0 // indirect
70+
golang.org/x/sys v0.6.0 // indirect
71+
golang.org/x/text v0.8.0 // indirect
72+
google.golang.org/appengine v1.6.7 // indirect
73+
google.golang.org/protobuf v1.28.1 // indirect
74+
gopkg.in/ini.v1 v1.67.0 // indirect
7075
)

0 commit comments

Comments
 (0)