From 3bd21e778a6eb0b0e69d3016c2db2aff0b2463f5 Mon Sep 17 00:00:00 2001 From: Lucas Kacher Date: Fri, 14 Mar 2025 17:13:28 -0700 Subject: [PATCH] feat: remove mongo --- Makefile | 2 +- README.md | 1 - database/mongodb/README.md | 24 - .../migrations/001_create_user.down.json | 5 - .../migrations/001_create_user.up.json | 12 - .../migrations/002_create_indexes.down.json | 10 - .../migrations/002_create_indexes.up.json | 21 - .../migrations/003_add_new_field.down.json | 16 - .../migrations/003_add_new_field.up.json | 16 - ...e_field_value_from_another_field.down.json | 14 - ...ace_field_value_from_another_field.up.json | 23 - database/mongodb/mongodb.go | 404 ---------------- database/mongodb/mongodb_test.go | 430 ------------------ go.mod | 6 - go.sum | 20 - internal/cli/build_mongodb.go | 8 - 16 files changed, 1 insertion(+), 1011 deletions(-) delete mode 100644 database/mongodb/README.md delete mode 100644 database/mongodb/examples/migrations/001_create_user.down.json delete mode 100644 database/mongodb/examples/migrations/001_create_user.up.json delete mode 100644 database/mongodb/examples/migrations/002_create_indexes.down.json delete mode 100644 database/mongodb/examples/migrations/002_create_indexes.up.json delete mode 100644 database/mongodb/examples/migrations/003_add_new_field.down.json delete mode 100644 database/mongodb/examples/migrations/003_add_new_field.up.json delete mode 100644 database/mongodb/examples/migrations/004_replace_field_value_from_another_field.down.json delete mode 100644 database/mongodb/examples/migrations/004_replace_field_value_from_another_field.up.json delete mode 100644 database/mongodb/mongodb.go delete mode 100644 database/mongodb/mongodb_test.go delete mode 100644 internal/cli/build_mongodb.go diff --git a/Makefile b/Makefile index 8e23a43c7..a8ed2d620 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ SOURCE ?= file go_bindata github github_ee bitbucket aws_s3 google_cloud_storage godoc_vfs gitlab -DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb yugabytedb clickhouse mongodb sqlserver firebird neo4j pgx pgx5 rqlite +DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb yugabytedb clickhouse sqlserver firebird neo4j pgx pgx5 rqlite DATABASE_TEST ?= $(DATABASE) sqlite sqlite3 sqlcipher VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-) TEST_FLAGS ?= diff --git a/README.md b/README.md index a79cc7b76..d86569ee0 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,6 @@ Database drivers run migrations. [Add a new database?](database/driver.go) * [SQLCipher](database/sqlcipher) * [MySQL / MariaDB](database/mysql) * [Neo4j](database/neo4j) -* [MongoDB](database/mongodb) * [CrateDB](database/crate) ([todo #170](https://github.com/mattes/migrate/issues/170)) * [Shell](database/shell) ([todo #171](https://github.com/mattes/migrate/issues/171)) * [Google Cloud Spanner](database/spanner) diff --git a/database/mongodb/README.md b/database/mongodb/README.md deleted file mode 100644 index bbebe02cd..000000000 --- a/database/mongodb/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# MongoDB - -* Driver work with mongo through [db.runCommands](https://docs.mongodb.com/manual/reference/command/) -* Migrations support json format. It contains array of commands for `db.runCommand`. Every command is executed in separate request to database -* All keys have to be in quotes `"` -* [Examples](./examples) - -# Usage - -`mongodb://user:password@host:port/dbname?query` (`mongodb+srv://` also works, but behaves a bit differently. See [docs](https://docs.mongodb.com/manual/reference/connection-string/#dns-seedlist-connection-format) for more information) - -| URL Query | WithInstance Config | Description | -|------------|---------------------|-------------| -| `x-migrations-collection` | `MigrationsCollection` | Name of the migrations collection | -| `x-transaction-mode` | `TransactionMode` | If set to `true` wrap commands in [transaction](https://docs.mongodb.com/manual/core/transactions). Available only for replica set. Driver is using [strconv.ParseBool](https://golang.org/pkg/strconv/#ParseBool) for parsing| -| `x-advisory-locking` | `true` | Feature flag for advisory locking, if set to false, disable advisory locking | -| `x-advisory-lock-collection` | `migrate_advisory_lock` | The name of the collection to use for advisory locking.| -| `x-advisory-lock-timeout` | `15` | The max time in seconds that migrate will wait to acquire a lock before failing. | -| `x-advisory-lock-timeout-interval` | `10` | The max time in seconds between attempts to acquire the advisory lock, the lock is attempted to be acquired using an exponential backoff algorithm. | -| `dbname` | `DatabaseName` | The name of the database to connect to | -| `user` | | The user to sign in as. Can be omitted | -| `password` | | The user's password. Can be omitted | -| `host` | | The host to connect to | -| `port` | | The port to bind to | \ No newline at end of file diff --git a/database/mongodb/examples/migrations/001_create_user.down.json b/database/mongodb/examples/migrations/001_create_user.down.json deleted file mode 100644 index 6bba284e5..000000000 --- a/database/mongodb/examples/migrations/001_create_user.down.json +++ /dev/null @@ -1,5 +0,0 @@ -[ - { - "dropUser": "deminem" - } -] \ No newline at end of file diff --git a/database/mongodb/examples/migrations/001_create_user.up.json b/database/mongodb/examples/migrations/001_create_user.up.json deleted file mode 100644 index 6c37cb702..000000000 --- a/database/mongodb/examples/migrations/001_create_user.up.json +++ /dev/null @@ -1,12 +0,0 @@ -[ - { - "createUser": "deminem", - "pwd": "gogo", - "roles": [ - { - "role": "readWrite", - "db": "testMigration" - } - ] - } -] \ No newline at end of file diff --git a/database/mongodb/examples/migrations/002_create_indexes.down.json b/database/mongodb/examples/migrations/002_create_indexes.down.json deleted file mode 100644 index 6bba481a6..000000000 --- a/database/mongodb/examples/migrations/002_create_indexes.down.json +++ /dev/null @@ -1,10 +0,0 @@ -[ - { - "dropIndexes": "mycollection", - "index": "username_sort_by_asc_created" - }, - { - "dropIndexes": "mycollection", - "index": "unique_email" - } -] \ No newline at end of file diff --git a/database/mongodb/examples/migrations/002_create_indexes.up.json b/database/mongodb/examples/migrations/002_create_indexes.up.json deleted file mode 100644 index e2995a20f..000000000 --- a/database/mongodb/examples/migrations/002_create_indexes.up.json +++ /dev/null @@ -1,21 +0,0 @@ -[{ - "createIndexes": "mycollection", - "indexes": [ - { - "key": { - "username": 1, - "created": -1 - }, - "name": "username_sort_by_asc_created", - "background": true - }, - { - "key": { - "email": 1 - }, - "name": "unique_email", - "unique": true, - "background": true - } - ] -}] \ No newline at end of file diff --git a/database/mongodb/examples/migrations/003_add_new_field.down.json b/database/mongodb/examples/migrations/003_add_new_field.down.json deleted file mode 100644 index 506c863cd..000000000 --- a/database/mongodb/examples/migrations/003_add_new_field.down.json +++ /dev/null @@ -1,16 +0,0 @@ -[ - { - "update": "users", - "updates": [ - { - "q": {}, - "u": { - "$unset": { - "status": "" - } - }, - "multi": true - } - ] - } -] diff --git a/database/mongodb/examples/migrations/003_add_new_field.up.json b/database/mongodb/examples/migrations/003_add_new_field.up.json deleted file mode 100644 index 6f53995e3..000000000 --- a/database/mongodb/examples/migrations/003_add_new_field.up.json +++ /dev/null @@ -1,16 +0,0 @@ -[ - { - "update": "users", - "updates": [ - { - "q": {}, - "u": { - "$set": { - "status": "active" - } - }, - "multi": true - } - ] - } -] diff --git a/database/mongodb/examples/migrations/004_replace_field_value_from_another_field.down.json b/database/mongodb/examples/migrations/004_replace_field_value_from_another_field.down.json deleted file mode 100644 index 0d2e65ebe..000000000 --- a/database/mongodb/examples/migrations/004_replace_field_value_from_another_field.down.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "update": "users", - "updates": [ - { - "q": {}, - "u": { - "fullname": "" - }, - "multi": true - } - ] - } -] diff --git a/database/mongodb/examples/migrations/004_replace_field_value_from_another_field.up.json b/database/mongodb/examples/migrations/004_replace_field_value_from_another_field.up.json deleted file mode 100644 index 1805e2688..000000000 --- a/database/mongodb/examples/migrations/004_replace_field_value_from_another_field.up.json +++ /dev/null @@ -1,23 +0,0 @@ -[ - { - "aggregate": "users", - "pipeline": [ - { - "$project": { - "_id": 1, - "firstname": 1, - "lastname": 1, - "username": 1, - "password": 1, - "email": 1, - "active": 1, - "fullname": { "$concat": ["$firstname", " ", "$lastname"] } - } - }, - { - "$out": "users" - } - ], - "cursor": {} - } -] diff --git a/database/mongodb/mongodb.go b/database/mongodb/mongodb.go deleted file mode 100644 index 3a9a6be9e..000000000 --- a/database/mongodb/mongodb.go +++ /dev/null @@ -1,404 +0,0 @@ -package mongodb - -import ( - "context" - "fmt" - "io" - "net/url" - "os" - "strconv" - "time" - - "github.com/cenkalti/backoff/v4" - "github.com/golang-migrate/migrate/v4/database" - "github.com/hashicorp/go-multierror" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - "go.mongodb.org/mongo-driver/x/mongo/driver/connstring" - "go.uber.org/atomic" -) - -func init() { - db := Mongo{} - database.Register("mongodb", &db) - database.Register("mongodb+srv", &db) -} - -var DefaultMigrationsCollection = "schema_migrations" - -const DefaultLockingCollection = "migrate_advisory_lock" // the collection to use for advisory locking by default. -const lockKeyUniqueValue = 0 // the unique value to lock on. If multiple clients try to insert the same key, it will fail (locked). -const DefaultLockTimeout = 15 // the default maximum time to wait for a lock to be released. -const DefaultLockTimeoutInterval = 10 // the default maximum intervals time for the locking timout. -const DefaultAdvisoryLockingFlag = true // the default value for the advisory locking feature flag. Default is true. -const LockIndexName = "lock_unique_key" // the name of the index which adds unique constraint to the locking_key field. -const contextWaitTimeout = 5 * time.Second // how long to wait for the request to mongo to block/wait for. - -var ( - ErrNoDatabaseName = fmt.Errorf("no database name") - ErrNilConfig = fmt.Errorf("no config") - ErrLockTimeoutConfigConflict = fmt.Errorf("both x-advisory-lock-timeout-interval and x-advisory-lock-timout-interval were specified") -) - -type Mongo struct { - client *mongo.Client - db *mongo.Database - config *Config - isLocked atomic.Bool -} - -type Locking struct { - CollectionName string - Timeout int - Enabled bool - Interval int -} -type Config struct { - DatabaseName string - MigrationsCollection string - TransactionMode bool - Locking Locking -} -type versionInfo struct { - Version int `bson:"version"` - Dirty bool `bson:"dirty"` -} - -type lockObj struct { - Key int `bson:"locking_key"` - Pid int `bson:"pid"` - Hostname string `bson:"hostname"` - CreatedAt time.Time `bson:"created_at"` -} -type findFilter struct { - Key int `bson:"locking_key"` -} - -func WithInstance(instance *mongo.Client, config *Config) (database.Driver, error) { - if config == nil { - return nil, ErrNilConfig - } - if len(config.DatabaseName) == 0 { - return nil, ErrNoDatabaseName - } - if len(config.MigrationsCollection) == 0 { - config.MigrationsCollection = DefaultMigrationsCollection - } - if len(config.Locking.CollectionName) == 0 { - config.Locking.CollectionName = DefaultLockingCollection - } - if config.Locking.Timeout <= 0 { - config.Locking.Timeout = DefaultLockTimeout - } - if config.Locking.Interval <= 0 { - config.Locking.Interval = DefaultLockTimeoutInterval - } - - mc := &Mongo{ - client: instance, - db: instance.Database(config.DatabaseName), - config: config, - } - - if mc.config.Locking.Enabled { - if err := mc.ensureLockTable(); err != nil { - return nil, err - } - } - if err := mc.ensureVersionTable(); err != nil { - return nil, err - } - - return mc, nil -} - -func (m *Mongo) Open(dsn string) (database.Driver, error) { - // connstring is experimental package, but it used for parse connection string in mongo.Connect function - uri, err := connstring.Parse(dsn) - if err != nil { - return nil, err - } - if len(uri.Database) == 0 { - return nil, ErrNoDatabaseName - } - unknown := url.Values(uri.UnknownOptions) - - migrationsCollection := unknown.Get("x-migrations-collection") - lockCollection := unknown.Get("x-advisory-lock-collection") - transactionMode, err := parseBoolean(unknown.Get("x-transaction-mode"), false) - if err != nil { - return nil, err - } - advisoryLockingFlag, err := parseBoolean(unknown.Get("x-advisory-locking"), DefaultAdvisoryLockingFlag) - if err != nil { - return nil, err - } - lockingTimout, err := parseInt(unknown.Get("x-advisory-lock-timeout"), DefaultLockTimeout) - if err != nil { - return nil, err - } - - lockTimeoutIntervalValue := unknown.Get("x-advisory-lock-timeout-interval") - // The initial release had a typo for this argument but for backwards compatibility sake, we will keep supporting it - // and we will error out if both values are set. - lockTimeoutIntervalValueFromTypo := unknown.Get("x-advisory-lock-timout-interval") - - lockTimeout := lockTimeoutIntervalValue - - if lockTimeoutIntervalValue != "" && lockTimeoutIntervalValueFromTypo != "" { - return nil, ErrLockTimeoutConfigConflict - } else if lockTimeoutIntervalValueFromTypo != "" { - lockTimeout = lockTimeoutIntervalValueFromTypo - } - - maxLockCheckInterval, err := parseInt(lockTimeout, DefaultLockTimeoutInterval) - - if err != nil { - return nil, err - } - client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(dsn)) - if err != nil { - return nil, err - } - - if err = client.Ping(context.TODO(), nil); err != nil { - return nil, err - } - mc, err := WithInstance(client, &Config{ - DatabaseName: uri.Database, - MigrationsCollection: migrationsCollection, - TransactionMode: transactionMode, - Locking: Locking{ - CollectionName: lockCollection, - Timeout: lockingTimout, - Enabled: advisoryLockingFlag, - Interval: maxLockCheckInterval, - }, - }) - if err != nil { - return nil, err - } - return mc, nil -} - -// Parse the url param, convert it to boolean -// returns error if param invalid. returns defaultValue if param not present -func parseBoolean(urlParam string, defaultValue bool) (bool, error) { - - // if parameter passed, parse it (otherwise return default value) - if urlParam != "" { - result, err := strconv.ParseBool(urlParam) - if err != nil { - return false, err - } - return result, nil - } - - // if no url Param passed, return default value - return defaultValue, nil -} - -// Parse the url param, convert it to int -// returns error if param invalid. returns defaultValue if param not present -func parseInt(urlParam string, defaultValue int) (int, error) { - - // if parameter passed, parse it (otherwise return default value) - if urlParam != "" { - result, err := strconv.Atoi(urlParam) - if err != nil { - return -1, err - } - return result, nil - } - - // if no url Param passed, return default value - return defaultValue, nil -} -func (m *Mongo) SetVersion(version int, dirty bool) error { - migrationsCollection := m.db.Collection(m.config.MigrationsCollection) - if err := migrationsCollection.Drop(context.TODO()); err != nil { - return &database.Error{OrigErr: err, Err: "drop migrations collection failed"} - } - _, err := migrationsCollection.InsertOne(context.TODO(), bson.M{"version": version, "dirty": dirty}) - if err != nil { - return &database.Error{OrigErr: err, Err: "save version failed"} - } - return nil -} - -func (m *Mongo) Version() (version int, dirty bool, err error) { - var versionInfo versionInfo - err = m.db.Collection(m.config.MigrationsCollection).FindOne(context.TODO(), bson.M{}).Decode(&versionInfo) - switch { - case err == mongo.ErrNoDocuments: - return database.NilVersion, false, nil - case err != nil: - return 0, false, &database.Error{OrigErr: err, Err: "failed to get migration version"} - default: - return versionInfo.Version, versionInfo.Dirty, nil - } -} - -func (m *Mongo) Run(migration io.Reader) error { - migr, err := io.ReadAll(migration) - if err != nil { - return err - } - var cmds []bson.D - err = bson.UnmarshalExtJSON(migr, true, &cmds) - if err != nil { - return fmt.Errorf("unmarshaling json error: %s", err) - } - if m.config.TransactionMode { - if err := m.executeCommandsWithTransaction(context.TODO(), cmds); err != nil { - return err - } - } else { - if err := m.executeCommands(context.TODO(), cmds); err != nil { - return err - } - } - return nil -} - -func (m *Mongo) executeCommandsWithTransaction(ctx context.Context, cmds []bson.D) error { - err := m.db.Client().UseSession(ctx, func(sessionContext mongo.SessionContext) error { - if err := sessionContext.StartTransaction(); err != nil { - return &database.Error{OrigErr: err, Err: "failed to start transaction"} - } - if err := m.executeCommands(sessionContext, cmds); err != nil { - // When command execution is failed, it's aborting transaction - // If you tried to call abortTransaction, it`s return error that transaction already aborted - return err - } - if err := sessionContext.CommitTransaction(sessionContext); err != nil { - return &database.Error{OrigErr: err, Err: "failed to commit transaction"} - } - return nil - }) - if err != nil { - return err - } - return nil -} - -func (m *Mongo) executeCommands(ctx context.Context, cmds []bson.D) error { - for _, cmd := range cmds { - err := m.db.RunCommand(ctx, cmd).Err() - if err != nil { - return &database.Error{OrigErr: err, Err: fmt.Sprintf("failed to execute command:%v", cmd)} - } - } - return nil -} - -func (m *Mongo) Close() error { - return m.client.Disconnect(context.TODO()) -} - -func (m *Mongo) Drop() error { - return m.db.Drop(context.TODO()) -} - -func (m *Mongo) ensureLockTable() error { - indexes := m.db.Collection(m.config.Locking.CollectionName).Indexes() - - indexOptions := options.Index().SetUnique(true).SetName(LockIndexName) - _, err := indexes.CreateOne(context.TODO(), mongo.IndexModel{ - Options: indexOptions, - Keys: findFilter{Key: -1}, - }) - if err != nil { - return err - } - return nil -} - -// ensureVersionTable checks if versions table exists and, if not, creates it. -// Note that this function locks the database, which deviates from the usual -// convention of "caller locks" in the MongoDb type. -func (m *Mongo) ensureVersionTable() (err error) { - if err = m.Lock(); err != nil { - return err - } - - defer func() { - if e := m.Unlock(); e != nil { - if err == nil { - err = e - } else { - err = multierror.Append(err, e) - } - } - }() - - if err != nil { - return err - } - if _, _, err = m.Version(); err != nil { - return err - } - return nil -} - -// Utilizes advisory locking on the config.LockingCollection collection -// This uses a unique index on the `locking_key` field. -func (m *Mongo) Lock() error { - return database.CasRestoreOnErr(&m.isLocked, false, true, database.ErrLocked, func() error { - if !m.config.Locking.Enabled { - return nil - } - - pid := os.Getpid() - hostname, err := os.Hostname() - if err != nil { - hostname = fmt.Sprintf("Could not determine hostname. Error: %s", err.Error()) - } - - newLockObj := lockObj{ - Key: lockKeyUniqueValue, - Pid: pid, - Hostname: hostname, - CreatedAt: time.Now(), - } - operation := func() error { - timeout, cancelFunc := context.WithTimeout(context.Background(), contextWaitTimeout) - _, err := m.db.Collection(m.config.Locking.CollectionName).InsertOne(timeout, newLockObj) - defer cancelFunc() - return err - } - exponentialBackOff := backoff.NewExponentialBackOff() - duration := time.Duration(m.config.Locking.Timeout) * time.Second - exponentialBackOff.MaxElapsedTime = duration - exponentialBackOff.MaxInterval = time.Duration(m.config.Locking.Interval) * time.Second - - err = backoff.Retry(operation, exponentialBackOff) - if err != nil { - return database.ErrLocked - } - - return nil - }) -} - -func (m *Mongo) Unlock() error { - return database.CasRestoreOnErr(&m.isLocked, true, false, database.ErrNotLocked, func() error { - if !m.config.Locking.Enabled { - return nil - } - - filter := findFilter{ - Key: lockKeyUniqueValue, - } - - ctx, cancel := context.WithTimeout(context.Background(), contextWaitTimeout) - _, err := m.db.Collection(m.config.Locking.CollectionName).DeleteMany(ctx, filter) - defer cancel() - - if err != nil { - return err - } - return nil - }) -} diff --git a/database/mongodb/mongodb_test.go b/database/mongodb/mongodb_test.go deleted file mode 100644 index a08d094ea..000000000 --- a/database/mongodb/mongodb_test.go +++ /dev/null @@ -1,430 +0,0 @@ -package mongodb - -import ( - "bytes" - "context" - "fmt" - - "log" - - "github.com/golang-migrate/migrate/v4" - "io" - "os" - "strconv" - "testing" - "time" -) - -import ( - "github.com/dhui/dktest" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" -) - -import ( - dt "github.com/golang-migrate/migrate/v4/database/testing" - "github.com/golang-migrate/migrate/v4/dktesting" - _ "github.com/golang-migrate/migrate/v4/source/file" -) - -var ( - opts = dktest.Options{PortRequired: true, ReadyFunc: isReady} - // Supported versions: https://www.mongodb.com/support-policy - specs = []dktesting.ContainerSpec{ - {ImageName: "mongo:5.0", Options: opts}, - {ImageName: "mongo:6.0", Options: opts}, - {ImageName: "mongo:7.0", Options: opts}, - {ImageName: "mongo:8.0", Options: opts}, - } -) - -func mongoConnectionString(host, port string) string { - // there is connect option for excluding serverConnection algorithm - // it's let avoid errors with mongo replica set connection in docker container - return fmt.Sprintf("mongodb://%s:%s/testMigration?connect=direct", host, port) -} - -func isReady(ctx context.Context, c dktest.ContainerInfo) bool { - ip, port, err := c.FirstPort() - if err != nil { - return false - } - - client, err := mongo.Connect(ctx, options.Client().ApplyURI(mongoConnectionString(ip, port))) - if err != nil { - return false - } - defer func() { - if err := client.Disconnect(ctx); err != nil { - log.Println("close error:", err) - } - }() - - if err = client.Ping(ctx, nil); err != nil { - switch err { - case io.EOF: - return false - default: - log.Println(err) - } - return false - } - return true -} - -func Test(t *testing.T) { - t.Run("test", test) - t.Run("testMigrate", testMigrate) - t.Run("testWithAuth", testWithAuth) - t.Run("testLockWorks", testLockWorks) - - t.Cleanup(func() { - for _, spec := range specs { - t.Log("Cleaning up ", spec.ImageName) - if err := spec.Cleanup(); err != nil { - t.Error("Error removing ", spec.ImageName, "error:", err) - } - } - }) -} - -func test(t *testing.T) { - dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { - ip, port, err := c.FirstPort() - if err != nil { - t.Fatal(err) - } - - addr := mongoConnectionString(ip, port) - p := &Mongo{} - d, err := p.Open(addr) - if err != nil { - t.Fatal(err) - } - defer func() { - if err := d.Close(); err != nil { - t.Error(err) - } - }() - dt.TestNilVersion(t, d) - dt.TestLockAndUnlock(t, d) - dt.TestRun(t, d, bytes.NewReader([]byte(`[{"insert":"hello","documents":[{"wild":"world"}]}]`))) - dt.TestSetVersion(t, d) - dt.TestDrop(t, d) - }) -} - -func testMigrate(t *testing.T) { - dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { - ip, port, err := c.FirstPort() - if err != nil { - t.Fatal(err) - } - - addr := mongoConnectionString(ip, port) - p := &Mongo{} - d, err := p.Open(addr) - if err != nil { - t.Fatal(err) - } - defer func() { - if err := d.Close(); err != nil { - t.Error(err) - } - }() - m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "", d) - if err != nil { - t.Fatal(err) - } - dt.TestMigrate(t, m) - }) -} - -func testWithAuth(t *testing.T) { - dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { - ip, port, err := c.FirstPort() - if err != nil { - t.Fatal(err) - } - - addr := mongoConnectionString(ip, port) - p := &Mongo{} - d, err := p.Open(addr) - if err != nil { - t.Fatal(err) - } - defer func() { - if err := d.Close(); err != nil { - t.Error(err) - } - }() - createUserCMD := []byte(`[{"createUser":"deminem","pwd":"gogo","roles":[{"role":"readWrite","db":"testMigration"}]}]`) - err = d.Run(bytes.NewReader(createUserCMD)) - if err != nil { - t.Fatal(err) - } - testcases := []struct { - name string - connectUri string - isErrorExpected bool - }{ - {"right auth data", "mongodb://deminem:gogo@%s:%v/testMigration", false}, - {"wrong auth data", "mongodb://wrong:auth@%s:%v/testMigration", true}, - } - - for _, tcase := range testcases { - t.Run(tcase.name, func(t *testing.T) { - mc := &Mongo{} - d, err := mc.Open(fmt.Sprintf(tcase.connectUri, ip, port)) - if err == nil { - defer func() { - if err := d.Close(); err != nil { - t.Error(err) - } - }() - } - - switch { - case tcase.isErrorExpected && err == nil: - t.Fatalf("no error when expected") - case !tcase.isErrorExpected && err != nil: - t.Fatalf("unexpected error: %v", err) - } - }) - } - }) -} - -func testLockWorks(t *testing.T) { - dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { - ip, port, err := c.FirstPort() - if err != nil { - t.Fatal(err) - } - - addr := mongoConnectionString(ip, port) - p := &Mongo{} - d, err := p.Open(addr) - if err != nil { - t.Fatal(err) - } - defer func() { - if err := d.Close(); err != nil { - t.Error(err) - } - }() - - dt.TestRun(t, d, bytes.NewReader([]byte(`[{"insert":"hello","documents":[{"wild":"world"}]}]`))) - - mc := d.(*Mongo) - - err = mc.Lock() - if err != nil { - t.Fatal(err) - } - err = mc.Unlock() - if err != nil { - t.Fatal(err) - } - - err = mc.Lock() - if err != nil { - t.Fatal(err) - } - err = mc.Unlock() - if err != nil { - t.Fatal(err) - } - - // enable locking, - //try to hit a lock conflict - mc.config.Locking.Enabled = true - mc.config.Locking.Timeout = 1 - err = mc.Lock() - if err != nil { - t.Fatal(err) - } - err = mc.Lock() - if err == nil { - t.Fatal("should have failed, mongo should be locked already") - } - }) -} - -func TestTransaction(t *testing.T) { - transactionSpecs := []dktesting.ContainerSpec{ - {ImageName: "mongo:4", Options: dktest.Options{PortRequired: true, ReadyFunc: isReady, - Cmd: []string{"mongod", "--bind_ip_all", "--replSet", "rs0"}}}, - } - t.Cleanup(func() { - for _, spec := range transactionSpecs { - t.Log("Cleaning up ", spec.ImageName) - if err := spec.Cleanup(); err != nil { - t.Error("Error removing ", spec.ImageName, "error:", err) - } - } - }) - - dktesting.ParallelTest(t, transactionSpecs, func(t *testing.T, c dktest.ContainerInfo) { - ip, port, err := c.FirstPort() - if err != nil { - t.Fatal(err) - } - - client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(mongoConnectionString(ip, port))) - if err != nil { - t.Fatal(err) - } - err = client.Ping(context.TODO(), nil) - if err != nil { - t.Fatal(err) - } - //rs.initiate() - err = client.Database("admin").RunCommand(context.TODO(), bson.D{bson.E{Key: "replSetInitiate", Value: bson.D{}}}).Err() - if err != nil { - t.Fatal(err) - } - err = waitForReplicaInit(client) - if err != nil { - t.Fatal(err) - } - d, err := WithInstance(client, &Config{ - DatabaseName: "testMigration", - }) - if err != nil { - t.Fatal(err) - } - defer func() { - if err := d.Close(); err != nil { - t.Error(err) - } - }() - //We have to create collection - //transactions don't support operations with creating new dbs, collections - //Unique index need for checking transaction aborting - insertCMD := []byte(`[ - {"create":"hello"}, - {"createIndexes": "hello", - "indexes": [{ - "key": { - "wild": 1 - }, - "name": "unique_wild", - "unique": true, - "background": true - }] - }]`) - err = d.Run(bytes.NewReader(insertCMD)) - if err != nil { - t.Fatal(err) - } - testcases := []struct { - name string - cmds []byte - documentsCount int64 - isErrorExpected bool - }{ - { - name: "success transaction", - cmds: []byte(`[{"insert":"hello","documents":[ - {"wild":"world"}, - {"wild":"west"}, - {"wild":"natural"} - ] - }]`), - documentsCount: 3, - isErrorExpected: false, - }, - { - name: "failure transaction", - //transaction have to be failure - duplicate unique key wild:west - //none of the documents should be added - cmds: []byte(`[{"insert":"hello","documents":[{"wild":"flower"}]}, - {"insert":"hello","documents":[ - {"wild":"cat"}, - {"wild":"west"} - ] - }]`), - documentsCount: 3, - isErrorExpected: true, - }, - } - for _, tcase := range testcases { - t.Run(tcase.name, func(t *testing.T) { - client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(mongoConnectionString(ip, port))) - if err != nil { - t.Fatal(err) - } - err = client.Ping(context.TODO(), nil) - if err != nil { - t.Fatal(err) - } - d, err := WithInstance(client, &Config{ - DatabaseName: "testMigration", - TransactionMode: true, - }) - if err != nil { - t.Fatal(err) - } - defer func() { - if err := d.Close(); err != nil { - t.Error(err) - } - }() - runErr := d.Run(bytes.NewReader(tcase.cmds)) - if runErr != nil { - if !tcase.isErrorExpected { - t.Fatal(runErr) - } - } - documentsCount, err := client.Database("testMigration").Collection("hello").CountDocuments(context.TODO(), bson.M{}) - if err != nil { - t.Fatal(err) - } - if tcase.documentsCount != documentsCount { - t.Fatalf("expected %d and actual %d documents count not equal. run migration error:%s", tcase.documentsCount, documentsCount, runErr) - } - }) - } - }) -} - -type isMaster struct { - IsMaster bool `bson:"ismaster"` -} - -func waitForReplicaInit(client *mongo.Client) error { - ticker := time.NewTicker(time.Second * 1) - defer ticker.Stop() - timeout, err := strconv.Atoi(os.Getenv("MIGRATE_TEST_MONGO_REPLICA_SET_INIT_TIMEOUT")) - if err != nil { - timeout = 30 - } - timeoutTimer := time.NewTimer(time.Duration(timeout) * time.Second) - defer timeoutTimer.Stop() - for { - select { - case <-ticker.C: - var status isMaster - //Check that node is primary because - //during replica set initialization, the first node first becomes a secondary and then becomes the primary - //should consider that initialization is completed only after the node has become the primary - result := client.Database("admin").RunCommand(context.TODO(), bson.D{bson.E{Key: "isMaster", Value: 1}}) - r, err := result.DecodeBytes() - if err != nil { - return err - } - err = bson.Unmarshal(r, &status) - if err != nil { - return err - } - if status.IsMaster { - return nil - } - case <-timeoutTimer.C: - return fmt.Errorf("replica init timeout") - } - } - -} diff --git a/go.mod b/go.mod index f16841a7c..1381fbff8 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,6 @@ require ( github.com/snowflakedb/gosnowflake v1.6.19 github.com/stretchr/testify v1.9.0 github.com/xanzy/go-gitlab v0.15.0 - go.mongodb.org/mongo-driver v1.7.5 go.uber.org/atomic v1.7.0 golang.org/x/oauth2 v0.18.0 golang.org/x/tools v0.24.0 @@ -115,7 +114,6 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect github.com/gabriel-vasile/mimetype v1.4.1 // indirect - github.com/go-stack/stack v1.8.0 // indirect github.com/goccy/go-json v0.9.11 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -169,10 +167,6 @@ require ( github.com/rqlite/gorqlite v0.0.0-20230708021416-2acd02b70b79 github.com/shopspring/decimal v1.2.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect - github.com/xdg-go/pbkdf2 v1.0.0 // indirect - github.com/xdg-go/scram v1.1.1 // indirect - github.com/xdg-go/stringprep v1.0.3 // indirect - github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect github.com/zeebo/xxh3 v1.0.2 // indirect gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b // indirect go.opencensus.io v0.24.0 // indirect diff --git a/go.sum b/go.sum index 3796f207d..c2a8d7ac4 100644 --- a/go.sum +++ b/go.sum @@ -203,7 +203,6 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobuffalo/here v0.6.0 h1:hYrd0a6gDmWxBM4TnrGw8mQg24iSVoIkHEk7FodQcBI= @@ -255,7 +254,6 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -420,7 +418,6 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= @@ -484,7 +481,6 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= @@ -567,20 +563,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/xanzy/go-gitlab v0.15.0 h1:rWtwKTgEnXyNUGrOArN7yyc3THRkpYcKXIXia9abywQ= github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs= -github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= -github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E= -github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= -github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= -github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs= -github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -591,8 +575,6 @@ github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaD github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b h1:7gd+rd8P3bqcn/96gOZa3F5dpJr/vEiDQYlNb/y2uNs= gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE= -go.mongodb.org/mongo-driver v1.7.5 h1:ny3p0reEpgsR2cfA5cjgwFZg3Cv/ofFh/8jbhGtz9VI= -go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -636,7 +618,6 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -803,7 +784,6 @@ golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= diff --git a/internal/cli/build_mongodb.go b/internal/cli/build_mongodb.go deleted file mode 100644 index 6b9f8f231..000000000 --- a/internal/cli/build_mongodb.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:build mongodb -// +build mongodb - -package cli - -import ( - _ "github.com/golang-migrate/migrate/v4/database/mongodb" -)