add migration (#148)

Author: zookzook

.tool-versions

@@ -1,3 +1,3 @@
-elixir 1.13.2-otp-24
-erlang 24.2
+elixir 1.13.4-otp-25
+erlang 25.0.3
 python 3.10.2

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 0.9.3 (2022-10-14)
+* Bugfix
+  * fix a bug in the hello handshake protocol (thanks to fireproofsocks for reporting)
+* Enhancements
+  * add migration
+
 ## 0.9.2 (2022-09-24)
 * Bugfix
   * fix a crash in the streaming hello monitor, if the server sends more than one response at once 

README.md

@@ -462,11 +462,129 @@ URI, as follows:
 
 Using an SRV URI also discovers all nodes of the deployment automatically.
 
+## Migration
+
+Despite the schema-free approach, migration is still desirable. Migrations are used to maintain the indexes 
+and to drop collections that are no longer needed. Capped collections must be migrated. 
+The driver provides a workflow similar to Ecto that can be used to create migrations.
+
+First we create a migration script:
+```elixir
+
+mix mongo.gen.migration add_indexes
+
+```
+
+In `priv/mongo/migrations` you will find an Elixir script like `20220322173354_add_indexes.exs`:
+
+```elixr
+defmodule Mongo.Migrations.AddIndexes do
+  def up() do
+    indexes = [
+      [key: [email: 1], name: "email_index", unique: true]
+    ]
+
+    Mongo.create_indexes(:my_db, "my_collection", indexes)
+  end
+
+  def down() do
+    Mongo.drop_index(:my_db, "my_collection", "email_index")
+  end
+end
+
+```
+
+After that you can run the migration using a task:
+
+```
+mix mongo.migrate
+
+🔒 migrations locked
+⚡️ Successfully migrated Elixir.Mongo.Migrations.CreateIndex
+🔓 migrations unlocked
+
+```
+
+Or let it run if your application starts:
+
+```elixir
+defmodule MyApp.Release do
+  @moduledoc """
+  Used for executing DB release tasks when run in production without Mix
+  installed.
+  """
+
+  def migrate() do
+    Application.load(:my_app)
+    Application.ensure_all_started(:ssl)
+    Application.ensure_all_started(:mongodb_driver)
+    Mongo.start_link(name: :mongo_db, url: "mongodb://localhost:27017/my-database", timeout: 60_000, pool_size: 1, idle_interval: 10_000)
+
+    Mongo.Migration.migrate()
+  end
+end
+```
+
+With the release features of Elixir you can add an overlay script like this:
+
+```shell
+#!/bin/sh
+cd -P -- "$(dirname -- "$0")"
+exec ./my_app eval MyApp.Release.migrate
+```
+
+```shell
+#!/bin/sh
+cd -P -- "$(dirname -- "$0")"
+PHX_SERVER=true exec ./my_app start
+```
+
+And then you need just to call migrate before you start the server:
+
+```shell
+/app/bin/migrate && /app/bin/server
+```
+
+Or if you use a Dockerfile:
+
+```dockerfile
+ENTRYPOINT /app/bin/migrate && /app/bin/server
+```
+
+The migration module tries to *lock* the migration collection to ensure that only one instance is running the migration. 
+Unfortunately MongoDB does not support collection locks, so need to use a software lock:
+
+```elixir
+Mongo.update_one(topology, 
+  "migrations", 
+  %{_id: "lock", used: false}, 
+  %{"$set": %{used: true}}, 
+  upsert: true)
+```
+You can lock and unlock the migration collection using these functions in case of an error:
+
+1. `Mongo.Migration.lock()` 
+2. `Mongo.Migration.unlock()` or `mix mongo.unlock`
+
+If nothing helps, just delete the document with `{_id: "lock"}` from the migration collection.
+
+For more information see:
+
+- `Mongo.Migration`
+- `Mix.Tasks.Mongo`
+- https://hexdocs.pm/mix/1.14/Mix.Tasks.Release.html
+
 ## Auth Mechanisms
 
 For versions of Mongo 3.0 and greater, the auth mechanism defaults to SCRAM.
-If you'd like to use [MONGODB-X509](https://docs.mongodb.com/manual/tutorial/configure-x509-client-authentication/#authenticate-with-a-x-509-certificate)
-authentication, you can specify that as a `start_link` option.
+If you'd like to use [MONGODB-X509](https://www.mongodb.com/docs/v6.0/tutorial/configure-x509-client-authentication/)
+authentication, you can specify that as a `start_link` option. 
+
+You need roughly three additional configuration steps:
+
+* Deploy with x.509 Authentication
+* Add x.509 Certificate subject as a User
+* Authenticate with an x.509 Certificate
 
 ```elixir
 {:ok, pid} = Mongo.start_link(database: "test", auth_mechanism: :x509)

config/config.exs

@@ -15,6 +15,12 @@ config :logger, :console,
   metadata: [:module, :function, :line]
 
 config :mongodb_driver,
-  log: true
+  log: true,
+  migration: [
+    path: "mongo/migrations",
+    otp_app: :mongodb_driver,
+    topology: :mongo,
+    collection: "migrations"
+  ]
 
 import_config "#{Mix.env()}.exs"

config/test.exs

@@ -5,4 +5,10 @@ config :mongodb_driver, Mongo.RepoTest.MyRepo,
   show_sensitive_data_on_connection_error: true
 
 config :mongodb_driver,
-  log: false
+  log: false,
+  migration: [
+    path: "mongo/migrations",
+    otp_app: :mongodb_driver,
+    topology: :mongo,
+    collection: "migrations"
+  ]

lib/mongo/migration.ex

@@ -0,0 +1,133 @@
+defmodule Mongo.Migration do
+  @moduledoc false
+  use Mongo.Collection
+
+  def migrate() do
+    with :locked <- lock() do
+      migration_files!()
+      |> compile_migrations()
+      |> Enum.each(fn {mod, version} -> run_up(version, mod) end)
+
+      unlock()
+    end
+  rescue
+    _ ->
+      unlock()
+  end
+
+  def drop() do
+    with :locked <- lock() do
+      migration_files!()
+      |> compile_migrations()
+      |> Enum.reverse()
+      |> Enum.each(fn {mod, version} -> run_down(version, mod) end)
+
+      unlock()
+    end
+  rescue
+    _ ->
+      unlock()
+  end
+
+  def lock() do
+    topology = get_config()[:topology]
+    collection = get_config()[:collection]
+    query = %{_id: "lock", used: false}
+    set = %{"$set": %{used: true}}
+
+    case Mongo.update_one(topology, collection, query, set, upsert: true) do
+      {:ok, %{modified_count: 1}} ->
+        IO.puts("🔒 #{collection} locked")
+        :locked
+
+      {:ok, %{upserted_ids: ["lock"]}} ->
+        IO.puts("🔒 #{collection} locked")
+        :locked
+
+      _other ->
+        {:error, :already_locked}
+    end
+  end
+
+  def unlock() do
+    topology = get_config()[:topology]
+    collection = get_config()[:collection]
+    query = %{_id: "lock", used: true}
+    set = %{"$set": %{used: false}}
+
+    case Mongo.update_one(topology, collection, query, set) do
+      {:ok, %{modified_count: 1}} ->
+        IO.puts("🔓 #{collection} unlocked")
+        :unlocked
+
+      _other ->
+        {:error, :not_locked}
+    end
+  end
+
+  defp run_up(version, mod) do
+    topology = get_config()[:topology]
+    collection = get_config()[:collection]
+
+    case Mongo.find_one(topology, collection, %{version: version}) do
+      nil ->
+        mod.up()
+        Mongo.insert_one(topology, collection, %{version: version})
+        IO.puts("⚡️ Successfully migrated #{mod}")
+
+      _other ->
+        :noop
+    end
+  end
+
+  defp run_down(version, mod) do
+    topology = get_config()[:topology]
+    collection = get_config()[:collection]
+
+    case Mongo.find_one(topology, collection, %{version: version}) do
+      %{version: _version} ->
+        mod.down()
+        Mongo.delete_one(topology, collection, %{version: version})
+        IO.puts("💥 Successfully dropped #{mod}")
+
+      _other ->
+        :noop
+    end
+  end
+
+  def get_config() do
+    defaults = [topology: :mongo, collection: "migrations", path: "mongo/migrations", otp_app: :mongodb_driver]
+    Keyword.merge(defaults, Application.get_env(:mongodb_driver, :migration, []))
+  end
+
+  def migration_file_path() do
+    path = get_config()[:path]
+    otp_app = get_config()[:otp_app]
+    Path.join([:code.priv_dir(otp_app), path])
+  end
+
+  defp migration_files!() do
+    case File.ls(migration_file_path()) do
+      {:ok, files} -> files
+      {:error, _} -> raise "Could not find migrations file path"
+    end
+  end
+
+  defp compile_migrations(files) do
+    Enum.map(files, fn file ->
+      mod =
+        (migration_file_path() <> "/" <> file)
+        |> Code.compile_file()
+        |> Enum.map(&elem(&1, 0))
+        |> List.first()
+
+      version =
+        ~r/[0-9]/
+        |> Regex.scan(file)
+        |> Enum.join()
+        |> String.to_integer()
+
+      {mod, version}
+    end)
+  end
+end

lib/tasks/gen/migration.ex

@@ -0,0 +1,59 @@
+defmodule Mix.Tasks.Mongo.Gen.Migration do
+  @moduledoc false
+
+  use Mix.Task
+
+  import Macro, only: [camelize: 1, underscore: 1]
+  import Mix.Generator
+
+  @shortdoc "Generates a new migration for Mongo"
+
+  @spec run([String.t()]) :: integer()
+  def run(args) do
+    migrations_path = migration_file_path()
+    name = List.first(args)
+    base_name = "#{underscore(name)}.exs"
+    current_timestamp = timestamp()
+    file = Path.join(migrations_path, "#{current_timestamp}_#{base_name}")
+    unless File.dir?(migrations_path), do: create_directory(migrations_path)
+    fuzzy_path = Path.join(migrations_path, "*_#{base_name}")
+
+    if Path.wildcard(fuzzy_path) != [] do
+      Mix.raise("Migration can't be created, there is already a migration file with name #{name}.")
+    end
+
+    assigns = [mod: Module.concat([Mongo, Migrations, camelize(name)])]
+    create_file(file, migration_template(assigns))
+    String.to_integer(current_timestamp)
+  end
+
+  defp timestamp do
+    {{y, m, d}, {hh, mm, ss}} = :calendar.universal_time()
+    "#{y}#{pad(m)}#{pad(d)}#{pad(hh)}#{pad(mm)}#{pad(ss)}"
+  end
+
+  defp pad(i) when i < 10, do: <<?0, ?0 + i>>
+  defp pad(i), do: to_string(i)
+
+  def migration_file_path() do
+    path = Mongo.Migration.get_config()[:path]
+    Path.join(["priv", path])
+  end
+
+  embed_template(:migration, """
+  defmodule <%= inspect @mod %> do
+    def up() do
+      # The `up` functions will be executed when running `mix mongo.migrate`
+      #
+      # indexes = [[key: [files_id: 1, n: 1], name: "files_n_index", unique: true]]
+      # Mongo.create_indexes(<%= inspect(Mongo.Migration.get_config()[:topology]) %>, "my_collection", indexes)
+    end
+
+    def down() do
+      # The `down` functions will be executed when running `mix mongo.drop`
+      #
+      # Mongo.drop_collection(<%= inspect(Mongo.Migration.get_config()[:topology]) %>, "my_collection")
+    end
+  end
+  """)
+end