Add SQLite3 support ans meta table (#31)

This change adds support for SQLite3 storage of errors and includes a new `error_tracker_meta` table to store metadata needed for ErrorTracker to function properly - like managing migration versions.

Aside from adding support to SQLite3, which was as easy as expected, we have also added a way to track migration versions that should work in any SQL database (although only PostgreSQL and SQLite3 are currently supported). That KV store table will also work in the future to store runtime configuration, for example.

In this change we are also migrating PostgreSQL systems to use this new table, which means migrating from using table comments to store migration version to use this new table. 

As for SQLite3 migrations, migrations start at `V02` to maintain the same version IDs on all platforms.

Author: odarriba

.gitignore

@@ -29,3 +29,4 @@ error_tracker-*.tar
 
 
 dev.local.exs
+dev.db*

dev.exs

@@ -13,11 +13,18 @@ Logger.configure(level: :debug)
 Code.require_file("dev.local.exs")
 
 # Prepare the repo
+
+adapter =
+  case Application.get_env(:error_tracker, :ecto_adapter) do
+    :postgres -> Ecto.Adapters.Postgres
+    :sqlite3 -> Ecto.Adapters.SQLite3
+  end
+
 defmodule ErrorTrackerDev.Repo do
-  use Ecto.Repo, otp_app: :error_tracker, adapter: Ecto.Adapters.Postgres
+  use Ecto.Repo, otp_app: :error_tracker, adapter: adapter
 end
 
-_ = Ecto.Adapters.Postgres.storage_up(ErrorTrackerDev.Repo.config())
+_ = adapter.storage_up(ErrorTrackerDev.Repo.config())
 
 # Configures the endpoint
 Application.put_env(:error_tracker, ErrorTrackerDevWeb.Endpoint,

dev.local.example.exs

@@ -1,6 +1,22 @@
-# Prepare the Repo URL
+# PostgreSQL adapter
+#
+# To use SQLite3 on your local development machine uncomment these lines and
+# comment the lines of other adapters.
+
+Application.put_env(:error_tracker, :ecto_adapter, :postgres)
+
 Application.put_env(
   :error_tracker,
   ErrorTrackerDev.Repo,
   url: "ecto://postgres:[email protected]/error_tracker_dev"
 )
+
+# SQlite3 adapter
+#
+# To use SQLite3 on your local development machine uncomment these lines and
+# comment the lines of other adapters.
+
+# Application.put_env(:error_tracker, :ecto_adapter, :sqlite3)
+
+# sqlite_db = System.get_env("SQLITE_DB") || "dev.db"
+# Application.put_env(:error_tracker, ErrorTrackerDev.Repo, database: sqlite_db)

guides/Getting Started.md

@@ -4,7 +4,7 @@ This guide is an introduction to ErrorTracker, an Elixir-based built-in error tr
 
 In this guide we will learn how to install ErrorTracker in an Elixir project so you can start reporting errors as soon as possible. We will also cover more advanced topics such as how to report custom errors and how to add extra context to reported errors.
 
-**This guide requires you to have set up Ecto with PostgreSQL beforehand.**
+**This guide requires you to have set up Ecto with PostgreSQL or SQLite3 beforehand.**
 
 ## Installing the ErrorTracker as a dependency
 

lib/error_tracker.ex

@@ -16,7 +16,8 @@ defmodule ErrorTracker do
 
   ## Requirements
 
-  ErrorTracker requires Elixir 1.15+, Ecto 3.11+, Phoenix LiveView 0.19+, and PostgreSQL.
+  ErrorTracker requires Elixir 1.15+, Ecto 3.11+, Phoenix LiveView 0.19+, and
+  PostgreSQL or SQLite3 as database.
 
   ## Integrations
 

lib/error_tracker/migration.ex

@@ -55,7 +55,7 @@ defmodule ErrorTracker.Migration do
   mix ecto.migrate
   ```
 
-  ## Custom prefix
+  ## Custom prefix - PostgreSQL only
 
   ErrorTracker supports namespacing its own tables using PostgreSQL schemas, also known
   as "prefixes" in Ecto. With prefixes your error tables can reside outside of your primary
@@ -112,6 +112,7 @@ defmodule ErrorTracker.Migration do
   defp migrator do
     case ErrorTracker.Repo.__adapter__() do
       Ecto.Adapters.Postgres -> ErrorTracker.Migration.Postgres
+      Ecto.Adapters.SQLite3 -> ErrorTracker.Migration.SQLite
       adapter -> raise "ErrorTracker does not support #{adapter}"
     end
   end

lib/error_tracker/migration/postgres.ex

@@ -4,80 +4,28 @@ defmodule ErrorTracker.Migration.Postgres do
   @behaviour ErrorTracker.Migration
 
   use Ecto.Migration
-
-  import Ecto.Query
+  alias ErrorTracker.Migration.SQLMigrator
 
   @initial_version 1
-  @current_version 1
+  @current_version 2
   @default_prefix "public"
 
   @impl ErrorTracker.Migration
   def up(opts) do
     opts = with_defaults(opts, @current_version)
-    initial = current_version(opts)
-
-    cond do
-      initial == 0 ->
-        change(@initial_version..opts.version, :up, opts)
-
-      initial < opts.version ->
-        change((initial + 1)..opts.version, :up, opts)
-
-      true ->
-        :ok
-    end
+    SQLMigrator.migrate_up(__MODULE__, opts, @initial_version)
   end
 
   @impl ErrorTracker.Migration
   def down(opts) do
     opts = with_defaults(opts, @initial_version)
-    initial = max(current_version(opts), @initial_version)
-
-    if initial >= opts.version do
-      change(initial..opts.version, :down, opts)
-    end
+    SQLMigrator.migrate_down(__MODULE__, opts, @initial_version)
   end
 
   @impl ErrorTracker.Migration
   def current_version(opts) do
     opts = with_defaults(opts, @initial_version)
-    repo = Map.get_lazy(opts, :repo, fn -> repo() end)
-
-    query =
-      from pg_class in "pg_class",
-        left_join: pg_description in "pg_description",
-        on: pg_description.objoid == pg_class.oid,
-        left_join: pg_namespace in "pg_namespace",
-        on: pg_namespace.oid == pg_class.relnamespace,
-        where: pg_class.relname == "error_tracker_errors",
-        where: pg_namespace.nspname == ^opts.escaped_prefix,
-        select: pg_description.description
-
-    case repo.one(query, log: false) do
-      version when is_binary(version) -> String.to_integer(version)
-      _other -> 0
-    end
-  end
-
-  defp change(versions_range, direction, opts) do
-    for version <- versions_range do
-      padded_version = String.pad_leading(to_string(version), 2, "0")
-
-      migration_module = Module.concat(__MODULE__, "V#{padded_version}")
-      apply(migration_module, direction, [opts])
-    end
-
-    case direction do
-      :up -> record_version(opts, Enum.max(versions_range))
-      :down -> record_version(opts, Enum.min(versions_range) - 1)
-    end
-  end
-
-  defp record_version(%{prefix: prefix}, version) do
-    case version do
-      0 -> :ok
-      _other -> execute "COMMENT ON TABLE #{inspect(prefix)}.error_tracker_errors IS '#{version}'"
-    end
+    SQLMigrator.current_version(opts)
   end
 
   defp with_defaults(opts, version) do

lib/error_tracker/migration/postgres/v01.ex

@@ -3,45 +3,83 @@ defmodule ErrorTracker.Migration.Postgres.V01 do
 
   use Ecto.Migration
 
-  def up(%{create_schema: create_schema, prefix: prefix}) do
-    if create_schema, do: execute("CREATE SCHEMA IF NOT EXISTS #{prefix}")
-
-    create table(:error_tracker_errors,
-             primary_key: [name: :id, type: :bigserial],
-             prefix: prefix
-           ) do
-      add :kind, :string, null: false
-      add :reason, :text, null: false
-      add :source_line, :text, null: false
-      add :source_function, :text, null: false
-      add :status, :string, null: false
-      add :fingerprint, :string, null: false
-      add :last_occurrence_at, :utc_datetime_usec, null: false
-
-      timestamps(type: :utc_datetime_usec)
-    end
+  import Ecto.Query
 
-    create unique_index(:error_tracker_errors, [:fingerprint], prefix: prefix)
+  def up(opts = %{create_schema: create_schema, prefix: prefix}) do
+    # Prior to V02 the migration version was stored in table comments.
+    # As of now the migration version is stored in a new table (created in V02).
+    #
+    # However, systems migrating to V02 may think they need to run V01 too, so
+    # we need to check for the legacy version storage to avoid running this
+    # migration twice.
+    if current_version_legacy(opts) == 0 do
+      if create_schema, do: execute("CREATE SCHEMA IF NOT EXISTS #{prefix}")
 
-    create table(:error_tracker_occurrences,
-             primary_key: [name: :id, type: :bigserial],
-             prefix: prefix
-           ) do
-      add :context, :map, null: false
-      add :reason, :text, null: false
-      add :stacktrace, :map, null: false
+      create table(:error_tracker_meta,
+               primary_key: [name: :key, type: :string],
+               prefix: prefix
+             ) do
+        add :value, :string, null: false
+      end
 
-      add :error_id, references(:error_tracker_errors, on_delete: :delete_all, type: :bigserial),
-        null: false
+      create table(:error_tracker_errors,
+               primary_key: [name: :id, type: :bigserial],
+               prefix: prefix
+             ) do
+        add :kind, :string, null: false
+        add :reason, :text, null: false
+        add :source_line, :text, null: false
+        add :source_function, :text, null: false
+        add :status, :string, null: false
+        add :fingerprint, :string, null: false
+        add :last_occurrence_at, :utc_datetime_usec, null: false
 
-      timestamps(type: :utc_datetime_usec, updated_at: false)
-    end
+        timestamps(type: :utc_datetime_usec)
+      end
+
+      create unique_index(:error_tracker_errors, [:fingerprint], prefix: prefix)
+
+      create table(:error_tracker_occurrences,
+               primary_key: [name: :id, type: :bigserial],
+               prefix: prefix
+             ) do
+        add :context, :map, null: false
+        add :reason, :text, null: false
+        add :stacktrace, :map, null: false
+
+        add :error_id,
+            references(:error_tracker_errors, on_delete: :delete_all, type: :bigserial),
+            null: false
+
+        timestamps(type: :utc_datetime_usec, updated_at: false)
+      end
 
-    create index(:error_tracker_occurrences, [:error_id], prefix: prefix)
+      create index(:error_tracker_occurrences, [:error_id], prefix: prefix)
+    else
+      :noop
+    end
   end
 
   def down(%{prefix: prefix}) do
     drop table(:error_tracker_occurrences, prefix: prefix)
     drop table(:error_tracker_errors, prefix: prefix)
+    drop_if_exists table(:error_tracker_meta, prefix: prefix)
+  end
+
+  def current_version_legacy(opts) do
+    query =
+      from pg_class in "pg_class",
+        left_join: pg_description in "pg_description",
+        on: pg_description.objoid == pg_class.oid,
+        left_join: pg_namespace in "pg_namespace",
+        on: pg_namespace.oid == pg_class.relnamespace,
+        where: pg_class.relname == "error_tracker_errors",
+        where: pg_namespace.nspname == ^opts.escaped_prefix,
+        select: pg_description.description
+
+    case repo().one(query, log: false) do
+      version when is_binary(version) -> String.to_integer(version)
+      _other -> 0
+    end
   end
 end

lib/error_tracker/migration/postgres/v02.ex

@@ -0,0 +1,25 @@
+defmodule ErrorTracker.Migration.Postgres.V02 do
+  @moduledoc false
+
+  use Ecto.Migration
+
+  def up(%{prefix: prefix}) do
+    # For systems which executed versions without this migration they may not
+    # have the error_tracker_meta table, so we need to create it conditionally
+    # to avoid errors.
+    create_if_not_exists table(:error_tracker_meta,
+                           primary_key: [name: :key, type: :string],
+                           prefix: prefix
+                         ) do
+      add :value, :string, null: false
+    end
+
+    execute "COMMENT ON TABLE #{inspect(prefix)}.error_tracker_errors IS ''"
+  end
+
+  def down(%{prefix: prefix}) do
+    # We do not delete the `error_tracker_meta` table because it's creation and
+    # deletion are controlled by V01 migration.
+    execute "COMMENT ON TABLE #{inspect(prefix)}.error_tracker_errors IS '1'"
+  end
+end