diff --git a/lib/algora/admin/migration.ex b/lib/algora/admin/migration.ex new file mode 100644 index 000000000..a7fc72739 --- /dev/null +++ b/lib/algora/admin/migration.ex @@ -0,0 +1,130 @@ +defmodule Algora.Admin.Migration do + @moduledoc false + import Ecto.Query + + alias Algora.Accounts.User + alias Algora.Payments.Transaction + alias Algora.Repo + + @balances_file ".local/db/balance-2025-02-13.json" + + def get_actual_balances(type) do + with {:ok, content} <- File.read(@balances_file), + {:ok, data} <- Jason.decode(content) do + Enum.map(data[Atom.to_string(type)], fn %{"provider_login" => login, "balance" => balance} -> + %{ + provider_login: login, + balance: Money.new!(:USD, Decimal.new(balance)) + } + end) + else + error -> raise "Failed to load balances: #{inspect(error)}" + end + end + + def get_balances(type) do + get_balances() + |> Enum.filter(&(&1.type == type)) + |> Enum.reject(&(&1.provider_login in ["algora-io", "Uber4Coding"])) + end + + def get_balances do + user_txs = + from t in Transaction, + group_by: [t.user_id, t.type], + select: %{ + user_id: t.user_id, + type: t.type, + net_amount: sum(t.net_amount) + } + + user_balances = + from ut in subquery(user_txs), + group_by: ut.user_id, + select: %{ + user_id: ut.user_id, + balance: + sum( + fragment( + """ + CASE + WHEN ? = 'credit' THEN ? + WHEN ? = 'debit' THEN -? + WHEN ? = 'charge' THEN ? + WHEN ? = 'transfer' THEN -? + ELSE ('USD', 0)::money_with_currency + END + """, + ut.type, + ut.net_amount, + ut.type, + ut.net_amount, + ut.type, + ut.net_amount, + ut.type, + ut.net_amount + ) + ) + } + + query = + from ub in subquery(user_balances), + join: u in User, + on: u.id == ub.user_id, + where: ub.balance != fragment("('USD', 0)::money_with_currency"), + order_by: [desc: u.type, desc: ub.balance, asc: u.provider_login], + select: %{ + type: u.type, + provider_login: u.provider_login, + balance: ub.balance + } + + query + |> Repo.all() + |> Enum.map(fn + user -> + {currency, amount} = user.balance + %{user | balance: Money.new!(currency, amount)} + end) + end + + def diff_balances do + diff_balances(:individual) ++ diff_balances(:organization) + end + + def diff_balances(type) do + actual = Enum.map(get_actual_balances(type), &Map.take(&1, [:provider_login, :balance])) + current = Enum.map(get_balances(type), &Map.take(&1, [:provider_login, :balance])) + + actual_map = Map.new(actual, &{&1.provider_login, &1.balance}) + current_map = Map.new(current, &{&1.provider_login, &1.balance}) + + all_logins = + MapSet.union( + MapSet.new(Map.keys(actual_map)), + MapSet.new(Map.keys(current_map)) + ) + + differences = + Enum.reduce(all_logins, [], fn login, acc -> + actual_balance = Map.get(actual_map, login) + current_balance = Map.get(current_map, login) + + cond do + actual_balance == current_balance -> + acc + + actual_balance == nil -> + [{:extra_in_current, login, current_balance} | acc] + + current_balance == nil -> + [{:missing_in_current, login, actual_balance} | acc] + + true -> + [{:different, login, actual_balance, current_balance} | acc] + end + end) + + Enum.sort_by(differences, &elem(&1, 1)) + end +end diff --git a/lib/algora/bounties/schemas/bounty.ex b/lib/algora/bounties/schemas/bounty.ex index 97346f085..5de8e5bdf 100644 --- a/lib/algora/bounties/schemas/bounty.ex +++ b/lib/algora/bounties/schemas/bounty.ex @@ -9,6 +9,7 @@ defmodule Algora.Bounties.Bounty do typed_schema "bounties" do field :amount, Algora.Types.Money field :status, Ecto.Enum, values: [:open, :cancelled, :paid] + field :number, :integer, default: 0 belongs_to :ticket, Algora.Workspace.Ticket belongs_to :owner, User @@ -33,7 +34,7 @@ defmodule Algora.Bounties.Bounty do |> foreign_key_constraint(:ticket) |> foreign_key_constraint(:owner) |> foreign_key_constraint(:creator) - |> unique_constraint([:ticket_id, :owner_id]) + |> unique_constraint([:ticket_id, :owner_id, :number]) |> Algora.Validations.validate_money_positive(:amount) end diff --git a/lib/algora/integrations/github/client.ex b/lib/algora/integrations/github/client.ex index c4703f9d1..484bf294f 100644 --- a/lib/algora/integrations/github/client.ex +++ b/lib/algora/integrations/github/client.ex @@ -8,24 +8,24 @@ defmodule Algora.Github.Client do @type token :: String.t() - # TODO: move to a separate module and use only for data migration between databases - def http_cached(host, method, path, headers, body, opts \\ []) do - cache_path = ".local/github/#{path}.json" - - with :error <- read_from_cache(cache_path), - {:ok, response_body} <- do_http_request(host, method, path, headers, body, opts) do - write_to_cache(cache_path, response_body) - {:ok, response_body} + def http(host, method, path, headers, body, opts \\ []) do + # TODO: remove after migration + if System.get_env("MIGRATION", "false") == "true" do + cache_path = ".local/github/#{path}.json" + + with :error <- read_from_cache(cache_path), + {:ok, response_body} <- do_http_request(host, method, path, headers, body, opts) do + write_to_cache(cache_path, response_body) + {:ok, response_body} + else + {:ok, cached_data} -> {:ok, cached_data} + {:error, reason} -> {:error, reason} + end else - {:ok, cached_data} -> {:ok, cached_data} - {:error, reason} -> {:error, reason} + do_http_request(host, method, path, headers, body, opts) end end - def http(host, method, path, headers, body, opts \\ []) do - do_http_request(host, method, path, headers, body, opts) - end - defp do_http_request(host, method, path, headers, body, opts) do url = "https://#{host}#{path}" headers = [{"Content-Type", "application/json"} | headers] diff --git a/lib/algora/payments/schemas/account.ex b/lib/algora/payments/schemas/account.ex index ec6de5847..07805fef5 100644 --- a/lib/algora/payments/schemas/account.ex +++ b/lib/algora/payments/schemas/account.ex @@ -9,7 +9,7 @@ defmodule Algora.Payments.Account do typed_schema "accounts" do field :provider, :string, null: false field :provider_id, :string, null: false - field :provider_meta, :map, null: false + field :provider_meta, :map field :name, :string field :details_submitted, :boolean, default: false, null: false @@ -19,7 +19,7 @@ defmodule Algora.Payments.Account do field :payout_speed, :integer field :default_currency, :string field :service_agreement, :string - field :country, :string, null: false + field :country, :string field :type, Ecto.Enum, values: [:standard, :express], null: false field :stale, :boolean, default: false, null: false diff --git a/lib/algora/workspace/schemas/command_response.ex b/lib/algora/workspace/schemas/command_response.ex index d4f4b173d..25192b4de 100644 --- a/lib/algora/workspace/schemas/command_response.ex +++ b/lib/algora/workspace/schemas/command_response.ex @@ -7,7 +7,7 @@ defmodule Algora.Workspace.CommandResponse do typed_schema "command_responses" do field :provider, :string, null: false - field :provider_meta, :map, null: false + field :provider_meta, :map field :provider_command_id, :string field :provider_response_id, :string, null: false field :command_source, Ecto.Enum, values: [:ticket, :comment], null: false diff --git a/lib/algora/workspace/schemas/installation.ex b/lib/algora/workspace/schemas/installation.ex index 33878269b..ec164c0f2 100644 --- a/lib/algora/workspace/schemas/installation.ex +++ b/lib/algora/workspace/schemas/installation.ex @@ -9,8 +9,8 @@ defmodule Algora.Workspace.Installation do typed_schema "installations" do field :provider, :string, null: false field :provider_id, :string, null: false - field :provider_meta, :map, null: false - field :provider_user_id, :string, null: false + field :provider_meta, :map + field :provider_user_id, :string field :avatar_url, :string field :repository_selection, :string diff --git a/mix.exs b/mix.exs index 5fc91e661..4e00827cb 100644 --- a/mix.exs +++ b/mix.exs @@ -94,7 +94,9 @@ defmodule Algora.MixProject do # monitoring, logging {:appsignal_phoenix, "~> 2.6"}, {:logfmt_ex, "~> 0.4"}, - {:oban_live_dashboard, "~> 0.1.0"} + {:oban_live_dashboard, "~> 0.1.0"}, + # TODO: delete after migration + {:yaml_elixir, "~> 2.9"} ] end diff --git a/mix.lock b/mix.lock index 31257ccfc..e9a713e17 100644 --- a/mix.lock +++ b/mix.lock @@ -88,4 +88,6 @@ "uri_query": {:hex, :uri_query, "0.1.2", "ae35b83b472f3568c2c159eee3f3ccf585375d8a94fb5382db1ea3589e75c3b4", [:mix], [], "hexpm", "e3bc81816c98502c36498b9b2f239b89c71ce5eadfff7ceb2d6c0a2e6ae2ea0c"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, "websock_adapter": {:hex, :websock_adapter, "0.5.8", "3b97dc94e407e2d1fc666b2fb9acf6be81a1798a2602294aac000260a7c4a47d", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "315b9a1865552212b5f35140ad194e67ce31af45bcee443d4ecb96b5fd3f3782"}, + "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"}, + "yaml_elixir": {:hex, :yaml_elixir, "2.11.0", "9e9ccd134e861c66b84825a3542a1c22ba33f338d82c07282f4f1f52d847bd50", [:mix], [{:yamerl, "~> 0.10", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "53cc28357ee7eb952344995787f4bb8cc3cecbf189652236e9b163e8ce1bc242"}, } diff --git a/priv/repo/migrations/20250208173814_make_provider_meta_nullable.exs b/priv/repo/migrations/20250208173814_make_provider_meta_nullable.exs new file mode 100644 index 000000000..3e42c4f54 --- /dev/null +++ b/priv/repo/migrations/20250208173814_make_provider_meta_nullable.exs @@ -0,0 +1,50 @@ +defmodule Algora.Repo.Migrations.MakeProviderMetaNullable do + use Ecto.Migration + + def change do + alter table(:command_responses) do + modify :provider_meta, :map, null: true + end + + alter table(:installations) do + modify :provider_meta, :map, null: true + modify :provider_user_id, :string, null: true + end + + alter table(:customers) do + modify :provider_meta, :map, null: true + end + + alter table(:payment_methods) do + modify :provider_meta, :map, null: true + end + + alter table(:accounts) do + modify :provider_meta, :map, null: true + modify :country, :string, null: true + end + + alter table(:identities) do + modify :provider_meta, :map, null: true + modify :provider_login, :string, null: true + end + + alter table(:claims) do + modify :source_id, :string, null: true + end + + alter table(:tips) do + modify :creator_id, :string, null: true + end + + alter table(:bounties) do + add :number, :integer, null: false, default: 0 + end + + drop unique_index(:bounties, [:ticket_id, :owner_id]) + create unique_index(:bounties, [:ticket_id, :owner_id, :number]) + + drop unique_index(:command_responses, [:ticket_id, :command_type]) + create unique_index(:command_responses, [:provider, :provider_command_id]) + end +end diff --git a/priv/repo/migrations/20250213155854_define_negate_operator.exs b/priv/repo/migrations/20250213155854_define_negate_operator.exs new file mode 100644 index 000000000..5395eb2e1 --- /dev/null +++ b/priv/repo/migrations/20250213155854_define_negate_operator.exs @@ -0,0 +1,38 @@ +defmodule Algora.Repo.Migrations.DefineNegateOperator do + use Ecto.Migration + + def up do + execute(""" + CREATE OR REPLACE FUNCTION money_negate(money_1 money_with_currency) + RETURNS money_with_currency + IMMUTABLE + STRICT + LANGUAGE plpgsql + AS $$ + DECLARE + currency varchar; + addition numeric; + BEGIN + currency := currency_code(money_1); + addition := amount(money_1) * -1; + return row(currency, addition); + END; + $$; + """) + |> Money.Migration.adjust_for_type(repo()) + + execute(""" + CREATE OPERATOR - ( + rightarg = money_with_currency, + procedure = money_negate + ); + """) + |> Money.Migration.adjust_for_type(repo()) + end + + def down do + execute("DROP OPERATOR IF EXISTS - (none, money_with_currency);") + + execute("DROP FUNCTION IF EXISTS money_negate(money_with_currency);") + end +end diff --git a/scripts/analyze_progress.exs b/scripts/analyze_progress.exs new file mode 100644 index 000000000..a62191cd2 --- /dev/null +++ b/scripts/analyze_progress.exs @@ -0,0 +1,133 @@ +defmodule ProgressAnalyzer do + @moduledoc false + require Logger + + def analyze_file(path, verbose \\ false) do + stats = + path + |> File.read!() + |> YamlElixir.read_from_string!() + |> count_statuses() + + display_stats(stats, verbose) + end + + defp count_statuses(yaml) do + # Initialize counters + initial_counts = %{completed: 0, skipped: 0, remaining: 0, undecided: 0, remaining_columns: []} + + # Flatten and count all column statuses + Enum.reduce(yaml, initial_counts, fn table, acc -> + columns = table |> Map.values() |> List.first() + + Enum.reduce(columns, acc, fn column, inner_acc -> + {col, status} = Enum.at(Map.to_list(column), 0) + + case status do + 1 -> + Map.update!(inner_acc, :completed, &(&1 + 1)) + + -2 -> + Map.update!(inner_acc, :undecided, &(&1 + 1)) + + -1 -> + Map.update!(inner_acc, :skipped, &(&1 + 1)) + + 0 -> + inner_acc + |> Map.update!(:remaining, &(&1 + 1)) + |> Map.update!(:remaining_columns, &(&1 ++ ["#{table |> Map.keys() |> List.first()}.#{col}"])) + end + end) + end) + end + + defp display_stats( + %{ + completed: done, + skipped: skipped, + remaining: todo, + undecided: undecided, + remaining_columns: remaining_columns + }, + verbose + ) do + total = done + skipped + todo + undecided + done_pct = percentage(done, total) + skipped_pct = percentage(skipped, total) + todo_pct = percentage(todo, total) + undecided_pct = percentage(undecided, total) + + base_output = """ + Migration Progress Report + ======================== + + Summary: + -------- + Total columns: #{total} + + Status Breakdown: + ---------------- + ✅ Completed: #{String.pad_leading("#{done_pct}", 2)}% #{String.pad_leading("(#{done})", 5)} + ⏳ Remaining: #{String.pad_leading("#{todo_pct}", 2)}% #{String.pad_leading("(#{todo})", 5)} + ❌ Skipped: #{String.pad_leading("#{skipped_pct}", 2)}% #{String.pad_leading("(#{skipped})", 5)} + ❓ Undecided: #{String.pad_leading("#{undecided_pct}", 2)}% #{String.pad_leading("(#{undecided})", 5)} + + Progress: + --------- + [#{progress_bar(done_pct, skipped_pct, todo_pct, undecided_pct)}] + """ + + if_result = + if verbose do + base_output <> + """ + + Remaining Columns: + ------------------ + #{Enum.join(remaining_columns, "\n")} + """ + else + base_output + end + + IO.puts(if_result) + end + + defp percentage(part, total) when total > 0 do + trunc(part / total * 100) + end + + defp progress_bar(done_pct, skipped_pct, todo_pct, undecided_pct) do + done_chars = round(done_pct / 2) + skipped_chars = round(skipped_pct / 2) + todo_chars = round(todo_pct / 2) + undecided_chars = round(undecided_pct / 2) + + String.duplicate("=", done_chars) <> + String.duplicate("x", skipped_chars) <> + String.duplicate("?", undecided_chars) <> + String.duplicate(".", todo_chars) + end +end + +case System.argv() do + [filename, "-v"] -> + ProgressAnalyzer.analyze_file(filename, true) + + ["-v", filename] -> + ProgressAnalyzer.analyze_file(filename, true) + + [filename] -> + ProgressAnalyzer.analyze_file(filename, false) + + _ -> + IO.puts(""" + Usage: elixir analyze_progress.exs [-v] + + Options: + -v Show detailed remaining columns list + """) + + System.halt(1) +end diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index 3975e5801..8accdced3 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -19,178 +19,835 @@ defmodule DatabaseMigration do - Set the output_file to your desired output file path. - Run the script using: elixir scripts/database_migration.exs """ + alias Algora.Accounts.Identity alias Algora.Accounts.User + alias Algora.Bounties.Attempt alias Algora.Bounties.Bounty + alias Algora.Bounties.Claim + alias Algora.Bounties.Tip + alias Algora.Organizations.Member + alias Algora.Payments.Account + alias Algora.Payments.Customer + alias Algora.Payments.PaymentMethod alias Algora.Payments.Transaction + alias Algora.Workspace.CommandResponse + alias Algora.Workspace.Installation alias Algora.Workspace.Ticket require Logger - @table_mappings %{ - "User" => "users", - "Org" => "users", - "GithubUser" => "users", - "Task" => "tickets", - "GithubIssue" => nil, - "GithubPullRequest" => nil, - "Bounty" => "bounties", - "Reward" => nil, - "BountyTransfer" => "transactions", - "Claim" => nil - } - - @schema_mappings %{ - "User" => User, - "Org" => User, - "GithubUser" => User, - "Task" => Ticket, - "GithubIssue" => nil, - "GithubPullRequest" => nil, - "Bounty" => Bounty, - "Reward" => nil, - "BountyTransfer" => Transaction, - "Claim" => nil - } - - @backfilled_tables ["repositories", "transactions", "bounties", "tickets", "users"] - - @relevant_tables Map.keys(@table_mappings) - - defp transform("Task", row, db) do - github_issue = - db |> Map.get("GithubIssue", []) |> Enum.find(&(&1["id"] == row["issue_id"])) - - github_pull_request = - db |> Map.get("GithubPullRequest", []) |> Enum.find(&(&1["id"] == row["pull_request_id"])) + @schema_mappings [ + {"User", User}, + {"Org", User}, + {"GithubUser", User}, + {"Account", Identity}, + {"OrgMember", Member}, + {"Task", Ticket}, + {"GithubIssue", nil}, + {"GithubPullRequest", nil}, + {"Bounty", Bounty}, + {"Bounty", CommandResponse}, + {"Reward", nil}, + {"Attempt", Attempt}, + {"Claim", Claim}, + {"BountyCharge", Transaction}, + {"BountyTransfer", Tip}, + {"BountyTransfer", Transaction}, + {"OrgBalanceTransaction", Transaction}, + {"GithubInstallation", Installation}, + {"StripeAccount", Account}, + {"StripeCustomer", Customer}, + {"StripePaymentMethod", PaymentMethod} + ] + + @index_fields [ + {"User", ["id"]}, + {"GithubUser", ["id", "user_id"]}, + {"Bounty", ["id"]}, + {"Task", ["id"]}, + {"Claim", ["id"]}, + {"BountyCharge", ["id"]}, + {"StripeCustomer", ["org_id"]}, + {"GithubIssue", ["id"]}, + {"GithubPullRequest", ["id", "task_id"]}, + {"Reward", ["bounty_id"]} + ] + + defp relevant_tables do + @schema_mappings + |> Enum.map(fn {k, _v} -> k end) + |> Enum.dedup() + end + + defp backfilled_tables do + @schema_mappings + |> Enum.map(fn {_, v} -> v end) + |> Enum.reject(&is_nil/1) + |> Enum.dedup() + |> Enum.reverse() + |> Enum.map(& &1.__schema__(:source)) + end + + defp transform({"Task", Ticket}, row, db) do + if row["forge"] != "github" do + raise "Unknown forge: #{row["forge"]}" + end + + github_issue = find_by_index(db, "GithubIssue", "id", row["issue_id"]) + github_pull_request = find_by_index(db, "GithubPullRequest", "id", row["pull_request_id"]) row = cond do github_issue -> - row - |> Map.put("type", "issue") - |> Map.put("title", github_issue["title"]) - |> Map.put("description", github_issue["body"]) - |> Map.put("inserted_at", github_issue["created_at"]) - |> Map.put("updated_at", github_issue["updated_at"]) - |> Map.put("url", github_issue["html_url"]) - |> Map.put("provider", "github") - |> Map.put("provider_id", github_issue["id"]) - |> Map.put("provider_meta", deserialize_value(github_issue)) + %{ + "id" => row["id"], + "provider" => "github", + "provider_id" => github_issue["id"], + "provider_meta" => deserialize_value(github_issue), + "type" => "issue", + "title" => github_issue["title"], + "description" => github_issue["body"], + "number" => github_issue["number"], + "url" => github_issue["html_url"], + "inserted_at" => github_issue["created_at"], + "updated_at" => github_issue["updated_at"] + } github_pull_request -> - row - |> Map.put("type", "pull_request") - |> Map.put("title", github_pull_request["title"]) - |> Map.put("description", github_pull_request["body"]) - |> Map.put("inserted_at", github_pull_request["created_at"]) - |> Map.put("updated_at", github_pull_request["updated_at"]) - |> Map.put("url", github_pull_request["html_url"]) - |> Map.put("provider", "github") - |> Map.put("provider_id", github_pull_request["id"]) - |> Map.put("provider_meta", deserialize_value(github_pull_request)) + %{ + "id" => row["id"], + "provider" => "github", + "provider_id" => github_pull_request["id"], + "provider_meta" => deserialize_value(github_pull_request), + "type" => "pull_request", + "title" => github_pull_request["title"], + "description" => github_pull_request["body"], + "number" => github_pull_request["number"], + "url" => github_pull_request["html_url"], + "inserted_at" => github_pull_request["created_at"], + "updated_at" => github_pull_request["updated_at"] + } true -> - row - # TODO: maybe discard altogther? - |> Map.put("inserted_at", "1970-01-01 00:00:00") - |> Map.put("updated_at", "1970-01-01 00:00:00") + %{ + "id" => row["id"], + "provider" => row["forge"], + "provider_id" => nil, + "provider_meta" => nil, + "type" => "issue", + "title" => row["title"], + "description" => row["body"], + "number" => row["number"], + "url" => "https://github.com/#{row["repo_owner"]}/#{row["repo_name"]}/issues/#{row["number"]}", + "inserted_at" => "1970-01-01 00:00:00", + "updated_at" => "1970-01-01 00:00:00" + } end row end - defp transform("User", row, db) do - github_user = - db |> Map.get("GithubUser", []) |> Enum.find(&(&1["user_id"] == row["id"])) + defp transform({"User", User}, row, db) do + # TODO: reenable + # if !row["\"emailVerified\""] || String.length(row["\"emailVerified\""]) < 10 do + # raise "Email not verified: #{inspect(row)}" + # end + + github_user = find_by_index(db, "GithubUser", "user_id", row["id"]) + + %{ + "id" => row["id"], + "provider" => github_user && "github", + "provider_id" => github_user && github_user["id"], + "provider_login" => github_user && github_user["login"], + "provider_meta" => github_user && deserialize_value(github_user), + "email" => row["email"], + "display_name" => row["name"], + "handle" => row["handle"], + "avatar_url" => update_url(row["image"]), + "external_homepage_url" => nil, + "type" => "individual", + "bio" => github_user && github_user["bio"], + "location" => row["loc"], + "country" => row["country"], + "timezone" => nil, + "stargazers_count" => row["stars_earned"], + "domain" => nil, + "tech_stack" => row["tech"], + "featured" => nil, + "priority" => nil, + "fee_pct" => nil, + "seeded" => nil, + "activated" => nil, + "max_open_attempts" => nil, + "manual_assignment" => nil, + "bounty_mode" => nil, + "hourly_rate_min" => nil, + "hourly_rate_max" => nil, + "hours_per_week" => nil, + "website_url" => nil, + "twitter_url" => nil, + "github_url" => nil, + "youtube_url" => row["youtube_handle"] && "https://www.youtube.com/#{row["youtube_handle"]}", + "twitch_url" => row["twitch_handle"] && "https://www.twitch.tv/#{row["twitch_handle"]}", + "discord_url" => nil, + "slack_url" => nil, + "linkedin_url" => nil, + "og_title" => nil, + "og_image_url" => nil, + "last_context" => row["handle"], + "need_avatar" => nil, + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"], + "is_admin" => row["is_admin"] + } + end - row = - if github_user do - row - |> Map.put("provider", "github") - |> Map.put("provider_id", github_user["id"]) - |> Map.put("provider_login", github_user["login"]) - |> Map.put("provider_meta", deserialize_value(github_user)) - else - row - end + defp transform({"Org", User}, row, db) do + merged_user = find_by_index(db, "_MergedUser", "id", row["id"]) + + if not user?(merged_user) do + %{ + "id" => row["id"], + "provider" => row["github_handle"] && "github", + "provider_id" => row["github_id"], + "provider_login" => row["github_handle"], + "provider_meta" => row["github_data"] && deserialize_value(row["github_data"]), + "email" => nil, + "display_name" => row["name"], + "handle" => row["handle"], + "avatar_url" => update_url(row["avatar_url"]), + "external_homepage_url" => nil, + "type" => "organization", + "bio" => row["description"], + "location" => nil, + "country" => nil, + "timezone" => nil, + "stargazers_count" => row["stargazers_count"], + "domain" => row["domain"], + "tech_stack" => row["tech"], + "featured" => row["featured"], + "priority" => row["priority"], + "fee_pct" => row["fee_pct"], + "seeded" => row["seeded"], + "activated" => row["active"], + "max_open_attempts" => row["max_open_attempts"], + "manual_assignment" => row["manual_assignment"], + "bounty_mode" => nil, + "hourly_rate_min" => nil, + "hourly_rate_max" => nil, + "hours_per_week" => nil, + "website_url" => row["website_url"], + "twitter_url" => row["twitter_url"], + "github_url" => nil, + "youtube_url" => row["youtube_url"], + "twitch_url" => nil, + "discord_url" => row["discord_url"], + "slack_url" => row["slack_url"], + "linkedin_url" => nil, + "og_title" => nil, + "og_image_url" => nil, + "last_context" => nil, + "need_avatar" => nil, + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"], + "is_admin" => false + } + end + end - row - |> Map.put("type", "individual") - |> rename_column("tech", "tech_stack") - |> rename_column("stars_earned", "stargazers_count") - |> rename_column("image", "avatar_url") - |> update_url_field("avatar_url") + defp transform({"GithubUser", User}, row, _db) do + if nullish?(row["user_id"]) do + %{ + "id" => row["id"], + "provider" => "github", + "provider_id" => row["id"], + "provider_login" => row["login"], + "provider_meta" => deserialize_value(row), + "email" => nil, + "display_name" => row["name"], + "handle" => nil, + "avatar_url" => row["avatar_url"], + "external_homepage_url" => nil, + "type" => "individual", + "bio" => row["bio"], + "location" => row["location"], + "country" => nil, + "timezone" => nil, + "stargazers_count" => nil, + "domain" => nil, + "tech_stack" => nil, + "featured" => nil, + "priority" => nil, + "fee_pct" => nil, + "seeded" => nil, + "activated" => nil, + "max_open_attempts" => nil, + "manual_assignment" => nil, + "bounty_mode" => nil, + "hourly_rate_min" => nil, + "hourly_rate_max" => nil, + "hours_per_week" => nil, + "website_url" => nil, + "twitter_url" => row["twitter_username"] && "https://www.twitter.com/#{row["twitter_username"]}", + "github_url" => nil, + "youtube_url" => nil, + "twitch_url" => nil, + "discord_url" => nil, + "slack_url" => nil, + "linkedin_url" => nil, + "og_title" => nil, + "og_image_url" => nil, + "last_context" => nil, + "need_avatar" => nil, + "inserted_at" => row["retrieved_at"], + "updated_at" => row["retrieved_at"], + "is_admin" => nil + } + end end - defp transform("Org", row, _db) do - row - |> Map.put("type", "organization") - |> Map.put("provider", row["github_handle"] && "github") - |> rename_column("github_handle", "provider_login") - |> rename_column("tech", "tech_stack") - |> update_url_field("avatar_url") + defp transform({"Account", Identity}, row, db) do + user = find_by_index(db, "User", "id", row["\"userId\""]) + + if !user do + raise "User not found: #{inspect(row)}" + end + + %{ + "id" => row["id"], + "user_id" => user["id"], + "provider" => row["provider"], + "provider_token" => row["access_token"], + "provider_email" => user["email"], + "provider_login" => nil, + "provider_id" => row["\"providerAccountId\""], + "provider_meta" => nil, + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"] + } end - defp transform("GithubUser", %{user_id: nil} = row, _db) do - row - |> Map.put("type", "individual") - |> Map.put("provider", "github") - |> Map.put("provider_id", row["id"]) - |> Map.put("provider_meta", deserialize_value(row)) + defp transform({"OrgMember", Member}, row, db) do + owner = find_by_index(db, "_MergedUser", "id", row["org_id"]) + + if !owner do + raise "Owner not found: #{inspect(row)}" + end + + if owner["id"] != row["user_id"] do + %{ + "id" => row["id"], + "org_id" => owner["id"], + "role" => row["role"], + "user_id" => row["user_id"], + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"] + } + end end - defp transform("GithubUser", _row, _db), do: nil + defp transform({"Bounty", Bounty}, row, db) do + reward = find_by_index(db, "Reward", "bounty_id", row["id"]) + owner = find_by_index(db, "_MergedUser", "id", row["org_id"]) - defp transform("Bounty", row, db) do - reward = db |> Map.get("Reward", []) |> Enum.find(&(&1["bounty_id"] == row["id"])) + amount = if reward, do: Money.from_integer(String.to_integer(reward["amount"]), reward["currency"]) - amount = - if reward, do: Money.from_integer(String.to_integer(reward["amount"]), reward["currency"]) + if !owner do + raise "Owner not found: #{inspect(row)}" + end - row - |> Map.put("ticket_id", row["task_id"]) - |> Map.put("owner_id", row["org_id"]) - |> Map.put("creator_id", row["poster_id"]) - |> Map.put("inserted_at", row["created_at"]) - |> Map.put("updated_at", row["updated_at"]) - |> Map.put("amount", amount) + transfer = find_by_index(db, "_BountyTransfer", "bounty_id", row["id"]) + + if row["type"] != "tip" do + %{ + "id" => row["id"], + "amount" => amount, + "status" => + cond do + not is_nil(transfer) -> :paid + row["deleted_at"] || row["status"] == "inactive" -> :cancelled + true -> :open + end, + "ticket_id" => row["task_id"], + "owner_id" => owner["id"], + "creator_id" => row["poster_id"], + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"], + "number" => row["number"] + } + end + end + + defp transform({"Bounty", CommandResponse}, row, _db) do + if !nullish?(row["github_res_comment_id"]) do + %{ + "id" => row["id"], + "provider" => "github", + "provider_meta" => nil, + "provider_command_id" => row["github_req_comment_id"], + "provider_response_id" => row["github_res_comment_id"], + "command_source" => "comment", + "command_type" => "bounty", + "ticket_id" => row["task_id"], + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"] + } + end end - defp transform("BountyTransfer", row, db) do - claim = - db |> Map.get("Claim", []) |> Enum.find(&(&1["id"] == row["claim_id"])) + defp transform({"Attempt", Attempt}, row, db) do + bounty = find_by_index(db, "Bounty", "id", row["bounty_id"]) + github_user = find_by_index(db, "GithubUser", "id", row["github_user_id"]) - github_user = - db |> Map.get("GithubUser", []) |> Enum.find(&(&1["id"] == claim["github_user_id"])) + user_id = or_else(github_user["user_id"], github_user["id"]) + + if !bounty do + raise "Bounty not found: #{inspect(row)}" + end + + if nullish?(user_id) do + raise "User not found: #{inspect(row)}" + end + + %{ + "id" => row["id"], + "status" => row["status"], + "warnings_count" => row["warnings_count"], + "ticket_id" => bounty["task_id"], + "user_id" => user_id, + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"] + } + end - user = - db |> Map.get("User", []) |> Enum.find(&(&1["id"] == github_user["user_id"])) + defp transform({"Claim", Claim}, row, db) do + bounty = find_by_index(db, "Bounty", "id", row["bounty_id"]) + task = find_by_index(db, "Task", "id", bounty["task_id"]) + github_user = find_by_index(db, "GithubUser", "id", row["github_user_id"]) + github_pull_request = find_by_index(db, "GithubPullRequest", "id", row["github_pull_request_id"]) + + user_id = or_else(github_user["user_id"], github_user["id"]) + + if !task do + raise "Task not found: #{inspect(row)}" + end + + if nullish?(user_id) do + raise "User not found: #{inspect(row)}" + end + + %{ + "id" => row["id"], + "status" => + case row["status"] do + "accepted" -> :approved + _ -> :pending + end, + "type" => + cond do + !nullish?(row["github_pull_request_id"]) -> "pull_request" + String.match?(row["github_url"], ~r{^https?://(?:www\.)?figma\.com/}) -> "design" + true -> "pull_request" + end, + "url" => or_else(row["github_url"], "https://algora.io"), + "group_id" => row["id"], + "group_share" => nil, + "source_id" => github_pull_request["task_id"], + "target_id" => task["id"], + "user_id" => user_id, + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"] + } + end + + defp transform({"BountyCharge", Transaction}, row, db) do + user = find_by_index(db, "_MergedUser", "id", row["org_id"]) amount = Money.from_integer(String.to_integer(row["amount"]), row["currency"]) - row = - if claim && user do - row - |> Map.put("type", "transfer") - |> Map.put("provider", "stripe") - |> Map.put("provider_id", row["transfer_id"]) - |> Map.put("net_amount", amount) - |> Map.put("gross_amount", amount) - |> Map.put("total_fee", Money.zero(:USD)) - |> Map.put("bounty_id", claim["bounty_id"]) - ## TODO: this might be null but shouldn't - |> Map.put("user_id", user["id"]) - |> Map.put("inserted_at", row["created_at"]) - |> Map.put("updated_at", row["updated_at"]) - |> Map.put("status", if(row["succeeded_at"] == nil, do: :initialized, else: :succeeded)) - |> Map.put("succeeded_at", row["succeeded_at"]) + if !user do + raise "User not found: #{inspect(row)}" + end + + if !nullish?(row["succeeded_at"]) do + %{ + "id" => row["id"], + "provider" => "stripe", + "provider_id" => row["charge_id"], + "provider_charge_id" => row["charge_id"], + "provider_payment_intent_id" => nil, + "provider_transfer_id" => nil, + "provider_invoice_id" => nil, + "provider_balance_transaction_id" => nil, + "provider_meta" => nil, + # TODO: incorrect + "gross_amount" => amount, + "net_amount" => amount, + # TODO: incorrect + "total_fee" => Money.zero(:USD), + "provider_fee" => nil, + "line_items" => nil, + "type" => "charge", + "status" => if(nullish?(row["succeeded_at"]), do: :initialized, else: :succeeded), + "succeeded_at" => row["succeeded_at"], + "reversed_at" => nil, + "group_id" => row["id"], + "user_id" => user["id"], + "contract_id" => nil, + "original_contract_id" => nil, + "timesheet_id" => nil, + "bounty_id" => nil, + "tip_id" => nil, + "linked_transaction_id" => nil, + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"], + "claim_id" => nil + } + end + end + + defp transform({"BountyTransfer", Tip}, row, db) do + claim = find_by_index(db, "Claim", "id", row["claim_id"]) + github_user = find_by_index(db, "GithubUser", "id", claim["github_user_id"]) + bounty = find_by_index(db, "Bounty", "id", claim["bounty_id"]) + owner = find_by_index(db, "_MergedUser", "id", bounty["org_id"]) + user_id = or_else(github_user["user_id"], github_user["id"]) + amount = Money.from_integer(String.to_integer(row["amount"]), row["currency"]) + + if !bounty do + raise "Bounty not found: #{inspect(row)}" + end + + if nullish?(user_id) do + raise "User not found: #{inspect(row)}" + end + + if !owner do + raise "Owner not found: #{inspect(row)}" + end + + if bounty["type"] == "tip" do + %{ + "id" => bounty["id"] <> user_id, + "amount" => amount, + "status" => nil, + "ticket_id" => bounty["task_id"], + "owner_id" => owner["id"], + "creator_id" => bounty["poster_id"], + "recipient_id" => user_id, + "inserted_at" => bounty["created_at"], + "updated_at" => bounty["updated_at"] + } + end + end + + defp transform({"BountyTransfer", Transaction}, row, db) do + claim = find_by_index(db, "Claim", "id", row["claim_id"]) + bounty = find_by_index(db, "Bounty", "id", claim["bounty_id"]) + github_user = find_by_index(db, "GithubUser", "id", claim["github_user_id"]) + org = find_by_index(db, "_MergedUser", "id", bounty["org_id"]) + bounty_charge = find_by_index(db, "BountyCharge", "id", row["bounty_charge_id"]) + + user_id = or_else(github_user["user_id"], github_user["id"]) + + if !bounty do + raise "Bounty not found: #{inspect(row)}" + end + + if nullish?(user_id) do + raise "User not found: #{inspect(row)}" + end + + if !org do + raise "Org not found: #{inspect(row)}" + end + + if !bounty_charge do + raise "BountyCharge not found: #{inspect(row)}" + end + + Enum.reject( + [ + maybe_create_transaction("debit", %{ + bounty_charge: bounty_charge, + bounty_transfer: row, + bounty: bounty, + claim: claim, + org: org, + user_id: user_id + }), + maybe_create_transaction("credit", %{ + bounty_charge: bounty_charge, + bounty_transfer: row, + bounty: bounty, + claim: claim, + org: org, + user_id: user_id + }), + maybe_create_transaction("transfer", %{ + bounty_charge: bounty_charge, + bounty_transfer: row, + bounty: bounty, + claim: claim, + org: org, + user_id: user_id + }) + ], + &is_nil/1 + ) + end + + defp transform({"OrgBalanceTransaction", Transaction}, row, db) do + user = find_by_index(db, "_MergedUser", "id", row["org_id"]) + + if !user do + raise "User not found: #{inspect(row)}" + end + + amount = Money.from_integer(String.to_integer(row["amount"]), row["currency"]) + + {abs_amount, type} = + if Money.positive?(amount) do + {amount, "credit"} + else + {Money.negate!(amount), "debit"} end - row + %{ + "id" => row["id"], + "provider" => "stripe", + "provider_id" => nil, + "provider_charge_id" => nil, + "provider_payment_intent_id" => nil, + "provider_transfer_id" => nil, + "provider_invoice_id" => nil, + "provider_balance_transaction_id" => nil, + "provider_meta" => nil, + "gross_amount" => abs_amount, + "net_amount" => abs_amount, + "total_fee" => Money.zero(:USD), + "provider_fee" => nil, + "line_items" => nil, + "type" => type, + "status" => :succeeded, + "succeeded_at" => row["created_at"], + "reversed_at" => nil, + "group_id" => row["id"], + "user_id" => user["id"], + "contract_id" => nil, + "original_contract_id" => nil, + "timesheet_id" => nil, + "bounty_id" => nil, + "tip_id" => nil, + "linked_transaction_id" => nil, + "inserted_at" => row["created_at"], + "updated_at" => row["created_at"], + "claim_id" => nil + } + end + + defp transform({"GithubInstallation", Installation}, row, db) do + connected_user = find_by_index(db, "_MergedUser", "id", row["org_id"]) + + if !connected_user do + raise "Connected user not found: #{inspect(row)}" + end + + %{ + "id" => row["id"], + "provider" => "github", + "provider_id" => row["github_id"], + "provider_meta" => nil, + "avatar_url" => nil, + "repository_selection" => nil, + "owner_id" => nil, + "connected_user_id" => connected_user["id"], + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"], + "provider_user_id" => nil + } + end + + defp transform({"StripeAccount", Account}, row, _db) do + %{ + "id" => row["id"], + "provider" => "stripe", + "provider_id" => row["id"], + "provider_meta" => nil, + "name" => nil, + "details_submitted" => row["details_submitted"], + "charges_enabled" => row["charges_enabled"], + "service_agreement" => row["service_agreement"], + "country" => row["country"], + "type" => row["type"], + "stale" => row["needs_refresh"], + "user_id" => row["user_id"], + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"], + "payouts_enabled" => nil, + "payout_interval" => nil, + "payout_speed" => nil, + "default_currency" => nil + } + end + + defp transform({"StripeCustomer", Customer}, row, db) do + owner = find_by_index(db, "_MergedUser", "id", row["org_id"]) + + if !owner do + raise "Owner not found: #{inspect(row)}" + end + + if owner["id"] not in ["clfqtao4h0001mo0gkp9az0bn", "cm251pvg40007ld031q5t2hj2", "cljo6j981000el60f1k1cvtns"] do + %{ + "id" => row["id"], + "provider" => "stripe", + "provider_id" => row["stripe_id"], + "provider_meta" => nil, + "name" => row["name"], + "user_id" => owner["id"], + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"] + } + end + end + + defp transform({"StripePaymentMethod", PaymentMethod}, row, db) do + owner = find_by_index(db, "_MergedUser", "id", row["org_id"]) + + if !owner do + raise "Owner not found: #{inspect(row)}" + end + + customer = find_by_index(db, "StripeCustomer", "org_id", row["org_id"]) + + if !customer do + raise "StripeCustomer not found: #{inspect(row)}" + end + + if owner["id"] not in ["clfqtao4h0001mo0gkp9az0bn", "cm251pvg40007ld031q5t2hj2", "cljo6j981000el60f1k1cvtns"] do + %{ + "id" => row["id"], + "provider" => "stripe", + "provider_id" => row["stripe_id"], + "provider_meta" => nil, + "provider_customer_id" => customer["stripe_id"], + "is_default" => row["is_default"], + "customer_id" => customer["id"], + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"] + } + end end defp transform(_, _row, _db), do: nil + defp maybe_create_transaction(type, %{ + bounty_charge: bounty_charge, + bounty_transfer: bounty_transfer, + bounty: bounty, + claim: claim, + org: org, + user_id: user_id + }) do + amount = Money.from_integer(String.to_integer(bounty_transfer["amount"]), bounty_transfer["currency"]) + + res = %{ + "id" => String.slice(type, 0, 2) <> "_" <> bounty_transfer["id"], + "provider" => "stripe", + "provider_id" => nil, + "provider_charge_id" => bounty_charge["charge_id"], + "provider_payment_intent_id" => nil, + "provider_transfer_id" => bounty_transfer["transfer_id"], + "provider_invoice_id" => nil, + "provider_balance_transaction_id" => nil, + "provider_meta" => nil, + "gross_amount" => amount, + "net_amount" => amount, + "total_fee" => Money.zero(:USD), + "provider_fee" => nil, + "line_items" => nil, + "type" => type, + "status" => nil, + "succeeded_at" => nil, + "reversed_at" => nil, + "group_id" => bounty_charge["id"], + "user_id" => nil, + "contract_id" => nil, + "original_contract_id" => nil, + "timesheet_id" => nil, + "bounty_id" => nil, + "tip_id" => nil, + "linked_transaction_id" => nil, + "inserted_at" => nil, + "updated_at" => nil, + "claim_id" => nil + } + + res = + if bounty["type"] == "tip" do + Map.put(res, "tip_id", bounty["id"] <> user_id) + else + res + |> Map.put("bounty_id", bounty["id"]) + |> Map.put("claim_id", claim["id"]) + end + + res = + case type do + "transfer" -> + Map.merge(res, %{ + "user_id" => user_id, + "provider_id" => bounty_transfer["transfer_id"] + }) + + "debit" -> + Map.merge(res, %{ + "user_id" => org["id"], + "linked_transaction_id" => "cr_" <> bounty_transfer["id"] + }) + + "credit" -> + Map.merge(res, %{ + "user_id" => user_id, + "linked_transaction_id" => "de_" <> bounty_transfer["id"] + }) + + _ -> + res + end + + cond do + type == "transfer" && !nullish?(bounty_transfer["succeeded_at"]) -> + Map.merge(res, %{ + "status" => :succeeded, + "succeeded_at" => bounty_transfer["succeeded_at"], + "inserted_at" => bounty_transfer["created_at"], + "updated_at" => bounty_transfer["updated_at"] + }) + + type == "debit" && !nullish?(bounty_charge["succeeded_at"]) -> + Map.merge(res, %{ + "status" => :succeeded, + "succeeded_at" => bounty_charge["succeeded_at"], + "inserted_at" => bounty_charge["succeeded_at"], + "updated_at" => bounty_charge["succeeded_at"] + }) + + type == "credit" && !nullish?(bounty_charge["succeeded_at"]) -> + Map.merge(res, %{ + "status" => :succeeded, + "succeeded_at" => bounty_charge["succeeded_at"], + "inserted_at" => bounty_charge["succeeded_at"], + "updated_at" => bounty_charge["succeeded_at"] + }) + + true -> + nil + end + end + def process_dump(input_file, output_file) do db = collect_data(input_file) @@ -207,24 +864,151 @@ defmodule DatabaseMigration do |> Stream.run() end + defp user?(row), do: not nullish?(row["email"]) + defp collect_data(input_file) do - input_file - |> File.stream!() - |> Stream.chunk_while( - nil, - &collect_chunk_fun/2, - &collect_after_fun/1 - ) - |> Enum.reduce(%{}, fn - {table, data}, acc when table in @relevant_tables -> - parsed_data = parse_copy_data(data) - Map.put(acc, table, parsed_data) + db = + input_file + |> File.stream!() + |> Stream.chunk_while( + nil, + &collect_chunk_fun/2, + &collect_after_fun/1 + ) + |> Enum.reduce(%{}, fn + {table, data}, acc -> + if table in relevant_tables() do + parsed_data = parse_copy_data(data) + Map.put(acc, table, parsed_data) + else + acc + end + end) + + indexes = + Enum.reduce(@index_fields, %{}, fn {table, columns}, acc -> + table_indexes = Map.new(columns, fn column -> {column, index_by_field(db, table, column)} end) + Map.put(acc, table, table_indexes) + end) - _, acc -> - acc + db = Map.put(db, :indexes, indexes) + + db + |> put_in([:indexes, "_MergedUser"], %{"id" => index_merged_users(db)}) + |> put_in([:indexes, "_BountyTransfer"], %{ + "bounty_id" => + Enum.group_by(db["BountyTransfer"], fn row -> + claim = find_by_index(db, "Claim", "id", row["claim_id"]) + charge = find_by_index(db, "BountyCharge", "id", row["bounty_charge_id"]) + + if charge["succeeded_at"] do + claim["bounty_id"] + end + end) + }) + end + + defp index_merged_users(db) do + entities = + (db["User"] ++ db["Org"]) + |> Enum.group_by(fn row -> + if user?(row) do + github_user = find_by_index(db, "GithubUser", "user_id", row["id"]) + + if is_nil(github_user) or nullish?(github_user["login"]) do + "algora_" <> row["id"] + else + "github_" <> github_user["login"] + end + else + if nullish?(row["github_handle"]) do + "algora_" <> row["id"] + else + "github_" <> row["github_handle"] + end + end + end) + |> Enum.flat_map(fn {_k, entities} -> + case entities do + [user] -> + [{:unmerged, user["id"], user}] + + entities -> + case Enum.find(entities, &user?/1) do + nil -> + raise "Unexpected number of users for #{inspect(entities)}" + + user -> + Enum.map(entities, fn row -> + # if row["id"] != user["id"], do: Logger.info("[same github user] #{row["handle"]} -> #{user["handle"]}") + {:merged, row["id"], user} + end) + end + end + end) + |> Enum.group_by(fn {type, _id, _user} -> type end) + + merged1 = + entities + |> Map.get(:merged, []) + |> Map.new(fn {_type, id, user} -> {id, user} end) + + merged2 = + entities + |> Map.get(:unmerged, []) + |> Enum.map(fn {_type, _id, row} -> row end) + |> Enum.group_by(fn row -> row["handle"] end) + |> Enum.flat_map(fn {handle, entities} -> + case entities do + [entity] -> + [{entity["id"], entity}] + + [_entity1, _entity2] -> + user = Enum.find(entities, &user?/1) + org = Enum.find(entities, &(not user?(&1))) + + if is_nil(user) or is_nil(org) do + raise "User or org not found for handle #{handle}: #{inspect(entities)}" + end + + if org["creator_id"] == user["id"] do + # Logger.info("[same handle] #{org["handle"]} -> #{user["handle"]}") + Enum.map(entities, fn row -> {row["id"], user} end) + else + Logger.warning("Org #{org["handle"]} was not created by user #{user["handle"]}") + Enum.map(entities, fn row -> {row["id"], row} end) + end + + _ -> + raise "Unexpected number of entities for handle #{handle}: #{inspect(entities)}" + end + end) + |> Map.new() + + Map.merge(merged1, merged2) + end + + defp index_by_field(db, table, field) do + db[table] + |> Enum.reject(fn row -> table == "StripeCustomer" and row["region"] == "EU" end) + |> Enum.group_by(&Map.get(&1, field)) + |> Enum.reject(fn {k, _v} -> nullish?(k) end) + |> Map.new(fn {k, v} -> + {k, + case v do + [v] -> v + v -> raise "Unexpected number of entities for #{table}.#{field}: #{inspect(v)}" + end} end) end + defp find_by_index(db, table, field, value) do + case get_in(db, [:indexes, table, field]) do + nil -> raise "Index not found for table #{table}.#{field}" + index -> index[value] + end + end + defp parse_copy_data([header | data]) do columns = header @@ -258,88 +1042,79 @@ defmodule DatabaseMigration do defp collect_after_fun({table, acc}), do: {:cont, {table, Enum.reverse(acc)}, nil} defp process_chunk(chunk, db) do - case_result = - case extract_copy_section(chunk) do - %{table: table} = section when table in @relevant_tables -> - transform_section(section, db) - - _ -> - nil - end + case extract_copy_section(chunk) do + %{table: table} = section -> + @schema_mappings + |> Enum.filter(fn {k, _v} -> k == table end) + |> Enum.map(fn {_k, v} -> transform_section(section, v, db) end) + |> Enum.reject(&is_nil/1) + |> Enum.map(&load_copy_section/1) - load_copy_section(case_result) + _ -> + [] + end end - defp transform_section(%{table: table, columns: _columns, data: data}, db) do + defp transform_section(%{table: table, columns: _columns, data: data}, schema, db) do transformed_data = data - |> Enum.map(fn row -> - try do - transform(table, row, db) - rescue - e -> - IO.puts("Error transforming row in table #{table}: #{inspect(row)}") - IO.puts("Error: #{inspect(e)}") - nil + |> Enum.flat_map(fn row -> + # try do + case transform({table, schema}, row, db) do + nil -> [] + xs when is_list(xs) -> xs + x -> [x] end - end) - |> Enum.reject(&is_nil/1) - |> Enum.map(&post_transform(table, &1)) - transformed_table_name = transform_table_name(table) + # rescue + # e -> + # IO.puts("Error transforming row in table #{table}: #{inspect(row)}") + # IO.puts("Error: #{inspect(e)}") + # nil + # end + end) + |> Enum.map(&post_transform(schema, &1)) if Enum.empty?(transformed_data) do nil else - transformed_columns = Map.keys(hd(transformed_data)) - %{table: transformed_table_name, columns: transformed_columns, data: transformed_data} + %{table: schema.__schema__(:source), columns: Map.keys(hd(transformed_data)), data: transformed_data} end end - defp transform_table_name(table_name), do: @table_mappings[table_name] - - defp post_transform(table_name, row) do - schema = @schema_mappings[table_name] - + defp post_transform(schema, row) do default_fields = schema.__struct__() |> Map.from_struct() |> Map.take(schema.__schema__(:fields)) + default_fields = + if schema == User do + Map.delete(default_fields, :name) + else + default_fields + end + fields = row |> Enum.reject(fn {_, v} -> v == "\\N" end) + |> Enum.reject(fn {_, v} -> v == nil end) |> Map.new(fn {k, v} -> {k, v} end) - |> conditionally_rename_created_at() |> Map.take(Enum.map(Map.keys(default_fields), &Atom.to_string/1)) |> Map.new(fn {k, v} -> {String.to_existing_atom(k), v} end) - # Ensure handle is unique + # TODO: do we need this? fields = ensure_unique_handle(fields) Map.merge(default_fields, fields) end - defp conditionally_rename_created_at(fields) do - case {Map.get(fields, "inserted_at"), Map.get(fields, "created_at")} do - {nil, created_at} when not is_nil(created_at) -> - fields - |> Map.put("inserted_at", created_at) - |> Map.delete("created_at") - - _ -> - fields - end - end - defp ensure_unique_handle(fields) do - case fields[:handle] do - nil -> - fields - - handle -> - new_handle = get_unique_handle(handle) - Map.put(fields, :handle, new_handle) + if nullish?(fields[:handle]) do + fields + else + new_handle = get_unique_handle(fields[:handle]) + Map.put(fields, :handle, new_handle) end end @@ -351,19 +1126,13 @@ defmodule DatabaseMigration do new_handle = if count > 0, do: "#{handle}#{count + 1}", else: handle Process.put(:handles, Map.put(handles, downcased_handle, count + 1)) - new_handle - end - - defp rename_column(row, from, to) do - value = Map.get(row, from) + if count > 0 do + Logger.warning("Unique handle collision: #{handle} -> #{new_handle}") + end - row - |> Map.put(to, value) - |> Map.delete(from) + new_handle end - defp load_copy_section(nil), do: [] - defp load_copy_section(%{table: table_name, columns: columns, data: data}) do copy_statement = "COPY #{table_name} (#{Enum.join(columns, ", ")}) FROM stdin;\n" @@ -442,13 +1211,10 @@ defmodule DatabaseMigration do # |> Enum.into(%{}) # end - defp update_url_field(fields, field) do - case Map.get(fields, field) do - "/" <> rest -> - Map.put(fields, field, "https://console.algora.io/" <> rest) - - _ -> - fields + defp update_url(url) do + case url do + "/" <> rest -> "https://console.algora.io/" <> rest + _ -> url end end @@ -524,12 +1290,16 @@ defmodule DatabaseMigration do defp deserialize_value(value), do: value + defp nullish?(value), do: is_nil(deserialize_value(value)) + + defp or_else(value, default), do: if(nullish?(value), do: default, else: value) + defp clear_tables! do commands = [ "BEGIN TRANSACTION;", "SET CONSTRAINTS ALL DEFERRED;", - Enum.map(@backfilled_tables, &"TRUNCATE TABLE #{&1} CASCADE;"), + Enum.map(backfilled_tables(), &"TRUNCATE TABLE #{&1} CASCADE;"), "SET CONSTRAINTS ALL IMMEDIATE;", "COMMIT;" ] @@ -560,24 +1330,31 @@ defmodule DatabaseMigration do end end - def run! do - input_file = ".local/prod_db.sql" - output_file = ".local/prod_db_new.sql" + defp time_step(description, function) do + IO.puts("⏳ #{description}...") + {time, result} = :timer.tc(function) + IO.puts("✅ #{description} completed in #{time / 1_000_000} seconds") + result + end - if File.exists?(input_file) or File.exists?(output_file) do - IO.puts("Processing dump...") - :ok = process_dump(input_file, output_file) + def run! do + input_file = ".local/db/v1-data-2025-02-13.sql" + output_file = ".local/db/v2-data-2025-02-13.sql" - IO.puts("Clearing tables...") - :ok = clear_tables!() + System.put_env("MIGRATION", "true") - IO.puts("Importing new data...") - {:ok, _} = psql(["-f", output_file]) + if File.exists?(input_file) or File.exists?(output_file) do + IO.puts("⏳ Starting migration...") - IO.puts("Backfilling repositories...") - :ok = Algora.Admin.backfill_repos!() + {total_time, _} = + :timer.tc(fn -> + :ok = time_step("Processing dump", fn -> process_dump(input_file, output_file) end) + :ok = time_step("Clearing tables", fn -> clear_tables!() end) + {:ok, _} = time_step("Importing new data", fn -> psql(["-f", output_file]) end) + :ok = time_step("Backfilling repositories", fn -> Algora.Admin.backfill_repos!() end) + end) - IO.puts("Migration completed successfully") + IO.puts("✅ Migration completed successfully in #{total_time / 1_000_000} seconds") end end end diff --git a/scripts/parse_structure.exs b/scripts/parse_structure.exs new file mode 100644 index 000000000..0e25fbfca --- /dev/null +++ b/scripts/parse_structure.exs @@ -0,0 +1,61 @@ +defmodule SqlParser do + @moduledoc false + def parse_file(path) do + path + |> File.read!() + |> parse_tables() + |> filter_tables() + |> format_yaml() + |> IO.puts() + end + + defp parse_tables(content) do + # Match CREATE TABLE statements + regex = ~r/CREATE TABLE public\.([^(]+)\s*\((.*?)\);/s + + regex + |> Regex.scan(content, capture: :all_but_first) + |> Enum.map(fn [table_name, columns] -> + { + String.trim(table_name), + parse_columns(columns) + } + end) + end + + defp parse_columns(columns_str) do + # Match column definitions - captures quotes if present + regex = ~r/^\s*("?\w+"?)[^,]*/m + + regex + |> Regex.scan(columns_str, capture: :all_but_first) + |> List.flatten() + |> Enum.map(&String.trim/1) + end + + defp filter_tables(tables) do + Enum.reject(tables, fn {table_name, _columns} -> + String.ends_with?(table_name, "_activities") + end) + end + + defp format_yaml(tables) do + Enum.map_join(tables, "\n", fn {table_name, columns} -> + columns_yaml = + Enum.map_join(columns, "\n", fn column -> + " - #{column}: 0" + end) + + "- #{table_name}:\n#{columns_yaml}" + end) + end +end + +case System.argv() do + [filename] -> + SqlParser.parse_file(filename) + + _ -> + IO.puts("Usage: elixir parse_sql.exs ") + System.halt(1) +end diff --git a/scripts/v1-progress.yaml b/scripts/v1-progress.yaml new file mode 100644 index 000000000..76f6ab349 --- /dev/null +++ b/scripts/v1-progress.yaml @@ -0,0 +1,589 @@ +- "AccessToken": + - id: -1 + - created_at: -1 + - org_id: -1 + - name: -1 + - hash: -1 + - token_last_eight: -1 +- "Account": + - id: 1 + - created_at: 1 + - updated_at: 1 + - "userId": 1 + - type: -1 + - provider: 1 + - "providerAccountId": 1 + - refresh_token: -1 + - access_token: 1 + - expires_at: -1 + - token_type: -1 + - scope: -1 + - id_token: -1 + - session_state: -1 +- "Actor": + - id: -1 + - created_at: -1 + - updated_at: -1 + - user_id: -1 + - github_user_id: -1 + - org_id: -1 +- "Attempt": + - id: 1 + - created_at: 1 + - updated_at: 1 + - bounty_id: 1 + - github_user_id: 1 + - warnings_count: 1 + - status: 1 +- "Bid": + - id: -1 + - created_at: -1 + - updated_at: -1 + - user_id: -1 + - bounty_id: -1 + - amount: -1 + - currency: -1 + - status: -1 +- "Bonus": + - id: -1 + - created_at: -1 + - updated_at: -1 + - amount: -1 + - currency: -1 + - poster_id: -1 + - org_id: -1 + - claim_id: -1 + - github_req_comment_id: -1 + - github_res_comment_id: -1 + - type: -1 +- "BotMessage": + - id: -1 + - created_at: -1 + - updated_at: -1 + - active: -1 + - template: -1 + - type: -1 + - org_id: -1 +- "Bounty": + - id: 1 + - created_at: 1 + - updated_at: 1 + - poster_id: 1 + - org_id: 1 + - task_id: 1 + - status: 1 + - github_req_comment_id: 1 + - github_res_comment_id: 1 + - type: 0 + - kind: 0 + - visibility: 0 + - number: 1 + - deleted: 0 + - reward_type: 0 + - autopay_disabled: 0 + - manual_assignments: 0 + - timeouts_disabled: 0 + - experts_only: 0 +- "BountyCharge": + - id: 1 + - created_at: 1 + - updated_at: 1 + - succeeded_at: 1 + - charge_id: 1 + - user_id: -1 + - org_id: 1 + - amount: 1 + - currency: 1 + - receipt_url: 0 + - invoice_url: 0 +- "BountyProposal": + - id: -1 + - created_at: -1 + - updated_at: -1 + - org_id: -1 + - task_id: -1 + - amount: -1 + - currency: -1 + - note: -1 + - complexity: -1 + - importance: -1 + - steps: -1 +- "BountyTransfer": + - id: 1 + - created_at: 1 + - updated_at: 1 + - succeeded_at: 1 + - bounty_charge_id: 1 + - claim_id: 1 + - transfer_id: 1 + - amount: 1 + - currency: 1 +- "Claim": + - id: 1 + - created_at: 1 + - updated_at: 1 + - bounty_id: 1 + - status: 1 + - github_user_id: 1 + - github_url: 1 + - charge_id: -1 + - transfer_id: -1 + - charged_at: -1 + - transferred_at: -1 + - transfer_amount: -1 + - transfer_currency: -1 + - ready_to_pay: -1 + - github_pull_request_id: 1 + - bounty_charge_id: -1 + - github_res_comment_id: 0 + - info: 0 + - share: -1 +- "Company": + - id: -1 + - created_at: -1 + - updated_at: -1 + - org_id: -1 + - handle: -1 + - dates_talked: -1 + - contract_quoted: -1 + - cash_collected: -1 + - notes: -1 + - status: -1 + - success_checklist: -1 +- "Echo": + - id: -1 + - created_at: -1 + - message: -1 +- "Event": + - id: -1 + - created_at: -1 + - updated_at: -1 + - stream_id: -1 + - url: -1 + - actor_id: -1 + - name: -1 + - visibility: -1 + - payload: -1 + - pinned: -1 + - kind: -1 + - description: -1 + - emoji: -1 +- "EventDelivery": + - id: -1 + - created_at: -1 + - delivered_at: -1 + - event_id: -1 + - integration_id: -1 +- "Example": + - id: -1 + - created_at: -1 + - updated_at: -1 +- "GenericCharge": + - id: -1 + - created_at: -1 + - updated_at: -1 + - succeeded_at: -1 + - giver_id: -1 + - receiver_id: -1 + - stripe_id: -1 + - amount: -1 + - currency: -1 +- "GenericTransfer": + - id: -1 + - created_at: -1 + - updated_at: -1 + - succeeded_at: -1 + - charge_id: -1 + - receiver_id: -1 + - stripe_id: -1 + - amount: -1 + - currency: -1 +- "GithubInstallation": + - id: 1 + - created_at: 1 + - updated_at: 1 + - github_id: 1 + - org_id: 1 + - github_org_handle: -1 +- "GithubIssue": + - id: 1 + - url: -1 + - repository_url: -1 + - labels_url: -1 + - comments_url: -1 + - events_url: -1 + - html_url: 1 + - number: 1 + - state: -1 + - state_reason: -1 + - title: 1 + - body: 1 + - user_id: -1 + - locked: -1 + - active_lock_reason: -1 + - comments: -1 + - closed_at: -1 + - created_at: 1 + - updated_at: 1 + - draft: -1 + - body_html: -1 + - body_text: -1 + - timeline_url: -1 + - author_association: -1 + - task_id: -1 + - retrieved_at: -1 + - api_version: -1 + - is_assigned: -1 + - labels: -1 +- "GithubPullRequest": + - id: 1 + - url: -1 + - repository_url: -1 + - labels_url: -1 + - comments_url: -1 + - events_url: -1 + - html_url: 1 + - number: 1 + - state: -1 + - state_reason: -1 + - title: 1 + - body: 1 + - user_id: -1 + - locked: -1 + - active_lock_reason: -1 + - comments: -1 + - closed_at: -1 + - created_at: 1 + - updated_at: 1 + - draft: -1 + - body_html: -1 + - body_text: -1 + - timeline_url: -1 + - author_association: -1 + - merged_at: -1 + - diff_url: -1 + - patch_url: -1 + - task_id: -1 + - retrieved_at: -1 + - api_version: -1 + - is_assigned: -1 + - labels: -1 +- "GithubUser": + - id: 1 + - login: 1 + - avatar_url: 1 + - gravatar_id: -1 + - url: -1 + - html_url: -1 + - followers_url: -1 + - following_url: -1 + - gists_url: -1 + - starred_url: -1 + - subscriptions_url: -1 + - organizations_url: -1 + - repos_url: -1 + - events_url: -1 + - received_events_url: -1 + - type: 0 + - site_admin: -1 + - name: 1 + - email: 0 + - user_id: -1 + - retrieved_at: 1 + - api_version: -1 + - bio: 1 + - company: -1 + - location: 1 + - twitter_username: 1 + - blocked_from_attempting: -1 + - blocked_from_earning: -1 +- "Integration": + - id: -1 + - created_at: -1 + - updated_at: -1 + - org_id: -1 + - user_id: -1 + - name: -1 + - url: -1 + - kind: -1 + - scope: -1 + - state: -1 + - token: -1 + - status: -1 +- "IssueSummary": + - id: -1 + - created_at: -1 + - updated_at: -1 + - body: -1 + - complexity: -1 + - importance: -1 + - steps: -1 +- "Org": + - id: 1 + - created_at: 1 + - updated_at: 1 + - handle: 1 + - creator_id: -1 + - name: 1 + - avatar_url: 1 + - domain: 1 + - webhook_url: -1 + - github_id: 1 + - github_data: 1 + - github_data_fetched: -1 + - fee_pct: 1 + - featured: 1 + - accepts_sponsorships: -1 + - tech: 1 + - description: 1 + - stargazers_count: 1 + - priority: 1 + - supervise_attempts: 0 + - manual_assignments: 1 + - is_personal: 0 + - max_open_attempts: 1 + - enabled_expert_recs: 0 + - enabled_private_bounties: 0 + - fee_pct_prev: -1 + - days_until_timeout: 0 + - days_until_timeout_reminder: 0 + - discord_url: 1 + - twitter_url: 1 + - website_url: 1 + - youtube_url: 1 + - enabled_community_mode: 0 + - webhook_url_discord: -1 + - webhook_url_slack: -1 + - accepts_community_bounties: 0 + - seeded: 1 + - slack_url: 1 + - github_handle: 1 + - active: 1 +- "OrgBalanceTransaction": + - id: 1 + - created_at: 1 + - org_id: 1 + - amount: 1 + - currency: 1 + - type: -1 + - effect: -1 + - tier: -1 + - description: -1 + - user_id: -1 + - plan: -1 +- "OrgEventSubscription": + - id: -1 + - created_at: -1 + - updated_at: -1 + - user_id: -1 + - org_id: -1 + - member_id: -1 + - kind: -1 +- "OrgMember": + - id: 1 + - created_at: 1 + - updated_at: 1 + - org_id: 1 + - user_id: 1 + - weight: -1 + - role: 1 +- "PointReward": + - id: -1 + - created_at: -1 + - updated_at: -1 + - amount: -1 + - bounty_id: -1 +- "Reward": + - id: -1 + - created_at: -1 + - updated_at: -1 + - amount: 1 + - bounty_id: 1 + - currency: 1 +- "RewardTier": + - id: -1 + - created_at: -1 + - updated_at: -1 + - amount: -1 + - currency: -1 + - bounty_id: -1 + - lower_bound: -1 + - upper_bound: -1 + - metric: -1 +- "Session": + - id: -1 + - created_at: -1 + - updated_at: -1 + - "sessionToken": -1 + - "userId": -1 + - expires: -1 +- "Sponsorship": + - id: -1 + - created_at: -1 + - updated_at: -1 + - succeeded_at: -1 + - creator_id: -1 + - receiver_id: -1 + - charge_id: -1 + - user_id: -1 + - amount: -1 + - currency: -1 +- "SponsorshipTransfer": + - id: -1 + - created_at: -1 + - updated_at: -1 + - succeeded_at: -1 + - sponsorship_id: -1 + - receiver_id: -1 + - transfer_id: -1 + - amount: -1 + - currency: -1 +- "StripeAccount": + - id: 1 + - created_at: 1 + - updated_at: 1 + - user_id: 1 + - details_submitted: 1 + - charges_enabled: 1 + - service_agreement: 1 + - country: 1 + - type: 1 + - region: -1 + - needs_refresh: 1 +- "StripeBalanceTransaction": + - stripe_id: -1 + - created: -1 + - currency: -1 + - amount: -1 + - fee: -1 + - net: -1 + - status: -1 + - type: -1 + - id: -1 + - period: -1 + - source: -1 +- "StripeCustomer": + - id: 1 + - created_at: 1 + - updated_at: 1 + - stripe_id: 1 + - org_id: 1 + - name: 1 + - region: -1 +- "StripePaymentMethod": + - id: 1 + - created_at: 1 + - updated_at: 1 + - stripe_id: 1 + - org_id: -1 + - is_default: 1 +- "StripeSubscription": + - id: -1 + - created_at: -1 + - updated_at: -1 + - charged_at: -1 + - charge_id: -1 + - customer_id: -1 + - amount: -1 + - plan: -1 + - active: -1 + - currency: -1 +- "SurveyResponse": + - id: -1 + - created_at: -1 + - updated_at: -1 + - email: -1 + - payload: -1 +- "SystemEvent": + - id: -1 + - created_at: -1 + - tag: -1 + - origin: -1 + - origin_id: -1 + - status: -1 +- "SystemEventDelivery": + - id: -1 + - created_at: -1 + - event_id: -1 + - status: -1 + - message: -1 + - payload: -1 +- "Task": + - id: 1 + - forge: 1 + - repo_owner: 1 + - repo_name: 1 + - number: 1 + - issue_id: 1 + - pull_request_id: 1 + - source: 0 + - image: 0 + - status: 0 + - body: 1 + - slug: 0 + - title: 1 + - og_image: 0 + - tech: 0 +- "User": + - id: 1 + - created_at: 1 + - updated_at: 1 + - handle: 1 + - name: 1 + - email: 1 + - "emailVerified": 1 + - image: 1 + - is_admin: 1 + - loc: 1 + - last_activity_at: 0 + - country: 1 + - twitch_handle: 1 + - youtube_handle: 1 + - tech: 1 + - contribs_last_mo: -1 + - stars_earned: 1 + - discovery_sort_by: -1 + - discovery_tech: -1 + - alipay_account_holder_name: -1 + - alipay_id: -1 + - wise_account_holder_name: -1 + - wise_email: -1 + - onboarding: -1 + - can_create_org: -1 +- "UserActivity": + - id: -1 + - created_at: -1 + - updated_at: -1 + - user_id: -1 + - ip: -1 +- "UserEventSubscription": + - id: -1 + - created_at: -1 + - updated_at: -1 + - last_notif_at: -1 + - user_id: -1 + - frequency: -1 + - kind: -1 +- "UserExpertise": + - id: -1 + - created_at: -1 + - updated_at: -1 + - language: -1 + - user_id: -1 +- "UserLanguage": + - id: -1 + - created_at: -1 + - updated_at: -1 + - language: -1 + - user_id: -1 + - pull_requests: -1 +- "VerificationToken": + - identifier: -1 + - token: -1 + - expires: -1 +- "_Event_participants": + - "A": -1 + - "B": -1 +- "_Event_subscribers": + - "A": -1 + - "B": -1 diff --git a/scripts/v2-progress.yaml b/scripts/v2-progress.yaml new file mode 100644 index 000000000..116c20a48 --- /dev/null +++ b/scripts/v2-progress.yaml @@ -0,0 +1,292 @@ +- accounts: + - id: 1 + - provider: 1 + - provider_id: 1 + - provider_meta: -1 + - name: -1 + - details_submitted: 1 + - charges_enabled: 1 + - service_agreement: 1 + - country: 1 + - type: 1 + - stale: 1 + - user_id: 1 + - inserted_at: 1 + - updated_at: 1 + - payouts_enabled: -1 + - payout_interval: -1 + - payout_speed: -1 + - default_currency: -1 +- attempts: + - id: 1 + - status: 1 + - warnings_count: 1 + - ticket_id: 1 + - user_id: 1 + - inserted_at: 1 + - updated_at: 1 +- bounties: + - id: 1 + - amount: 1 + - status: 1 + - ticket_id: 1 + - owner_id: 1 + - creator_id: 1 + - inserted_at: 1 + - updated_at: 1 +- chat_participants: + - id: -1 + - last_read_at: -1 + - thread_id: -1 + - user_id: -1 + - inserted_at: -1 + - updated_at: -1 +- claims: + - id: 1 + - status: 1 + - type: 1 + - url: 1 + - group_id: 1 + - group_share: -1 + - source_id: 1 + - target_id: 1 + - user_id: 1 + - inserted_at: 1 + - updated_at: 1 +- command_responses: + - id: 1 + - provider: 1 + - provider_meta: -1 + - provider_command_id: 1 + - provider_response_id: 1 + - command_source: 1 + - command_type: 1 + - ticket_id: 1 + - inserted_at: 1 + - updated_at: 1 +- comment_cursors: + - id: -1 + - provider: -1 + - repo_owner: -1 + - repo_name: -1 + - "timestamp": -1 + - last_polled_at: -1 + - last_comment_id: -1 + - inserted_at: -1 + - updated_at: -1 +- contracts: + - id: -1 + - status: -1 + - sequence_number: -1 + - hourly_rate: -1 + - hourly_rate_min: -1 + - hourly_rate_max: -1 + - hours_per_week: -1 + - start_date: -1 + - end_date: -1 + - original_contract_id: -1 + - client_id: -1 + - contractor_id: -1 + - inserted_at: -1 + - updated_at: -1 +- customers: + - id: 1 + - provider: 1 + - provider_id: 1 + - provider_meta: -1 + - name: 1 + - user_id: 1 + - inserted_at: 1 + - updated_at: 1 +- event_cursors: + - id: -1 + - provider: -1 + - repo_owner: -1 + - repo_name: -1 + - last_event_id: -1 + - last_polled_at: -1 + - inserted_at: -1 + - updated_at: -1 +- identities: + - id: 1 + - user_id: 1 + - provider: 1 + - provider_token: 1 + - provider_email: 1 + - provider_login: 1 + - provider_id: 1 + - provider_meta: 1 + - inserted_at: 1 + - updated_at: 1 +- installations: + - id: 1 + - provider: 1 + - provider_id: 1 + - provider_meta: -1 + - avatar_url: -1 + - repository_selection: -1 + - owner_id: -2 + - connected_user_id: 1 + - inserted_at: 1 + - updated_at: 1 + - provider_user_id: -1 +- members: + - id: 1 + - role: 1 + - org_id: 1 + - user_id: 1 + - inserted_at: 1 + - updated_at: 1 +- messages: + - id: -1 + - content: -1 + - thread_id: -1 + - sender_id: -1 + - inserted_at: -1 + - updated_at: -1 +- payment_methods: + - id: 1 + - provider: 1 + - provider_id: 1 + - provider_meta: -1 + - provider_customer_id: 1 + - is_default: 1 + - customer_id: 1 + - inserted_at: 1 + - updated_at: 1 +- repositories: + - id: -1 + - provider: -1 + - provider_id: -1 + - provider_meta: -1 + - name: -1 + - url: -1 + - user_id: -1 + - inserted_at: -1 + - updated_at: -1 + - description: -1 + - og_image_url: -1 + - og_image_updated_at: -1 +- reviews: + - id: -1 + - rating: -1 + - content: -1 + - visibility: -1 + - contract_id: -1 + - bounty_id: -1 + - organization_id: -1 + - reviewer_id: -1 + - reviewee_id: -1 + - inserted_at: -1 + - updated_at: -1 +- threads: + - id: -1 + - title: -1 + - inserted_at: -1 + - updated_at: -1 +- tickets: + - id: 1 + - provider: 1 + - provider_id: 1 + - provider_meta: 1 + - type: 1 + - title: 1 + - description: 1 + - number: 1 + - url: 1 + - repository_id: -1 + - inserted_at: 1 + - updated_at: 1 +- timesheets: + - id: -1 + - hours_worked: -1 + - description: -1 + - contract_id: -1 + - inserted_at: -1 + - updated_at: -1 +- tips: + - id: 1 + - amount: 1 + - status: -2 + - ticket_id: 1 + - owner_id: 1 + - creator_id: 1 + - recipient_id: 1 + - inserted_at: 1 + - updated_at: 1 +- transactions: + - id: 1 + - provider: 1 + - provider_id: 1 + - provider_charge_id: 1 + - provider_payment_intent_id: -1 + - provider_transfer_id: 1 + - provider_invoice_id: -1 + - provider_balance_transaction_id: -1 + - provider_meta: -1 + - gross_amount: 1 + - net_amount: 1 + - total_fee: 1 + - provider_fee: -1 + - line_items: -1 + - type: 1 + - status: 1 + - succeeded_at: 1 + - reversed_at: -1 + - group_id: 1 + - user_id: 1 + - contract_id: -1 + - original_contract_id: -1 + - timesheet_id: -1 + - bounty_id: 1 + - tip_id: 1 + - linked_transaction_id: 1 + - inserted_at: 1 + - updated_at: 1 + - claim_id: 1 +- users: + - id: 1 + - provider: 1 + - provider_id: 1 + - provider_login: 1 + - provider_meta: 1 + - email: 1 + - display_name: 1 + - handle: 1 + - avatar_url: 1 + - external_homepage_url: -2 + - type: 1 + - bio: 1 + - location: 1 + - country: 1 + - timezone: 1 + - stargazers_count: 1 + - domain: 1 + - tech_stack: 1 + - featured: 1 + - priority: 1 + - fee_pct: 1 + - seeded: 1 + - activated: 1 + - max_open_attempts: 1 + - manual_assignment: 1 + - bounty_mode: -1 + - hourly_rate_min: -1 + - hourly_rate_max: -1 + - hours_per_week: -1 + - website_url: 1 + - twitter_url: 1 + - github_url: -2 + - youtube_url: 1 + - twitch_url: 1 + - discord_url: 1 + - slack_url: 1 + - linkedin_url: -1 + - og_title: -1 + - og_image_url: -1 + - last_context: 1 + - need_avatar: -2 + - inserted_at: 1 + - updated_at: 1 + - name: -1 + - is_admin: 1