From aa319195b4e360723ff72c2c9ed3b1447bbdaa40 Mon Sep 17 00:00:00 2001 From: zafer Date: Fri, 7 Feb 2025 21:03:47 +0300 Subject: [PATCH 01/51] in midst of getting migration script up to date --- scripts/database_migration.exs | 228 +++++++++++++++++++++++++-------- 1 file changed, 172 insertions(+), 56 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index 3975e5801..2c7ae2ed7 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -92,6 +92,7 @@ defmodule DatabaseMigration do true -> row # TODO: maybe discard altogther? + |> Map.put("url", "https://github.com/#{row["repo_owner"]}/#{row["repo_name"]}/issues/#{row["number"]}") |> Map.put("inserted_at", "1970-01-01 00:00:00") |> Map.put("updated_at", "1970-01-01 00:00:00") end @@ -100,43 +101,161 @@ defmodule DatabaseMigration do end defp transform("User", row, db) do - github_user = - db |> Map.get("GithubUser", []) |> Enum.find(&(&1["user_id"] == row["id"])) - - row = - if github_user do - row - |> Map.put("provider", "github") - |> Map.put("provider_id", github_user["id"]) - |> Map.put("provider_login", github_user["login"]) - |> Map.put("provider_meta", deserialize_value(github_user)) - else - row - end - - row - |> Map.put("type", "individual") - |> rename_column("tech", "tech_stack") - |> rename_column("stars_earned", "stargazers_count") - |> rename_column("image", "avatar_url") - |> update_url_field("avatar_url") + # TODO: reenable + # if !row["\"emailVerified\""] || String.length(row["\"emailVerified\""]) < 10 do + # raise "Email not verified: #{inspect(row)}" + # end + + github_user = db |> Map.get("GithubUser", []) |> Enum.find(&(&1["user_id"] == row["id"])) + + %{ + "id" => row["id"], + "provider" => github_user && "github", + "provider_id" => github_user && github_user["id"], + "provider_login" => github_user && github_user["login"], + "provider_meta" => github_user && deserialize_value(github_user), + "email" => row["email"], + "display_name" => row["name"], + "handle" => row["handle"], + "avatar_url" => update_url(row["image"]), + "external_homepage_url" => nil, + "type" => "individual", + "bio" => github_user && github_user["bio"], + "location" => row["loc"], + "country" => row["country"], + "timezone" => nil, + "stargazers_count" => row["stars_earned"], + "domain" => nil, + "tech_stack" => row["tech"], + "featured" => nil, + "priority" => nil, + "fee_pct" => nil, + "seeded" => nil, + "activated" => nil, + "max_open_attempts" => nil, + "manual_assignment" => nil, + "bounty_mode" => nil, + "hourly_rate_min" => nil, + "hourly_rate_max" => nil, + "hours_per_week" => nil, + "website_url" => nil, + "twitter_url" => nil, + "github_url" => nil, + "youtube_url" => row["youtube_handle"] && "https://www.youtube.com/#{row["youtube_handle"]}", + "twitch_url" => row["twitch_handle"] && "https://www.twitch.tv/#{row["twitch_handle"]}", + "discord_url" => nil, + "slack_url" => nil, + "linkedin_url" => nil, + "og_title" => nil, + "og_image_url" => nil, + "last_context" => row["handle"], + "need_avatar" => nil, + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"], + "is_admin" => row["is_admin"] + } end defp transform("Org", row, _db) do - row - |> Map.put("type", "organization") - |> Map.put("provider", row["github_handle"] && "github") - |> rename_column("github_handle", "provider_login") - |> rename_column("tech", "tech_stack") - |> update_url_field("avatar_url") + %{ + "id" => row["id"], + "provider" => row["github_handle"] && "github", + "provider_id" => row["github_id"], + "provider_login" => row["github_handle"], + "provider_meta" => row["github_data"] && deserialize_value(row["github_data"]), + "email" => nil, + "display_name" => row["name"], + "handle" => row["handle"], + "avatar_url" => update_url(row["avatar_url"]), + "external_homepage_url" => nil, + "type" => "organization", + "bio" => row["description"], + "location" => nil, + "country" => nil, + "timezone" => nil, + "stargazers_count" => row["stargazers_count"], + "domain" => row["domain"], + "tech_stack" => row["tech"], + "featured" => row["featured"], + "priority" => row["priority"], + "fee_pct" => row["fee_pct"], + "seeded" => row["seeded"], + "activated" => row["active"], + "max_open_attempts" => row["max_open_attempts"], + "manual_assignment" => row["manual_assignment"], + "bounty_mode" => nil, + "hourly_rate_min" => nil, + "hourly_rate_max" => nil, + "hours_per_week" => nil, + "website_url" => row["website_url"], + "twitter_url" => row["twitter_url"], + "github_url" => nil, + "youtube_url" => row["youtube_url"], + "twitch_url" => nil, + "discord_url" => row["discord_url"], + "slack_url" => row["slack_url"], + "linkedin_url" => nil, + "og_title" => nil, + "og_image_url" => nil, + "last_context" => nil, + "need_avatar" => nil, + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"], + "is_admin" => false + } end defp transform("GithubUser", %{user_id: nil} = row, _db) do - row - |> Map.put("type", "individual") - |> Map.put("provider", "github") - |> Map.put("provider_id", row["id"]) - |> Map.put("provider_meta", deserialize_value(row)) + if row["type"] != "User" do + raise "GithubUser is not a User: #{inspect(row)}" + end + + %{ + "id" => row["id"], + "provider" => "github", + "provider_id" => row["id"], + "provider_login" => row["login"], + "provider_meta" => deserialize_value(row), + "email" => nil, + "display_name" => row["name"], + "handle" => nil, + "avatar_url" => row["avatar_url"], + "external_homepage_url" => nil, + "type" => "individual", + "bio" => row["bio"], + "location" => row["location"], + "country" => nil, + "timezone" => nil, + "stargazers_count" => nil, + "domain" => nil, + "tech_stack" => nil, + "featured" => nil, + "priority" => nil, + "fee_pct" => nil, + "seeded" => nil, + "activated" => nil, + "max_open_attempts" => nil, + "manual_assignment" => nil, + "bounty_mode" => nil, + "hourly_rate_min" => nil, + "hourly_rate_max" => nil, + "hours_per_week" => nil, + "website_url" => nil, + "twitter_url" => row["twitter_username"] && "https://www.twitter.com/#{row["twitter_username"]}", + "github_url" => nil, + "youtube_url" => nil, + "twitch_url" => nil, + "discord_url" => nil, + "slack_url" => nil, + "linkedin_url" => nil, + "og_title" => nil, + "og_image_url" => nil, + "last_context" => nil, + "need_avatar" => nil, + "inserted_at" => row["retrieved_at"], + "updated_at" => row["retrieved_at"], + "is_admin" => nil + } end defp transform("GithubUser", _row, _db), do: nil @@ -274,14 +393,14 @@ defmodule DatabaseMigration do transformed_data = data |> Enum.map(fn row -> - try do - transform(table, row, db) - rescue - e -> - IO.puts("Error transforming row in table #{table}: #{inspect(row)}") - IO.puts("Error: #{inspect(e)}") - nil - end + # try do + transform(table, row, db) + # rescue + # e -> + # IO.puts("Error transforming row in table #{table}: #{inspect(row)}") + # IO.puts("Error: #{inspect(e)}") + # nil + # end end) |> Enum.reject(&is_nil/1) |> Enum.map(&post_transform(table, &1)) @@ -306,9 +425,17 @@ defmodule DatabaseMigration do |> Map.from_struct() |> Map.take(schema.__schema__(:fields)) + default_fields = + if schema == User do + Map.delete(default_fields, :name) + else + default_fields + end + fields = row |> Enum.reject(fn {_, v} -> v == "\\N" end) + |> Enum.reject(fn {_, v} -> v == nil end) |> Map.new(fn {k, v} -> {k, v} end) |> conditionally_rename_created_at() |> Map.take(Enum.map(Map.keys(default_fields), &Atom.to_string/1)) @@ -354,14 +481,6 @@ defmodule DatabaseMigration do new_handle end - defp rename_column(row, from, to) do - value = Map.get(row, from) - - row - |> Map.put(to, value) - |> Map.delete(from) - end - defp load_copy_section(nil), do: [] defp load_copy_section(%{table: table_name, columns: columns, data: data}) do @@ -442,13 +561,10 @@ defmodule DatabaseMigration do # |> Enum.into(%{}) # end - defp update_url_field(fields, field) do - case Map.get(fields, field) do - "/" <> rest -> - Map.put(fields, field, "https://console.algora.io/" <> rest) - - _ -> - fields + defp update_url(url) do + case url do + "/" <> rest -> "https://console.algora.io/" <> rest + _ -> url end end @@ -565,8 +681,8 @@ defmodule DatabaseMigration do output_file = ".local/prod_db_new.sql" if File.exists?(input_file) or File.exists?(output_file) do - IO.puts("Processing dump...") - :ok = process_dump(input_file, output_file) + # IO.puts("Processing dump...") + # :ok = process_dump(input_file, output_file) IO.puts("Clearing tables...") :ok = clear_tables!() From c9a4b53f5ae917cd9eb4a9c18aa89ca1e16e2d4c Mon Sep 17 00:00:00 2001 From: zafer Date: Fri, 7 Feb 2025 21:04:03 +0300 Subject: [PATCH 02/51] add scripts to track progress --- mix.exs | 4 +- mix.lock | 2 + scripts/analyze_progress.exs | 85 ++++++++++++++++++++++++++++++++++++ scripts/parse_structure.exs | 61 ++++++++++++++++++++++++++ 4 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 scripts/analyze_progress.exs create mode 100644 scripts/parse_structure.exs diff --git a/mix.exs b/mix.exs index 5fc91e661..4e00827cb 100644 --- a/mix.exs +++ b/mix.exs @@ -94,7 +94,9 @@ defmodule Algora.MixProject do # monitoring, logging {:appsignal_phoenix, "~> 2.6"}, {:logfmt_ex, "~> 0.4"}, - {:oban_live_dashboard, "~> 0.1.0"} + {:oban_live_dashboard, "~> 0.1.0"}, + # TODO: delete after migration + {:yaml_elixir, "~> 2.9"} ] end diff --git a/mix.lock b/mix.lock index 31257ccfc..e9a713e17 100644 --- a/mix.lock +++ b/mix.lock @@ -88,4 +88,6 @@ "uri_query": {:hex, :uri_query, "0.1.2", "ae35b83b472f3568c2c159eee3f3ccf585375d8a94fb5382db1ea3589e75c3b4", [:mix], [], "hexpm", "e3bc81816c98502c36498b9b2f239b89c71ce5eadfff7ceb2d6c0a2e6ae2ea0c"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, "websock_adapter": {:hex, :websock_adapter, "0.5.8", "3b97dc94e407e2d1fc666b2fb9acf6be81a1798a2602294aac000260a7c4a47d", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "315b9a1865552212b5f35140ad194e67ce31af45bcee443d4ecb96b5fd3f3782"}, + "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"}, + "yaml_elixir": {:hex, :yaml_elixir, "2.11.0", "9e9ccd134e861c66b84825a3542a1c22ba33f338d82c07282f4f1f52d847bd50", [:mix], [{:yamerl, "~> 0.10", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "53cc28357ee7eb952344995787f4bb8cc3cecbf189652236e9b163e8ce1bc242"}, } diff --git a/scripts/analyze_progress.exs b/scripts/analyze_progress.exs new file mode 100644 index 000000000..82207aabd --- /dev/null +++ b/scripts/analyze_progress.exs @@ -0,0 +1,85 @@ +defmodule ProgressAnalyzer do + @moduledoc false + def analyze_file(path) do + stats = + path + |> File.read!() + |> YamlElixir.read_from_string!() + |> count_statuses() + + display_stats(stats) + end + + defp count_statuses(yaml) do + # Initialize counters + initial_counts = %{completed: 0, wontdo: 0, remaining: 0, undecided: 0} + + # Flatten and count all column statuses + Enum.reduce(yaml, initial_counts, fn table, acc -> + columns = table |> Map.values() |> List.first() + + Enum.reduce(columns, acc, fn column, inner_acc -> + {_col, status} = Enum.at(Map.to_list(column), 0) + + case status do + 1 -> Map.update!(inner_acc, :completed, &(&1 + 1)) + -2 -> Map.update!(inner_acc, :undecided, &(&1 + 1)) + -1 -> Map.update!(inner_acc, :wontdo, &(&1 + 1)) + 0 -> Map.update!(inner_acc, :remaining, &(&1 + 1)) + end + end) + end) + end + + defp display_stats(%{completed: done, wontdo: wont, remaining: todo, undecided: undecided}) do + total = done + wont + todo + undecided + done_pct = percentage(done, total) + wont_pct = percentage(wont, total) + todo_pct = percentage(todo, total) + undecided_pct = percentage(undecided, total) + + IO.puts(""" + Migration Progress Report + ======================== + + Summary: + -------- + Total columns: #{total} + + Status Breakdown: + ---------------- + ✅ Completed: #{done} (#{done_pct}%) + ⏳ Remaining: #{todo} (#{todo_pct}%) + ❌ Won't Do: #{wont} (#{wont_pct}%) + ❓ Undecided: #{undecided} (#{undecided_pct}%) + Progress: + --------- + [#{progress_bar(done_pct, wont_pct, todo_pct, undecided_pct)}] + """) + end + + defp percentage(part, total) when total > 0 do + Float.round(part / total * 100, 1) + end + + defp progress_bar(done_pct, wont_pct, todo_pct, undecided_pct) do + done_chars = round(done_pct / 2) + wont_chars = round(wont_pct / 2) + todo_chars = round(todo_pct / 2) + undecided_chars = round(undecided_pct / 2) + + String.duplicate("=", done_chars) <> + String.duplicate("x", wont_chars) <> + String.duplicate("?", undecided_chars) <> + String.duplicate(".", todo_chars) + end +end + +case System.argv() do + [filename] -> + ProgressAnalyzer.analyze_file(filename) + + _ -> + IO.puts("Usage: elixir analyze_progress.exs ") + System.halt(1) +end diff --git a/scripts/parse_structure.exs b/scripts/parse_structure.exs new file mode 100644 index 000000000..0e25fbfca --- /dev/null +++ b/scripts/parse_structure.exs @@ -0,0 +1,61 @@ +defmodule SqlParser do + @moduledoc false + def parse_file(path) do + path + |> File.read!() + |> parse_tables() + |> filter_tables() + |> format_yaml() + |> IO.puts() + end + + defp parse_tables(content) do + # Match CREATE TABLE statements + regex = ~r/CREATE TABLE public\.([^(]+)\s*\((.*?)\);/s + + regex + |> Regex.scan(content, capture: :all_but_first) + |> Enum.map(fn [table_name, columns] -> + { + String.trim(table_name), + parse_columns(columns) + } + end) + end + + defp parse_columns(columns_str) do + # Match column definitions - captures quotes if present + regex = ~r/^\s*("?\w+"?)[^,]*/m + + regex + |> Regex.scan(columns_str, capture: :all_but_first) + |> List.flatten() + |> Enum.map(&String.trim/1) + end + + defp filter_tables(tables) do + Enum.reject(tables, fn {table_name, _columns} -> + String.ends_with?(table_name, "_activities") + end) + end + + defp format_yaml(tables) do + Enum.map_join(tables, "\n", fn {table_name, columns} -> + columns_yaml = + Enum.map_join(columns, "\n", fn column -> + " - #{column}: 0" + end) + + "- #{table_name}:\n#{columns_yaml}" + end) + end +end + +case System.argv() do + [filename] -> + SqlParser.parse_file(filename) + + _ -> + IO.puts("Usage: elixir parse_sql.exs ") + System.halt(1) +end From b24a15a8b2865cf9bbb144fe2757bfdf82f41757 Mon Sep 17 00:00:00 2001 From: zafer Date: Fri, 7 Feb 2025 21:04:15 +0300 Subject: [PATCH 03/51] use cached http method during migration --- lib/algora/integrations/github/client.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/algora/integrations/github/client.ex b/lib/algora/integrations/github/client.ex index c4703f9d1..d18af5d3c 100644 --- a/lib/algora/integrations/github/client.ex +++ b/lib/algora/integrations/github/client.ex @@ -9,7 +9,7 @@ defmodule Algora.Github.Client do @type token :: String.t() # TODO: move to a separate module and use only for data migration between databases - def http_cached(host, method, path, headers, body, opts \\ []) do + def http(host, method, path, headers, body, opts \\ []) do cache_path = ".local/github/#{path}.json" with :error <- read_from_cache(cache_path), @@ -22,7 +22,7 @@ defmodule Algora.Github.Client do end end - def http(host, method, path, headers, body, opts \\ []) do + def http0(host, method, path, headers, body, opts \\ []) do do_http_request(host, method, path, headers, body, opts) end From 6a24ac09e12003c5ac0efce381a4c6a783d59a6b Mon Sep 17 00:00:00 2001 From: zafer Date: Fri, 7 Feb 2025 22:03:43 +0300 Subject: [PATCH 04/51] in midst of getting migration script up to date --- scripts/database_migration.exs | 68 +++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 25 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index 2c7ae2ed7..ebbd1385b 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -66,35 +66,53 @@ defmodule DatabaseMigration do row = cond do github_issue -> - row - |> Map.put("type", "issue") - |> Map.put("title", github_issue["title"]) - |> Map.put("description", github_issue["body"]) - |> Map.put("inserted_at", github_issue["created_at"]) - |> Map.put("updated_at", github_issue["updated_at"]) - |> Map.put("url", github_issue["html_url"]) - |> Map.put("provider", "github") - |> Map.put("provider_id", github_issue["id"]) - |> Map.put("provider_meta", deserialize_value(github_issue)) + %{ + "id" => row["id"], + "provider" => "github", + "provider_id" => github_issue["id"], + "provider_meta" => deserialize_value(github_issue), + "type" => "issue", + "title" => github_issue["title"], + "description" => github_issue["body"], + "number" => github_issue["number"], + "url" => github_issue["html_url"], + "inserted_at" => github_issue["created_at"], + "updated_at" => github_issue["updated_at"] + } github_pull_request -> - row - |> Map.put("type", "pull_request") - |> Map.put("title", github_pull_request["title"]) - |> Map.put("description", github_pull_request["body"]) - |> Map.put("inserted_at", github_pull_request["created_at"]) - |> Map.put("updated_at", github_pull_request["updated_at"]) - |> Map.put("url", github_pull_request["html_url"]) - |> Map.put("provider", "github") - |> Map.put("provider_id", github_pull_request["id"]) - |> Map.put("provider_meta", deserialize_value(github_pull_request)) + %{ + "id" => row["id"], + "provider" => "github", + "provider_id" => github_pull_request["id"], + "provider_meta" => deserialize_value(github_pull_request), + "type" => "pull_request", + "title" => github_pull_request["title"], + "description" => github_pull_request["body"], + "number" => github_pull_request["number"], + "url" => github_pull_request["html_url"], + "inserted_at" => github_pull_request["created_at"], + "updated_at" => github_pull_request["updated_at"] + } true -> - row - # TODO: maybe discard altogther? - |> Map.put("url", "https://github.com/#{row["repo_owner"]}/#{row["repo_name"]}/issues/#{row["number"]}") - |> Map.put("inserted_at", "1970-01-01 00:00:00") - |> Map.put("updated_at", "1970-01-01 00:00:00") + if row["forge"] != "github" do + raise "Unknown forge: #{row["forge"]}" + end + + %{ + "id" => row["id"], + "provider" => row["forge"], + "provider_id" => nil, + "provider_meta" => nil, + "type" => "issue", + "title" => row["title"], + "description" => row["body"], + "number" => row["number"], + "url" => "https://github.com/#{row["repo_owner"]}/#{row["repo_name"]}/issues/#{row["number"]}", + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"] + } end row From 0d327081b17f0c94ac1ef38adaad47118a12c79a Mon Sep 17 00:00:00 2001 From: zafer Date: Fri, 7 Feb 2025 22:03:55 +0300 Subject: [PATCH 05/51] update analyze_progress script --- scripts/analyze_progress.exs | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/scripts/analyze_progress.exs b/scripts/analyze_progress.exs index 82207aabd..f28291d2b 100644 --- a/scripts/analyze_progress.exs +++ b/scripts/analyze_progress.exs @@ -12,7 +12,7 @@ defmodule ProgressAnalyzer do defp count_statuses(yaml) do # Initialize counters - initial_counts = %{completed: 0, wontdo: 0, remaining: 0, undecided: 0} + initial_counts = %{completed: 0, skipped: 0, remaining: 0, undecided: 0} # Flatten and count all column statuses Enum.reduce(yaml, initial_counts, fn table, acc -> @@ -24,17 +24,17 @@ defmodule ProgressAnalyzer do case status do 1 -> Map.update!(inner_acc, :completed, &(&1 + 1)) -2 -> Map.update!(inner_acc, :undecided, &(&1 + 1)) - -1 -> Map.update!(inner_acc, :wontdo, &(&1 + 1)) + -1 -> Map.update!(inner_acc, :skipped, &(&1 + 1)) 0 -> Map.update!(inner_acc, :remaining, &(&1 + 1)) end end) end) end - defp display_stats(%{completed: done, wontdo: wont, remaining: todo, undecided: undecided}) do - total = done + wont + todo + undecided + defp display_stats(%{completed: done, skipped: skipped, remaining: todo, undecided: undecided}) do + total = done + skipped + todo + undecided done_pct = percentage(done, total) - wont_pct = percentage(wont, total) + skipped_pct = percentage(skipped, total) todo_pct = percentage(todo, total) undecided_pct = percentage(undecided, total) @@ -48,28 +48,29 @@ defmodule ProgressAnalyzer do Status Breakdown: ---------------- - ✅ Completed: #{done} (#{done_pct}%) - ⏳ Remaining: #{todo} (#{todo_pct}%) - ❌ Won't Do: #{wont} (#{wont_pct}%) - ❓ Undecided: #{undecided} (#{undecided_pct}%) + ✅ Completed: #{done_pct}% (#{done}) + ⏳ Remaining: #{todo_pct}% (#{todo}) + ❌ Skipped: #{skipped_pct}% (#{skipped}) + ❓ Undecided: #{undecided_pct}% (#{undecided}) + Progress: --------- - [#{progress_bar(done_pct, wont_pct, todo_pct, undecided_pct)}] + [#{progress_bar(done_pct, skipped_pct, todo_pct, undecided_pct)}] """) end defp percentage(part, total) when total > 0 do - Float.round(part / total * 100, 1) + trunc(part / total * 100) end - defp progress_bar(done_pct, wont_pct, todo_pct, undecided_pct) do + defp progress_bar(done_pct, skipped_pct, todo_pct, undecided_pct) do done_chars = round(done_pct / 2) - wont_chars = round(wont_pct / 2) + skipped_chars = round(skipped_pct / 2) todo_chars = round(todo_pct / 2) undecided_chars = round(undecided_pct / 2) String.duplicate("=", done_chars) <> - String.duplicate("x", wont_chars) <> + String.duplicate("x", skipped_chars) <> String.duplicate("?", undecided_chars) <> String.duplicate(".", todo_chars) end From 2600a6609d70d36672fcdeea9cc89fe5abe873c3 Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 05:31:55 +0300 Subject: [PATCH 06/51] in midst of getting migration script up to date --- scripts/database_migration.exs | 97 ++++++++++++++++++++-------------- 1 file changed, 56 insertions(+), 41 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index ebbd1385b..90fd58765 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -57,11 +57,13 @@ defmodule DatabaseMigration do @relevant_tables Map.keys(@table_mappings) defp transform("Task", row, db) do - github_issue = - db |> Map.get("GithubIssue", []) |> Enum.find(&(&1["id"] == row["issue_id"])) + if row["forge"] != "github" do + raise "Unknown forge: #{row["forge"]}" + end + + github_issue = db |> Map.get("GithubIssue", []) |> Enum.find(&(&1["id"] == row["issue_id"])) - github_pull_request = - db |> Map.get("GithubPullRequest", []) |> Enum.find(&(&1["id"] == row["pull_request_id"])) + github_pull_request = db |> Map.get("GithubPullRequest", []) |> Enum.find(&(&1["id"] == row["pull_request_id"])) row = cond do @@ -96,10 +98,6 @@ defmodule DatabaseMigration do } true -> - if row["forge"] != "github" do - raise "Unknown forge: #{row["forge"]}" - end - %{ "id" => row["id"], "provider" => row["forge"], @@ -281,49 +279,66 @@ defmodule DatabaseMigration do defp transform("Bounty", row, db) do reward = db |> Map.get("Reward", []) |> Enum.find(&(&1["bounty_id"] == row["id"])) - amount = - if reward, do: Money.from_integer(String.to_integer(reward["amount"]), reward["currency"]) + amount = if reward, do: Money.from_integer(String.to_integer(reward["amount"]), reward["currency"]) - row - |> Map.put("ticket_id", row["task_id"]) - |> Map.put("owner_id", row["org_id"]) - |> Map.put("creator_id", row["poster_id"]) - |> Map.put("inserted_at", row["created_at"]) - |> Map.put("updated_at", row["updated_at"]) - |> Map.put("amount", amount) + %{ + "id" => row["id"], + "amount" => amount, + "ticket_id" => row["task_id"], + "owner_id" => row["org_id"], + "creator_id" => row["poster_id"], + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"] + } end defp transform("BountyTransfer", row, db) do - claim = - db |> Map.get("Claim", []) |> Enum.find(&(&1["id"] == row["claim_id"])) + claim = db |> Map.get("Claim", []) |> Enum.find(&(&1["id"] == row["claim_id"])) - github_user = - db |> Map.get("GithubUser", []) |> Enum.find(&(&1["id"] == claim["github_user_id"])) + github_user = db |> Map.get("GithubUser", []) |> Enum.find(&(&1["id"] == claim["github_user_id"])) - user = - db |> Map.get("User", []) |> Enum.find(&(&1["id"] == github_user["user_id"])) + user = db |> Map.get("User", []) |> Enum.find(&(&1["id"] == github_user["user_id"])) amount = Money.from_integer(String.to_integer(row["amount"]), row["currency"]) - row = - if claim && user do - row - |> Map.put("type", "transfer") - |> Map.put("provider", "stripe") - |> Map.put("provider_id", row["transfer_id"]) - |> Map.put("net_amount", amount) - |> Map.put("gross_amount", amount) - |> Map.put("total_fee", Money.zero(:USD)) - |> Map.put("bounty_id", claim["bounty_id"]) - ## TODO: this might be null but shouldn't - |> Map.put("user_id", user["id"]) - |> Map.put("inserted_at", row["created_at"]) - |> Map.put("updated_at", row["updated_at"]) - |> Map.put("status", if(row["succeeded_at"] == nil, do: :initialized, else: :succeeded)) - |> Map.put("succeeded_at", row["succeeded_at"]) - end + if !claim || !user do + raise "Claim or User not found: #{inspect(row)}" + end - row + # TODO: add corresponding credit & debit transactions + %{ + "id" => row["id"], + "provider" => "stripe", + "provider_id" => row["transfer_id"], + "provider_charge_id" => nil, + "provider_payment_intent_id" => nil, + "provider_transfer_id" => row["transfer_id"], + "provider_invoice_id" => nil, + "provider_balance_transaction_id" => nil, + "provider_meta" => nil, + "gross_amount" => amount, + "net_amount" => amount, + "total_fee" => Money.zero(:USD), + "provider_fee" => nil, + "line_items" => nil, + "type" => "transfer", + # TODO: only add debit & credit transactions if not succeeded + "status" => if(row["succeeded_at"] == nil, do: :initialized, else: :succeeded), + "succeeded_at" => row["succeeded_at"], + "reversed_at" => nil, + "group_id" => nil, + ## TODO: this might be null but shouldn't + "user_id" => user["id"], + "contract_id" => nil, + "original_contract_id" => nil, + "timesheet_id" => nil, + "bounty_id" => claim["bounty_id"], + "tip_id" => nil, + "linked_transaction_id" => nil, + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"], + "claim_id" => claim["id"] + } end defp transform(_, _row, _db), do: nil From 0b980415c4ad4f980ccb1883b2e798449a020602 Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 05:36:56 +0300 Subject: [PATCH 07/51] add transform for BountyCharge --- scripts/database_migration.exs | 47 ++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index 90fd58765..0c992c148 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -35,6 +35,7 @@ defmodule DatabaseMigration do "GithubPullRequest" => nil, "Bounty" => "bounties", "Reward" => nil, + "BountyCharge" => "transactions", "BountyTransfer" => "transactions", "Claim" => nil } @@ -48,6 +49,7 @@ defmodule DatabaseMigration do "GithubPullRequest" => nil, "Bounty" => Bounty, "Reward" => nil, + "BountyCharge" => Transaction, "BountyTransfer" => Transaction, "Claim" => nil } @@ -292,6 +294,51 @@ defmodule DatabaseMigration do } end + defp transform("BountyCharge", row, db) do + user = db |> Map.get("Org", []) |> Enum.find(&(&1["id"] == row["org_id"])) + + amount = Money.from_integer(String.to_integer(row["amount"]), row["currency"]) + + if !user || row["succeeded_at"] == nil do + raise "User not found: #{inspect(row)}" + end + + %{ + "id" => row["id"], + "provider" => "stripe", + "provider_id" => row["charge_id"], + "provider_charge_id" => row["charge_id"], + "provider_payment_intent_id" => nil, + "provider_transfer_id" => nil, + "provider_invoice_id" => nil, + "provider_balance_transaction_id" => nil, + "provider_meta" => nil, + # TODO: incorrect + "gross_amount" => amount, + "net_amount" => amount, + # TODO: incorrect + "total_fee" => Money.zero(:USD), + "provider_fee" => nil, + "line_items" => nil, + "type" => "charge", + "status" => if(row["succeeded_at"] == nil, do: :initialized, else: :succeeded), + "succeeded_at" => row["succeeded_at"], + "reversed_at" => nil, + "group_id" => nil, + ## TODO: this might be null but shouldn't + "user_id" => user["id"], + "contract_id" => nil, + "original_contract_id" => nil, + "timesheet_id" => nil, + "bounty_id" => nil, + "tip_id" => nil, + "linked_transaction_id" => nil, + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"], + "claim_id" => nil + } + end + defp transform("BountyTransfer", row, db) do claim = db |> Map.get("Claim", []) |> Enum.find(&(&1["id"] == row["claim_id"])) From d7f2cd65db00f1fa9384da38958b8154e0e2db34 Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 05:48:01 +0300 Subject: [PATCH 08/51] add transform for Claim --- scripts/database_migration.exs | 41 ++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index 0c992c148..2381554ca 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -21,6 +21,7 @@ defmodule DatabaseMigration do """ alias Algora.Accounts.User alias Algora.Bounties.Bounty + alias Algora.Bounties.Claim alias Algora.Payments.Transaction alias Algora.Workspace.Ticket @@ -35,9 +36,9 @@ defmodule DatabaseMigration do "GithubPullRequest" => nil, "Bounty" => "bounties", "Reward" => nil, + "Claim" => "claims", "BountyCharge" => "transactions", - "BountyTransfer" => "transactions", - "Claim" => nil + "BountyTransfer" => "transactions" } @schema_mappings %{ @@ -49,9 +50,9 @@ defmodule DatabaseMigration do "GithubPullRequest" => nil, "Bounty" => Bounty, "Reward" => nil, + "Claim" => Claim, "BountyCharge" => Transaction, - "BountyTransfer" => Transaction, - "Claim" => nil + "BountyTransfer" => Transaction } @backfilled_tables ["repositories", "transactions", "bounties", "tickets", "users"] @@ -294,6 +295,38 @@ defmodule DatabaseMigration do } end + defp transform("Claim", row, db) do + bounty = db |> Map.get("Bounty", []) |> Enum.find(&(&1["id"] == row["bounty_id"])) + + task = db |> Map.get("Task", []) |> Enum.find(&(&1["id"] == bounty["task_id"])) + + github_user = db |> Map.get("GithubUser", []) |> Enum.find(&(&1["id"] == row["github_user_id"])) + + user = db |> Map.get("User", []) |> Enum.find(&(&1["id"] == github_user["user_id"])) + + # TODO: this might be null + github_pull_request = + db |> Map.get("GithubPullRequest", []) |> Enum.find(&(&1["id"] == row["github_pull_request_id"])) + + if !task || !user do + raise "Task or User not found: #{inspect(row)}" + end + + %{ + "id" => row["id"], + "status" => nil, + "type" => nil, + "url" => row["github_url"], + "group_id" => nil, + "group_share" => nil, + "source_id" => github_pull_request["task_id"], + "target_id" => task["id"], + "user_id" => user["id"], + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"] + } + end + defp transform("BountyCharge", row, db) do user = db |> Map.get("Org", []) |> Enum.find(&(&1["id"] == row["org_id"])) From 5151a09e6275fef374e634417b6550eb215c0ccd Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 05:53:32 +0300 Subject: [PATCH 09/51] add transform for GithubInstallation --- scripts/database_migration.exs | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index 2381554ca..07a175b6a 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -38,7 +38,8 @@ defmodule DatabaseMigration do "Reward" => nil, "Claim" => "claims", "BountyCharge" => "transactions", - "BountyTransfer" => "transactions" + "BountyTransfer" => "transactions", + "GithubInstallation" => "installations" } @schema_mappings %{ @@ -52,10 +53,11 @@ defmodule DatabaseMigration do "Reward" => nil, "Claim" => Claim, "BountyCharge" => Transaction, - "BountyTransfer" => Transaction + "BountyTransfer" => Transaction, + "GithubInstallation" => Installation } - @backfilled_tables ["repositories", "transactions", "bounties", "tickets", "users"] + @backfilled_tables ["installations", "repositories", "transactions", "claims", "bounties", "tickets", "users"] @relevant_tables Map.keys(@table_mappings) @@ -421,6 +423,22 @@ defmodule DatabaseMigration do } end + defp transform("GithubInstallation", row, _db) do + %{ + "id" => row["id"], + "provider" => "github", + "provider_id" => row["github_id"], + "provider_meta" => nil, + "avatar_url" => nil, + "repository_selection" => nil, + "owner_id" => nil, + "connected_user_id" => nil, + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"], + "provider_user_id" => nil + } + end + defp transform(_, _row, _db), do: nil def process_dump(input_file, output_file) do From a672b9f3ab67c76b755b5b3074c14598060c17a8 Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 05:54:28 +0300 Subject: [PATCH 10/51] add progress yamls --- scripts/v1-progress.yaml | 589 +++++++++++++++++++++++++++++++++++++++ scripts/v2-progress.yaml | 315 +++++++++++++++++++++ 2 files changed, 904 insertions(+) create mode 100644 scripts/v1-progress.yaml create mode 100644 scripts/v2-progress.yaml diff --git a/scripts/v1-progress.yaml b/scripts/v1-progress.yaml new file mode 100644 index 000000000..33d7bb930 --- /dev/null +++ b/scripts/v1-progress.yaml @@ -0,0 +1,589 @@ +- "AccessToken": + - id: -1 + - created_at: -1 + - org_id: -1 + - name: -1 + - hash: -1 + - token_last_eight: -1 +- "Account": + - id: -1 + - created_at: -1 + - updated_at: -1 + - "userId": -1 + - type: -1 + - provider: -1 + - "providerAccountId": -1 + - refresh_token: -1 + - access_token: 0 + - expires_at: 0 + - token_type: -1 + - scope: -1 + - id_token: -1 + - session_state: -1 +- "Actor": + - id: -1 + - created_at: -1 + - updated_at: -1 + - user_id: -1 + - github_user_id: -1 + - org_id: -1 +- "Attempt": + - id: 0 + - created_at: 0 + - updated_at: 0 + - bounty_id: 0 + - github_user_id: 0 + - warnings_count: 0 + - status: 0 +- "Bid": + - id: -1 + - created_at: -1 + - updated_at: -1 + - user_id: -1 + - bounty_id: -1 + - amount: -1 + - currency: -1 + - status: -1 +- "Bonus": + - id: 0 + - created_at: 0 + - updated_at: 0 + - amount: 0 + - currency: 0 + - poster_id: 0 + - org_id: 0 + - claim_id: 0 + - github_req_comment_id: 0 + - github_res_comment_id: 0 + - type: 0 +- "BotMessage": + - id: -1 + - created_at: -1 + - updated_at: -1 + - active: -1 + - template: -1 + - type: -1 + - org_id: -1 +- "Bounty": + - id: 1 + - created_at: 1 + - updated_at: 1 + - poster_id: 1 + - org_id: 1 + - task_id: 1 + - status: 0 + - github_req_comment_id: 0 + - github_res_comment_id: 0 + - type: 0 + - kind: 0 + - visibility: 0 + - number: 0 + - deleted: 0 + - reward_type: 0 + - autopay_disabled: 0 + - manual_assignments: 0 + - timeouts_disabled: 0 + - experts_only: 0 +- "BountyCharge": + - id: 1 + - created_at: 1 + - updated_at: 1 + - succeeded_at: 1 + - charge_id: 1 + - user_id: 0 + - org_id: 1 + - amount: 1 + - currency: 1 + - receipt_url: 0 + - invoice_url: 0 +- "BountyProposal": + - id: -1 + - created_at: -1 + - updated_at: -1 + - org_id: -1 + - task_id: -1 + - amount: -1 + - currency: -1 + - note: -1 + - complexity: -1 + - importance: -1 + - steps: -1 +- "BountyTransfer": + - id: 1 + - created_at: 1 + - updated_at: 1 + - succeeded_at: 1 + - bounty_charge_id: 0 + - claim_id: 1 + - transfer_id: 1 + - amount: 1 + - currency: 1 +- "Claim": + - id: 1 + - created_at: 1 + - updated_at: 1 + - bounty_id: 1 + - status: 0 + - github_user_id: 1 + - github_url: 1 + - charge_id: -1 + - transfer_id: -1 + - charged_at: -1 + - transferred_at: -1 + - transfer_amount: -1 + - transfer_currency: -1 + - ready_to_pay: -1 + - github_pull_request_id: 1 + - bounty_charge_id: -1 + - github_res_comment_id: 0 + - info: 0 + - share: 0 +- "Company": + - id: -1 + - created_at: -1 + - updated_at: -1 + - org_id: -1 + - handle: -1 + - dates_talked: -1 + - contract_quoted: -1 + - cash_collected: -1 + - notes: -1 + - status: -1 + - success_checklist: -1 +- "Echo": + - id: -1 + - created_at: -1 + - message: -1 +- "Event": + - id: -1 + - created_at: -1 + - updated_at: -1 + - stream_id: -1 + - url: -1 + - actor_id: -1 + - name: -1 + - visibility: -1 + - payload: -1 + - pinned: -1 + - kind: -1 + - description: -1 + - emoji: -1 +- "EventDelivery": + - id: -1 + - created_at: -1 + - delivered_at: -1 + - event_id: -1 + - integration_id: -1 +- "Example": + - id: -1 + - created_at: -1 + - updated_at: -1 +- "GenericCharge": + - id: -1 + - created_at: -1 + - updated_at: -1 + - succeeded_at: -1 + - giver_id: -1 + - receiver_id: -1 + - stripe_id: -1 + - amount: -1 + - currency: -1 +- "GenericTransfer": + - id: -1 + - created_at: -1 + - updated_at: -1 + - succeeded_at: -1 + - charge_id: -1 + - receiver_id: -1 + - stripe_id: -1 + - amount: -1 + - currency: -1 +- "GithubInstallation": + - id: 1 + - created_at: 1 + - updated_at: 1 + - github_id: 1 + - org_id: 0 + - github_org_handle: 0 +- "GithubIssue": + - id: 1 + - url: -1 + - repository_url: -1 + - labels_url: -1 + - comments_url: -1 + - events_url: -1 + - html_url: 1 + - number: 1 + - state: -1 + - state_reason: -1 + - title: 1 + - body: 1 + - user_id: -1 + - locked: -1 + - active_lock_reason: -1 + - comments: -1 + - closed_at: -1 + - created_at: 1 + - updated_at: 1 + - draft: -1 + - body_html: -1 + - body_text: -1 + - timeline_url: -1 + - author_association: -1 + - task_id: -1 + - retrieved_at: -1 + - api_version: -1 + - is_assigned: -1 + - labels: -1 +- "GithubPullRequest": + - id: 1 + - url: -1 + - repository_url: -1 + - labels_url: -1 + - comments_url: -1 + - events_url: -1 + - html_url: 1 + - number: 1 + - state: -1 + - state_reason: -1 + - title: 1 + - body: 1 + - user_id: -1 + - locked: -1 + - active_lock_reason: -1 + - comments: -1 + - closed_at: -1 + - created_at: 1 + - updated_at: 1 + - draft: -1 + - body_html: -1 + - body_text: -1 + - timeline_url: -1 + - author_association: -1 + - merged_at: -1 + - diff_url: -1 + - patch_url: -1 + - task_id: -1 + - retrieved_at: -1 + - api_version: -1 + - is_assigned: -1 + - labels: -1 +- "GithubUser": + - id: 1 + - login: 1 + - avatar_url: 1 + - gravatar_id: -1 + - url: -1 + - html_url: -1 + - followers_url: -1 + - following_url: -1 + - gists_url: -1 + - starred_url: -1 + - subscriptions_url: -1 + - organizations_url: -1 + - repos_url: -1 + - events_url: -1 + - received_events_url: -1 + - type: 0 + - site_admin: -1 + - name: 1 + - email: 0 + - user_id: -1 + - retrieved_at: 1 + - api_version: -1 + - bio: 1 + - company: -1 + - location: 1 + - twitter_username: 1 + - blocked_from_attempting: -1 + - blocked_from_earning: -1 +- "Integration": + - id: -1 + - created_at: -1 + - updated_at: -1 + - org_id: -1 + - user_id: -1 + - name: -1 + - url: -1 + - kind: -1 + - scope: -1 + - state: -1 + - token: -1 + - status: -1 +- "IssueSummary": + - id: -1 + - created_at: -1 + - updated_at: -1 + - body: -1 + - complexity: -1 + - importance: -1 + - steps: -1 +- "Org": + - id: 1 + - created_at: 1 + - updated_at: 1 + - handle: 1 + - creator_id: -1 + - name: 1 + - avatar_url: 1 + - domain: 1 + - webhook_url: -1 + - github_id: 1 + - github_data: 1 + - github_data_fetched: -1 + - fee_pct: 1 + - featured: 1 + - accepts_sponsorships: -1 + - tech: 1 + - description: 1 + - stargazers_count: 1 + - priority: 1 + - supervise_attempts: 0 + - manual_assignments: 1 + - is_personal: 0 + - max_open_attempts: 1 + - enabled_expert_recs: 0 + - enabled_private_bounties: 0 + - fee_pct_prev: -1 + - days_until_timeout: 0 + - days_until_timeout_reminder: 0 + - discord_url: 1 + - twitter_url: 1 + - website_url: 1 + - youtube_url: 1 + - enabled_community_mode: 0 + - webhook_url_discord: -1 + - webhook_url_slack: -1 + - accepts_community_bounties: 0 + - seeded: 1 + - slack_url: 1 + - github_handle: 1 + - active: 1 +- "OrgBalanceTransaction": + - id: 0 + - created_at: 0 + - org_id: 0 + - amount: 0 + - currency: 0 + - type: 0 + - effect: 0 + - tier: 0 + - description: 0 + - user_id: 0 + - plan: 0 +- "OrgEventSubscription": + - id: -1 + - created_at: -1 + - updated_at: -1 + - user_id: -1 + - org_id: -1 + - member_id: -1 + - kind: -1 +- "OrgMember": + - id: 0 + - created_at: 0 + - updated_at: 0 + - org_id: 0 + - user_id: 0 + - weight: 0 + - role: 0 +- "PointReward": + - id: -1 + - created_at: -1 + - updated_at: -1 + - amount: -1 + - bounty_id: -1 +- "Reward": + - id: -1 + - created_at: -1 + - updated_at: -1 + - amount: 1 + - bounty_id: 1 + - currency: 1 +- "RewardTier": + - id: -1 + - created_at: -1 + - updated_at: -1 + - amount: -1 + - currency: -1 + - bounty_id: -1 + - lower_bound: -1 + - upper_bound: -1 + - metric: -1 +- "Session": + - id: -1 + - created_at: -1 + - updated_at: -1 + - "sessionToken": -1 + - "userId": -1 + - expires: -1 +- "Sponsorship": + - id: -1 + - created_at: -1 + - updated_at: -1 + - succeeded_at: -1 + - creator_id: -1 + - receiver_id: -1 + - charge_id: -1 + - user_id: -1 + - amount: -1 + - currency: -1 +- "SponsorshipTransfer": + - id: -1 + - created_at: -1 + - updated_at: -1 + - succeeded_at: -1 + - sponsorship_id: -1 + - receiver_id: -1 + - transfer_id: -1 + - amount: -1 + - currency: -1 +- "StripeAccount": + - id: 0 + - created_at: 0 + - updated_at: 0 + - user_id: 0 + - details_submitted: 0 + - charges_enabled: 0 + - service_agreement: 0 + - country: 0 + - type: 0 + - region: 0 + - needs_refresh: 0 +- "StripeBalanceTransaction": + - stripe_id: -1 + - created: -1 + - currency: -1 + - amount: -1 + - fee: -1 + - net: -1 + - status: -1 + - type: -1 + - id: -1 + - period: -1 + - source: -1 +- "StripeCustomer": + - id: 0 + - created_at: 0 + - updated_at: 0 + - stripe_id: 0 + - org_id: 0 + - name: 0 + - region: 0 +- "StripePaymentMethod": + - id: 0 + - created_at: 0 + - updated_at: 0 + - stripe_id: 0 + - org_id: 0 + - is_default: 0 +- "StripeSubscription": + - id: -1 + - created_at: -1 + - updated_at: -1 + - charged_at: -1 + - charge_id: -1 + - customer_id: -1 + - amount: -1 + - plan: -1 + - active: -1 + - currency: -1 +- "SurveyResponse": + - id: -1 + - created_at: -1 + - updated_at: -1 + - email: -1 + - payload: -1 +- "SystemEvent": + - id: -1 + - created_at: -1 + - tag: -1 + - origin: -1 + - origin_id: -1 + - status: -1 +- "SystemEventDelivery": + - id: -1 + - created_at: -1 + - event_id: -1 + - status: -1 + - message: -1 + - payload: -1 +- "Task": + - id: 1 + - forge: 1 + - repo_owner: 1 + - repo_name: 1 + - number: 1 + - issue_id: 1 + - pull_request_id: 1 + - source: 0 + - image: 0 + - status: 0 + - body: 1 + - slug: 0 + - title: 1 + - og_image: 0 + - tech: 0 +- "User": + - id: 1 + - created_at: 1 + - updated_at: 1 + - handle: 1 + - name: 1 + - email: 1 + - "emailVerified": 1 + - image: 1 + - is_admin: 1 + - loc: 1 + - last_activity_at: 0 + - country: 1 + - twitch_handle: 1 + - youtube_handle: 1 + - tech: 1 + - contribs_last_mo: -1 + - stars_earned: 1 + - discovery_sort_by: -1 + - discovery_tech: -1 + - alipay_account_holder_name: -1 + - alipay_id: -1 + - wise_account_holder_name: -1 + - wise_email: -1 + - onboarding: -1 + - can_create_org: -1 +- "UserActivity": + - id: -1 + - created_at: -1 + - updated_at: -1 + - user_id: -1 + - ip: -1 +- "UserEventSubscription": + - id: -1 + - created_at: -1 + - updated_at: -1 + - last_notif_at: -1 + - user_id: -1 + - frequency: -1 + - kind: -1 +- "UserExpertise": + - id: -1 + - created_at: -1 + - updated_at: -1 + - language: -1 + - user_id: -1 +- "UserLanguage": + - id: -1 + - created_at: -1 + - updated_at: -1 + - language: -1 + - user_id: -1 + - pull_requests: -1 +- "VerificationToken": + - identifier: -1 + - token: -1 + - expires: -1 +- "_Event_participants": + - "A": -1 + - "B": -1 +- "_Event_subscribers": + - "A": -1 + - "B": -1 diff --git a/scripts/v2-progress.yaml b/scripts/v2-progress.yaml new file mode 100644 index 000000000..7e67ae2f9 --- /dev/null +++ b/scripts/v2-progress.yaml @@ -0,0 +1,315 @@ +- accounts: + - id: 0 + - provider: 0 + - provider_id: 0 + - provider_meta: 0 + - name: 0 + - details_submitted: 0 + - charges_enabled: 0 + - service_agreement: 0 + - country: 0 + - type: 0 + - stale: 0 + - user_id: 0 + - inserted_at: 0 + - updated_at: 0 + - payouts_enabled: 0 + - payout_interval: 0 + - payout_speed: 0 + - default_currency: 0 +- attempts: + - id: 0 + - status: 0 + - warnings_count: 0 + - ticket_id: 0 + - user_id: 0 + - inserted_at: 0 + - updated_at: 0 +- bounties: + - id: 1 + - amount: 1 + - status: 0 + - ticket_id: 1 + - owner_id: 1 + - creator_id: 1 + - inserted_at: 1 + - updated_at: 1 +- chat_participants: + - id: 0 + - last_read_at: 0 + - thread_id: 0 + - user_id: 0 + - inserted_at: 0 + - updated_at: 0 +- claims: + - id: 1 + - status: 0 + - type: 0 + - url: 0 + - group_id: 0 + - group_share: 0 + - source_id: 1 + - target_id: 1 + - user_id: 1 + - inserted_at: 1 + - updated_at: 1 +- command_responses: + - id: 0 + - provider: 0 + - provider_meta: 0 + - provider_command_id: 0 + - provider_response_id: 0 + - command_source: 0 + - command_type: 0 + - ticket_id: 0 + - inserted_at: 0 + - updated_at: 0 +- comment_cursors: + - id: 0 + - provider: 0 + - repo_owner: 0 + - repo_name: 0 + - "timestamp": 0 + - last_polled_at: 0 + - last_comment_id: 0 + - inserted_at: 0 + - updated_at: 0 +- contracts: + - id: 0 + - status: 0 + - sequence_number: 0 + - hourly_rate: 0 + - hourly_rate_min: 0 + - hourly_rate_max: 0 + - hours_per_week: 0 + - start_date: 0 + - end_date: 0 + - original_contract_id: 0 + - client_id: 0 + - contractor_id: 0 + - inserted_at: 0 + - updated_at: 0 +- customers: + - id: 0 + - provider: 0 + - provider_id: 0 + - provider_meta: 0 + - name: 0 + - user_id: 0 + - inserted_at: 0 + - updated_at: 0 +- event_cursors: + - id: 0 + - provider: 0 + - repo_owner: 0 + - repo_name: 0 + - last_event_id: 0 + - last_polled_at: 0 + - inserted_at: 0 + - updated_at: 0 +- identities: + - id: 0 + - user_id: 0 + - provider: 0 + - provider_token: 0 + - provider_email: 0 + - provider_login: 0 + - provider_id: 0 + - provider_meta: 0 + - inserted_at: 0 + - updated_at: 0 +- installations: + - id: 1 + - provider: 1 + - provider_id: 1 + - provider_meta: -1 + - avatar_url: -1 + - repository_selection: -1 + - owner_id: 0 + - connected_user_id: 0 + - inserted_at: 1 + - updated_at: 1 + - provider_user_id: 0 +- members: + - id: 0 + - role: 0 + - org_id: 0 + - user_id: 0 + - inserted_at: 0 + - updated_at: 0 +- messages: + - id: 0 + - content: 0 + - thread_id: 0 + - sender_id: 0 + - inserted_at: 0 + - updated_at: 0 +- oban_jobs: + - id: 0 + - state: 0 + - queue: 0 + - worker: 0 + - args: 0 + - errors: 0 + - attempt: 0 + - max_attempts: 0 + - inserted_at: 0 + - scheduled_at: 0 + - attempted_at: 0 + - completed_at: 0 + - attempted_by: 0 + - discarded_at: 0 + - priority: 0 + - tags: 0 + - meta: 0 + - cancelled_at: 0 + - CONSTRAINT: 0 + - CONSTRAINT: 0 + - CONSTRAINT: 0 + - CONSTRAINT: 0 +- payment_methods: + - id: 0 + - provider: 0 + - provider_id: 0 + - provider_meta: 0 + - provider_customer_id: 0 + - is_default: 0 + - customer_id: 0 + - inserted_at: 0 + - updated_at: 0 +- repositories: + - id: 0 + - provider: 0 + - provider_id: 0 + - provider_meta: 0 + - name: 0 + - url: 0 + - user_id: 0 + - inserted_at: 0 + - updated_at: 0 + - description: 0 + - og_image_url: 0 + - og_image_updated_at: 0 +- reviews: + - id: 0 + - rating: 0 + - content: 0 + - visibility: 0 + - contract_id: 0 + - bounty_id: 0 + - organization_id: 0 + - reviewer_id: 0 + - reviewee_id: 0 + - inserted_at: 0 + - updated_at: 0 +- threads: + - id: 0 + - title: 0 + - inserted_at: 0 + - updated_at: 0 +- tickets: + - id: 1 + - provider: 1 + - provider_id: 1 + - provider_meta: 1 + - type: 1 + - title: 1 + - description: 1 + - number: 1 + - url: 1 + - repository_id: 0 + - inserted_at: 1 + - updated_at: 1 +- timesheets: + - id: 0 + - hours_worked: 0 + - description: 0 + - contract_id: 0 + - inserted_at: 0 + - updated_at: 0 +- tips: + - id: 0 + - amount: 0 + - status: 0 + - ticket_id: 0 + - owner_id: 0 + - creator_id: 0 + - recipient_id: 0 + - inserted_at: 0 + - updated_at: 0 +- transactions: + - id: 1 + - provider: 1 + - provider_id: 1 + - provider_charge_id: 1 + - provider_payment_intent_id: -1 + - provider_transfer_id: 1 + - provider_invoice_id: -1 + - provider_balance_transaction_id: -1 + - provider_meta: -1 + - gross_amount: 1 + - net_amount: 1 + - total_fee: 1 + - provider_fee: -1 + - line_items: -1 + - type: 1 + - status: 1 + - succeeded_at: 1 + - reversed_at: -1 + - group_id: 0 + - user_id: 0 + - contract_id: -1 + - original_contract_id: -1 + - timesheet_id: -1 + - bounty_id: 1 + - tip_id: 0 + - linked_transaction_id: 0 + - inserted_at: 1 + - updated_at: 1 + - claim_id: 1 +- users: + - id: 1 + - provider: 1 + - provider_id: 1 + - provider_login: 1 + - provider_meta: 1 + - email: 1 + - display_name: 1 + - handle: 1 + - avatar_url: 1 + - external_homepage_url: -2 + - type: 1 + - bio: 1 + - location: 1 + - country: 1 + - timezone: 1 + - stargazers_count: 1 + - domain: 1 + - tech_stack: 1 + - featured: 1 + - priority: 1 + - fee_pct: 1 + - seeded: 1 + - activated: 1 + - max_open_attempts: 1 + - manual_assignment: 1 + - bounty_mode: -1 + - hourly_rate_min: -1 + - hourly_rate_max: -1 + - hours_per_week: -1 + - website_url: 1 + - twitter_url: 1 + - github_url: -2 + - youtube_url: 1 + - twitch_url: 1 + - discord_url: 1 + - slack_url: 1 + - linkedin_url: -1 + - og_title: -1 + - og_image_url: -1 + - last_context: 1 + - need_avatar: -2 + - inserted_at: 1 + - updated_at: 1 + - name: -1 + - is_admin: 1 From cae3734cb6883f514f7dffcfd052905c202f5c29 Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 05:58:20 +0300 Subject: [PATCH 11/51] add transform for StripeAccount --- scripts/database_migration.exs | 40 +++++++++++++++++++++++++++++++--- scripts/v1-progress.yaml | 22 +++++++++---------- scripts/v2-progress.yaml | 36 +++++++++++++++--------------- 3 files changed, 66 insertions(+), 32 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index 07a175b6a..f0fb08236 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -39,7 +39,8 @@ defmodule DatabaseMigration do "Claim" => "claims", "BountyCharge" => "transactions", "BountyTransfer" => "transactions", - "GithubInstallation" => "installations" + "GithubInstallation" => "installations", + "StripeAccount" => "accounts" } @schema_mappings %{ @@ -54,10 +55,20 @@ defmodule DatabaseMigration do "Claim" => Claim, "BountyCharge" => Transaction, "BountyTransfer" => Transaction, - "GithubInstallation" => Installation + "GithubInstallation" => Installation, + "StripeAccount" => Account } - @backfilled_tables ["installations", "repositories", "transactions", "claims", "bounties", "tickets", "users"] + @backfilled_tables [ + "accounts", + "installations", + "repositories", + "transactions", + "claims", + "bounties", + "tickets", + "users" + ] @relevant_tables Map.keys(@table_mappings) @@ -439,6 +450,29 @@ defmodule DatabaseMigration do } end + defp transform("StripeAccount", row, _db) do + %{ + "id" => row["id"], + "provider" => "stripe", + "provider_id" => row["stripe_id"], + "provider_meta" => nil, + "name" => nil, + "details_submitted" => row["details_submitted"], + "charges_enabled" => row["charges_enabled"], + "service_agreement" => row["service_agreement"], + "country" => row["country"], + "type" => row["type"], + "stale" => row["needs_refresh"], + "user_id" => row["user_id"], + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"], + "payouts_enabled" => nil, + "payout_interval" => nil, + "payout_speed" => nil, + "default_currency" => nil + } + end + defp transform(_, _row, _db), do: nil def process_dump(input_file, output_file) do diff --git a/scripts/v1-progress.yaml b/scripts/v1-progress.yaml index 33d7bb930..0b09633f7 100644 --- a/scripts/v1-progress.yaml +++ b/scripts/v1-progress.yaml @@ -439,17 +439,17 @@ - amount: -1 - currency: -1 - "StripeAccount": - - id: 0 - - created_at: 0 - - updated_at: 0 - - user_id: 0 - - details_submitted: 0 - - charges_enabled: 0 - - service_agreement: 0 - - country: 0 - - type: 0 - - region: 0 - - needs_refresh: 0 + - id: 1 + - created_at: 1 + - updated_at: 1 + - user_id: 1 + - details_submitted: 1 + - charges_enabled: 1 + - service_agreement: 1 + - country: 1 + - type: 1 + - region: -1 + - needs_refresh: 1 - "StripeBalanceTransaction": - stripe_id: -1 - created: -1 diff --git a/scripts/v2-progress.yaml b/scripts/v2-progress.yaml index 7e67ae2f9..02540d9ea 100644 --- a/scripts/v2-progress.yaml +++ b/scripts/v2-progress.yaml @@ -1,22 +1,22 @@ - accounts: - - id: 0 - - provider: 0 - - provider_id: 0 - - provider_meta: 0 - - name: 0 - - details_submitted: 0 - - charges_enabled: 0 - - service_agreement: 0 - - country: 0 - - type: 0 - - stale: 0 - - user_id: 0 - - inserted_at: 0 - - updated_at: 0 - - payouts_enabled: 0 - - payout_interval: 0 - - payout_speed: 0 - - default_currency: 0 + - id: 1 + - provider: 1 + - provider_id: 1 + - provider_meta: -1 + - name: -1 + - details_submitted: 1 + - charges_enabled: 1 + - service_agreement: 1 + - country: 1 + - type: 1 + - stale: 1 + - user_id: 1 + - inserted_at: 1 + - updated_at: 1 + - payouts_enabled: -1 + - payout_interval: -1 + - payout_speed: -1 + - default_currency: -1 - attempts: - id: 0 - status: 0 From 85a62dbdbc42660f2fcd69aeb6d6c4088f68dc9d Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 06:00:00 +0300 Subject: [PATCH 12/51] add transform for StripeCustomer --- scripts/database_migration.exs | 20 ++++++++++++++++++-- scripts/v1-progress.yaml | 14 +++++++------- scripts/v2-progress.yaml | 16 ++++++++-------- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index f0fb08236..6ee2740ac 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -40,7 +40,8 @@ defmodule DatabaseMigration do "BountyCharge" => "transactions", "BountyTransfer" => "transactions", "GithubInstallation" => "installations", - "StripeAccount" => "accounts" + "StripeAccount" => "accounts", + "StripeCustomer" => "customers" } @schema_mappings %{ @@ -56,11 +57,13 @@ defmodule DatabaseMigration do "BountyCharge" => Transaction, "BountyTransfer" => Transaction, "GithubInstallation" => Installation, - "StripeAccount" => Account + "StripeAccount" => Account, + "StripeCustomer" => Customer } @backfilled_tables [ "accounts", + "customers", "installations", "repositories", "transactions", @@ -473,6 +476,19 @@ defmodule DatabaseMigration do } end + defp transform("StripeCustomer", row, _db) do + %{ + "id" => row["id"], + "provider" => "stripe", + "provider_id" => row["stripe_id"], + "provider_meta" => nil, + "name" => row["name"], + "user_id" => row["org_id"], + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"] + } + end + defp transform(_, _row, _db), do: nil def process_dump(input_file, output_file) do diff --git a/scripts/v1-progress.yaml b/scripts/v1-progress.yaml index 0b09633f7..bd27f6bef 100644 --- a/scripts/v1-progress.yaml +++ b/scripts/v1-progress.yaml @@ -463,13 +463,13 @@ - period: -1 - source: -1 - "StripeCustomer": - - id: 0 - - created_at: 0 - - updated_at: 0 - - stripe_id: 0 - - org_id: 0 - - name: 0 - - region: 0 + - id: 1 + - created_at: 1 + - updated_at: 1 + - stripe_id: 1 + - org_id: 1 + - name: 1 + - region: -1 - "StripePaymentMethod": - id: 0 - created_at: 0 diff --git a/scripts/v2-progress.yaml b/scripts/v2-progress.yaml index 02540d9ea..aff477721 100644 --- a/scripts/v2-progress.yaml +++ b/scripts/v2-progress.yaml @@ -90,14 +90,14 @@ - inserted_at: 0 - updated_at: 0 - customers: - - id: 0 - - provider: 0 - - provider_id: 0 - - provider_meta: 0 - - name: 0 - - user_id: 0 - - inserted_at: 0 - - updated_at: 0 + - id: 1 + - provider: 1 + - provider_id: 1 + - provider_meta: -1 + - name: 1 + - user_id: 1 + - inserted_at: 1 + - updated_at: 1 - event_cursors: - id: 0 - provider: 0 From dcb6a95b99d8eee44b08a820bfba20b8dc89463c Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 06:00:46 +0300 Subject: [PATCH 13/51] add missing aliases --- scripts/database_migration.exs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index 6ee2740ac..62bf14ddc 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -22,7 +22,10 @@ defmodule DatabaseMigration do alias Algora.Accounts.User alias Algora.Bounties.Bounty alias Algora.Bounties.Claim + alias Algora.Payments.Account + alias Algora.Payments.Customer alias Algora.Payments.Transaction + alias Algora.Workspace.Installation alias Algora.Workspace.Ticket require Logger From 119f40de5b83fa7097ceb014dae7e2abed19bc4d Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 06:03:17 +0300 Subject: [PATCH 14/51] add transform for StripePaymentMethod --- scripts/database_migration.exs | 28 ++++++++++++++++++++++++++-- scripts/v1-progress.yaml | 12 ++++++------ scripts/v2-progress.yaml | 18 +++++++++--------- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index 62bf14ddc..eba20f311 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -24,6 +24,7 @@ defmodule DatabaseMigration do alias Algora.Bounties.Claim alias Algora.Payments.Account alias Algora.Payments.Customer + alias Algora.Payments.PaymentMethod alias Algora.Payments.Transaction alias Algora.Workspace.Installation alias Algora.Workspace.Ticket @@ -44,7 +45,8 @@ defmodule DatabaseMigration do "BountyTransfer" => "transactions", "GithubInstallation" => "installations", "StripeAccount" => "accounts", - "StripeCustomer" => "customers" + "StripeCustomer" => "customers", + "StripePaymentMethod" => "payment_methods" } @schema_mappings %{ @@ -61,11 +63,13 @@ defmodule DatabaseMigration do "BountyTransfer" => Transaction, "GithubInstallation" => Installation, "StripeAccount" => Account, - "StripeCustomer" => Customer + "StripeCustomer" => Customer, + "StripePaymentMethod" => PaymentMethod } @backfilled_tables [ "accounts", + "payment_methods", "customers", "installations", "repositories", @@ -492,6 +496,26 @@ defmodule DatabaseMigration do } end + defp transform("StripePaymentMethod", row, db) do + customer = db |> Map.get("StripeCustomer", []) |> Enum.find(&(&1["id"] == row["customer_id"])) + + if !customer do + raise "StripeCustomer not found: #{inspect(row)}" + end + + %{ + "id" => row["id"], + "provider" => "stripe", + "provider_id" => row["stripe_id"], + "provider_meta" => nil, + "provider_customer_id" => customer["stripe_id"], + "is_default" => row["is_default"], + "customer_id" => row["customer_id"], + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"] + } + end + defp transform(_, _row, _db), do: nil def process_dump(input_file, output_file) do diff --git a/scripts/v1-progress.yaml b/scripts/v1-progress.yaml index bd27f6bef..073221175 100644 --- a/scripts/v1-progress.yaml +++ b/scripts/v1-progress.yaml @@ -471,12 +471,12 @@ - name: 1 - region: -1 - "StripePaymentMethod": - - id: 0 - - created_at: 0 - - updated_at: 0 - - stripe_id: 0 - - org_id: 0 - - is_default: 0 + - id: 1 + - created_at: 1 + - updated_at: 1 + - stripe_id: 1 + - org_id: -1 + - is_default: 1 - "StripeSubscription": - id: -1 - created_at: -1 diff --git a/scripts/v2-progress.yaml b/scripts/v2-progress.yaml index aff477721..34cb9e57c 100644 --- a/scripts/v2-progress.yaml +++ b/scripts/v2-progress.yaml @@ -168,15 +168,15 @@ - CONSTRAINT: 0 - CONSTRAINT: 0 - payment_methods: - - id: 0 - - provider: 0 - - provider_id: 0 - - provider_meta: 0 - - provider_customer_id: 0 - - is_default: 0 - - customer_id: 0 - - inserted_at: 0 - - updated_at: 0 + - id: 1 + - provider: 1 + - provider_id: 1 + - provider_meta: -1 + - provider_customer_id: 1 + - is_default: 1 + - customer_id: 1 + - inserted_at: 1 + - updated_at: 1 - repositories: - id: 0 - provider: 0 From 7a3407738bb59bcda3d58cd6b8d9c60afc24f3d0 Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 06:06:16 +0300 Subject: [PATCH 15/51] skip OrgBalanceTransaction --- scripts/v1-progress.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/v1-progress.yaml b/scripts/v1-progress.yaml index 073221175..b4e289f9f 100644 --- a/scripts/v1-progress.yaml +++ b/scripts/v1-progress.yaml @@ -360,17 +360,17 @@ - github_handle: 1 - active: 1 - "OrgBalanceTransaction": - - id: 0 - - created_at: 0 - - org_id: 0 - - amount: 0 - - currency: 0 - - type: 0 - - effect: 0 - - tier: 0 - - description: 0 - - user_id: 0 - - plan: 0 + - id: -1 + - created_at: -1 + - org_id: -1 + - amount: -1 + - currency: -1 + - type: -1 + - effect: -1 + - tier: -1 + - description: -1 + - user_id: -1 + - plan: -1 - "OrgEventSubscription": - id: -1 - created_at: -1 From 6e12eb2bc48420e5a96d7f1eae3ef4823bd826e9 Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 06:09:19 +0300 Subject: [PATCH 16/51] add transform for OrgMember --- scripts/database_migration.exs | 19 +++++++++++++++++++ scripts/v1-progress.yaml | 14 +++++++------- scripts/v2-progress.yaml | 12 ++++++------ 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index eba20f311..7cb50e104 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -22,6 +22,7 @@ defmodule DatabaseMigration do alias Algora.Accounts.User alias Algora.Bounties.Bounty alias Algora.Bounties.Claim + alias Algora.Organizations.Member alias Algora.Payments.Account alias Algora.Payments.Customer alias Algora.Payments.PaymentMethod @@ -35,6 +36,7 @@ defmodule DatabaseMigration do "User" => "users", "Org" => "users", "GithubUser" => "users", + "OrgMember" => "members", "Task" => "tickets", "GithubIssue" => nil, "GithubPullRequest" => nil, @@ -53,6 +55,7 @@ defmodule DatabaseMigration do "User" => User, "Org" => User, "GithubUser" => User, + "OrgMember" => Member, "Task" => Ticket, "GithubIssue" => nil, "GithubPullRequest" => nil, @@ -77,6 +80,7 @@ defmodule DatabaseMigration do "claims", "bounties", "tickets", + "members", "users" ] @@ -302,6 +306,21 @@ defmodule DatabaseMigration do defp transform("GithubUser", _row, _db), do: nil + defp transform("OrgMember", row, _db) do + if row["role"] not in ["admin", "mod", "expert"] do + raise "OrgMember has unknown role: #{inspect(row)}" + end + + %{ + "id" => row["id"], + "org_id" => row["org_id"], + "role" => row["role"], + "user_id" => row["user_id"], + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"] + } + end + defp transform("Bounty", row, db) do reward = db |> Map.get("Reward", []) |> Enum.find(&(&1["bounty_id"] == row["id"])) diff --git a/scripts/v1-progress.yaml b/scripts/v1-progress.yaml index b4e289f9f..e8e9942c4 100644 --- a/scripts/v1-progress.yaml +++ b/scripts/v1-progress.yaml @@ -380,13 +380,13 @@ - member_id: -1 - kind: -1 - "OrgMember": - - id: 0 - - created_at: 0 - - updated_at: 0 - - org_id: 0 - - user_id: 0 - - weight: 0 - - role: 0 + - id: 1 + - created_at: 1 + - updated_at: 1 + - org_id: 1 + - user_id: 1 + - weight: -1 + - role: 1 - "PointReward": - id: -1 - created_at: -1 diff --git a/scripts/v2-progress.yaml b/scripts/v2-progress.yaml index 34cb9e57c..001fbdf9a 100644 --- a/scripts/v2-progress.yaml +++ b/scripts/v2-progress.yaml @@ -131,12 +131,12 @@ - updated_at: 1 - provider_user_id: 0 - members: - - id: 0 - - role: 0 - - org_id: 0 - - user_id: 0 - - inserted_at: 0 - - updated_at: 0 + - id: 1 + - role: 1 + - org_id: 1 + - user_id: 1 + - inserted_at: 1 + - updated_at: 1 - messages: - id: 0 - content: 0 From 8eb41fb3e458606e09a2cb0b3856ab2945834f79 Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 06:18:15 +0300 Subject: [PATCH 17/51] add transform for Attempt --- scripts/database_migration.exs | 34 ++++++++++++++++++++++++++++++---- scripts/v1-progress.yaml | 14 +++++++------- scripts/v2-progress.yaml | 14 +++++++------- 3 files changed, 44 insertions(+), 18 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index 7cb50e104..1d788713d 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -20,6 +20,7 @@ defmodule DatabaseMigration do - Run the script using: elixir scripts/database_migration.exs """ alias Algora.Accounts.User + alias Algora.Bounties.Attempt alias Algora.Bounties.Bounty alias Algora.Bounties.Claim alias Algora.Organizations.Member @@ -42,6 +43,7 @@ defmodule DatabaseMigration do "GithubPullRequest" => nil, "Bounty" => "bounties", "Reward" => nil, + "Attempt" => "attempts", "Claim" => "claims", "BountyCharge" => "transactions", "BountyTransfer" => "transactions", @@ -61,6 +63,7 @@ defmodule DatabaseMigration do "GithubPullRequest" => nil, "Bounty" => Bounty, "Reward" => nil, + "Attempt" => Attempt, "Claim" => Claim, "BountyCharge" => Transaction, "BountyTransfer" => Transaction, @@ -78,6 +81,7 @@ defmodule DatabaseMigration do "repositories", "transactions", "claims", + "attempts", "bounties", "tickets", "members", @@ -307,10 +311,6 @@ defmodule DatabaseMigration do defp transform("GithubUser", _row, _db), do: nil defp transform("OrgMember", row, _db) do - if row["role"] not in ["admin", "mod", "expert"] do - raise "OrgMember has unknown role: #{inspect(row)}" - end - %{ "id" => row["id"], "org_id" => row["org_id"], @@ -337,6 +337,32 @@ defmodule DatabaseMigration do } end + defp transform("Attempt", row, db) do + bounty = db |> Map.get("Bounty", []) |> Enum.find(&(&1["id"] == row["bounty_id"])) + + github_user = db |> Map.get("GithubUser", []) |> Enum.find(&(&1["id"] == row["github_user_id"])) + + user = db |> Map.get("User", []) |> Enum.find(&(&1["id"] == github_user["user_id"])) + + if !bounty do + raise "Bounty not found: #{inspect(row)}" + end + + if !user do + raise "User not found: #{inspect(row)}" + end + + %{ + "id" => row["id"], + "status" => row["status"], + "warnings_count" => row["warnings_count"], + "ticket_id" => bounty["task_id"], + "user_id" => user["id"], + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"] + } + end + defp transform("Claim", row, db) do bounty = db |> Map.get("Bounty", []) |> Enum.find(&(&1["id"] == row["bounty_id"])) diff --git a/scripts/v1-progress.yaml b/scripts/v1-progress.yaml index e8e9942c4..d44a82237 100644 --- a/scripts/v1-progress.yaml +++ b/scripts/v1-progress.yaml @@ -28,13 +28,13 @@ - github_user_id: -1 - org_id: -1 - "Attempt": - - id: 0 - - created_at: 0 - - updated_at: 0 - - bounty_id: 0 - - github_user_id: 0 - - warnings_count: 0 - - status: 0 + - id: 1 + - created_at: 1 + - updated_at: 1 + - bounty_id: 1 + - github_user_id: 1 + - warnings_count: 1 + - status: 1 - "Bid": - id: -1 - created_at: -1 diff --git a/scripts/v2-progress.yaml b/scripts/v2-progress.yaml index 001fbdf9a..b8ff076ce 100644 --- a/scripts/v2-progress.yaml +++ b/scripts/v2-progress.yaml @@ -18,13 +18,13 @@ - payout_speed: -1 - default_currency: -1 - attempts: - - id: 0 - - status: 0 - - warnings_count: 0 - - ticket_id: 0 - - user_id: 0 - - inserted_at: 0 - - updated_at: 0 + - id: 1 + - status: 1 + - warnings_count: 1 + - ticket_id: 1 + - user_id: 1 + - inserted_at: 1 + - updated_at: 1 - bounties: - id: 1 - amount: 1 From 6138f58776c53a9b7e93fd1e98eee939a20a0cde Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 06:20:03 +0300 Subject: [PATCH 18/51] update v2 progress --- scripts/v2-progress.yaml | 175 +++++++++++++++++---------------------- 1 file changed, 76 insertions(+), 99 deletions(-) diff --git a/scripts/v2-progress.yaml b/scripts/v2-progress.yaml index b8ff076ce..f7f5b99b5 100644 --- a/scripts/v2-progress.yaml +++ b/scripts/v2-progress.yaml @@ -35,12 +35,12 @@ - inserted_at: 1 - updated_at: 1 - chat_participants: - - id: 0 - - last_read_at: 0 - - thread_id: 0 - - user_id: 0 - - inserted_at: 0 - - updated_at: 0 + - id: -1 + - last_read_at: -1 + - thread_id: -1 + - user_id: -1 + - inserted_at: -1 + - updated_at: -1 - claims: - id: 1 - status: 0 @@ -65,30 +65,30 @@ - inserted_at: 0 - updated_at: 0 - comment_cursors: - - id: 0 - - provider: 0 - - repo_owner: 0 - - repo_name: 0 - - "timestamp": 0 - - last_polled_at: 0 - - last_comment_id: 0 - - inserted_at: 0 - - updated_at: 0 + - id: -1 + - provider: -1 + - repo_owner: -1 + - repo_name: -1 + - "timestamp": -1 + - last_polled_at: -1 + - last_comment_id: -1 + - inserted_at: -1 + - updated_at: -1 - contracts: - - id: 0 - - status: 0 - - sequence_number: 0 - - hourly_rate: 0 - - hourly_rate_min: 0 - - hourly_rate_max: 0 - - hours_per_week: 0 - - start_date: 0 - - end_date: 0 - - original_contract_id: 0 - - client_id: 0 - - contractor_id: 0 - - inserted_at: 0 - - updated_at: 0 + - id: -1 + - status: -1 + - sequence_number: -1 + - hourly_rate: -1 + - hourly_rate_min: -1 + - hourly_rate_max: -1 + - hours_per_week: -1 + - start_date: -1 + - end_date: -1 + - original_contract_id: -1 + - client_id: -1 + - contractor_id: -1 + - inserted_at: -1 + - updated_at: -1 - customers: - id: 1 - provider: 1 @@ -99,14 +99,14 @@ - inserted_at: 1 - updated_at: 1 - event_cursors: - - id: 0 - - provider: 0 - - repo_owner: 0 - - repo_name: 0 - - last_event_id: 0 - - last_polled_at: 0 - - inserted_at: 0 - - updated_at: 0 + - id: -1 + - provider: -1 + - repo_owner: -1 + - repo_name: -1 + - last_event_id: -1 + - last_polled_at: -1 + - inserted_at: -1 + - updated_at: -1 - identities: - id: 0 - user_id: 0 @@ -138,35 +138,12 @@ - inserted_at: 1 - updated_at: 1 - messages: - - id: 0 - - content: 0 - - thread_id: 0 - - sender_id: 0 - - inserted_at: 0 - - updated_at: 0 -- oban_jobs: - - id: 0 - - state: 0 - - queue: 0 - - worker: 0 - - args: 0 - - errors: 0 - - attempt: 0 - - max_attempts: 0 - - inserted_at: 0 - - scheduled_at: 0 - - attempted_at: 0 - - completed_at: 0 - - attempted_by: 0 - - discarded_at: 0 - - priority: 0 - - tags: 0 - - meta: 0 - - cancelled_at: 0 - - CONSTRAINT: 0 - - CONSTRAINT: 0 - - CONSTRAINT: 0 - - CONSTRAINT: 0 + - id: -1 + - content: -1 + - thread_id: -1 + - sender_id: -1 + - inserted_at: -1 + - updated_at: -1 - payment_methods: - id: 1 - provider: 1 @@ -178,35 +155,35 @@ - inserted_at: 1 - updated_at: 1 - repositories: - - id: 0 - - provider: 0 - - provider_id: 0 - - provider_meta: 0 - - name: 0 - - url: 0 - - user_id: 0 - - inserted_at: 0 - - updated_at: 0 - - description: 0 - - og_image_url: 0 - - og_image_updated_at: 0 + - id: -1 + - provider: -1 + - provider_id: -1 + - provider_meta: -1 + - name: -1 + - url: -1 + - user_id: -1 + - inserted_at: -1 + - updated_at: -1 + - description: -1 + - og_image_url: -1 + - og_image_updated_at: -1 - reviews: - - id: 0 - - rating: 0 - - content: 0 - - visibility: 0 - - contract_id: 0 - - bounty_id: 0 - - organization_id: 0 - - reviewer_id: 0 - - reviewee_id: 0 - - inserted_at: 0 - - updated_at: 0 + - id: -1 + - rating: -1 + - content: -1 + - visibility: -1 + - contract_id: -1 + - bounty_id: -1 + - organization_id: -1 + - reviewer_id: -1 + - reviewee_id: -1 + - inserted_at: -1 + - updated_at: -1 - threads: - - id: 0 - - title: 0 - - inserted_at: 0 - - updated_at: 0 + - id: -1 + - title: -1 + - inserted_at: -1 + - updated_at: -1 - tickets: - id: 1 - provider: 1 @@ -221,12 +198,12 @@ - inserted_at: 1 - updated_at: 1 - timesheets: - - id: 0 - - hours_worked: 0 - - description: 0 - - contract_id: 0 - - inserted_at: 0 - - updated_at: 0 + - id: -1 + - hours_worked: -1 + - description: -1 + - contract_id: -1 + - inserted_at: -1 + - updated_at: -1 - tips: - id: 0 - amount: 0 From 929850107e1d49342777d12673cb25eb6a4d7647 Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 06:35:20 +0300 Subject: [PATCH 19/51] update progress formatting --- scripts/analyze_progress.exs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/analyze_progress.exs b/scripts/analyze_progress.exs index f28291d2b..83ec5a73d 100644 --- a/scripts/analyze_progress.exs +++ b/scripts/analyze_progress.exs @@ -48,10 +48,10 @@ defmodule ProgressAnalyzer do Status Breakdown: ---------------- - ✅ Completed: #{done_pct}% (#{done}) - ⏳ Remaining: #{todo_pct}% (#{todo}) - ❌ Skipped: #{skipped_pct}% (#{skipped}) - ❓ Undecided: #{undecided_pct}% (#{undecided}) + ✅ Completed: #{String.pad_leading("#{done_pct}", 2)}% #{String.pad_leading("(#{done})", 5)} + ⏳ Remaining: #{String.pad_leading("#{todo_pct}", 2)}% #{String.pad_leading("(#{todo})", 5)} + ❌ Skipped: #{String.pad_leading("#{skipped_pct}", 2)}% #{String.pad_leading("(#{skipped})", 5)} + ❓ Undecided: #{String.pad_leading("#{undecided_pct}", 2)}% #{String.pad_leading("(#{undecided})", 5)} Progress: --------- From 5d841a3428f6c24a8a76b79bfc8ae2c6ea776c47 Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 06:47:17 +0300 Subject: [PATCH 20/51] add transform for Account --- scripts/database_migration.exs | 19 +++++++++++++++++++ scripts/v1-progress.yaml | 16 ++++++++-------- scripts/v2-progress.yaml | 20 ++++++++++---------- 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index 1d788713d..89f5f1768 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -19,6 +19,7 @@ defmodule DatabaseMigration do - Set the output_file to your desired output file path. - Run the script using: elixir scripts/database_migration.exs """ + alias Algora.Accounts.Identity alias Algora.Accounts.User alias Algora.Bounties.Attempt alias Algora.Bounties.Bounty @@ -37,6 +38,7 @@ defmodule DatabaseMigration do "User" => "users", "Org" => "users", "GithubUser" => "users", + "Account" => "identities", "OrgMember" => "members", "Task" => "tickets", "GithubIssue" => nil, @@ -57,6 +59,7 @@ defmodule DatabaseMigration do "User" => User, "Org" => User, "GithubUser" => User, + "Account" => Identity, "OrgMember" => Member, "Task" => Ticket, "GithubIssue" => nil, @@ -85,6 +88,7 @@ defmodule DatabaseMigration do "bounties", "tickets", "members", + "identities", "users" ] @@ -310,6 +314,21 @@ defmodule DatabaseMigration do defp transform("GithubUser", _row, _db), do: nil + defp transform("Account", row, _db) do + %{ + "id" => row["id"], + "user_id" => row["\"userId\""], + "provider" => row["provider"], + "provider_token" => row["access_token"], + "provider_email" => nil, + "provider_login" => nil, + "provider_id" => row["\"providerAccountId\""], + "provider_meta" => nil, + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"] + } + end + defp transform("OrgMember", row, _db) do %{ "id" => row["id"], diff --git a/scripts/v1-progress.yaml b/scripts/v1-progress.yaml index d44a82237..7190701f3 100644 --- a/scripts/v1-progress.yaml +++ b/scripts/v1-progress.yaml @@ -6,16 +6,16 @@ - hash: -1 - token_last_eight: -1 - "Account": - - id: -1 - - created_at: -1 - - updated_at: -1 - - "userId": -1 + - id: 1 + - created_at: 1 + - updated_at: 1 + - "userId": 1 - type: -1 - - provider: -1 - - "providerAccountId": -1 + - provider: 1 + - "providerAccountId": 1 - refresh_token: -1 - - access_token: 0 - - expires_at: 0 + - access_token: 1 + - expires_at: -1 - token_type: -1 - scope: -1 - id_token: -1 diff --git a/scripts/v2-progress.yaml b/scripts/v2-progress.yaml index f7f5b99b5..92e09114f 100644 --- a/scripts/v2-progress.yaml +++ b/scripts/v2-progress.yaml @@ -108,16 +108,16 @@ - inserted_at: -1 - updated_at: -1 - identities: - - id: 0 - - user_id: 0 - - provider: 0 - - provider_token: 0 - - provider_email: 0 - - provider_login: 0 - - provider_id: 0 - - provider_meta: 0 - - inserted_at: 0 - - updated_at: 0 + - id: 1 + - user_id: 1 + - provider: 1 + - provider_token: 1 + - provider_email: 1 + - provider_login: 1 + - provider_id: 1 + - provider_meta: 1 + - inserted_at: 1 + - updated_at: 1 - installations: - id: 1 - provider: 1 From be26168c7522ba41cce3ec0508ac7dcbec70e25f Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 06:53:35 +0300 Subject: [PATCH 21/51] derive table mappings from schema mappings --- scripts/database_migration.exs | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index 89f5f1768..f07b4d804 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -34,27 +34,6 @@ defmodule DatabaseMigration do require Logger - @table_mappings %{ - "User" => "users", - "Org" => "users", - "GithubUser" => "users", - "Account" => "identities", - "OrgMember" => "members", - "Task" => "tickets", - "GithubIssue" => nil, - "GithubPullRequest" => nil, - "Bounty" => "bounties", - "Reward" => nil, - "Attempt" => "attempts", - "Claim" => "claims", - "BountyCharge" => "transactions", - "BountyTransfer" => "transactions", - "GithubInstallation" => "installations", - "StripeAccount" => "accounts", - "StripeCustomer" => "customers", - "StripePaymentMethod" => "payment_methods" - } - @schema_mappings %{ "User" => User, "Org" => User, @@ -92,7 +71,7 @@ defmodule DatabaseMigration do "users" ] - @relevant_tables Map.keys(@table_mappings) + @relevant_tables Map.keys(@schema_mappings) defp transform("Task", row, db) do if row["forge"] != "github" do @@ -687,7 +666,7 @@ defmodule DatabaseMigration do end end - defp transform_table_name(table_name), do: @table_mappings[table_name] + defp transform_table_name(table_name), do: if(schema = @schema_mappings[table_name], do: schema.__schema__(:source)) defp post_transform(table_name, row) do schema = @schema_mappings[table_name] From 1609c7bcec7976f9ff65e6e91324bf55dcebd77b Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 07:23:35 +0300 Subject: [PATCH 22/51] allow duplicate keys to handle multi migrations --- scripts/database_migration.exs | 122 ++++++++++++++++----------------- 1 file changed, 59 insertions(+), 63 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index f07b4d804..2fd93e638 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -34,26 +34,26 @@ defmodule DatabaseMigration do require Logger - @schema_mappings %{ - "User" => User, - "Org" => User, - "GithubUser" => User, - "Account" => Identity, - "OrgMember" => Member, - "Task" => Ticket, - "GithubIssue" => nil, - "GithubPullRequest" => nil, - "Bounty" => Bounty, - "Reward" => nil, - "Attempt" => Attempt, - "Claim" => Claim, - "BountyCharge" => Transaction, - "BountyTransfer" => Transaction, - "GithubInstallation" => Installation, - "StripeAccount" => Account, - "StripeCustomer" => Customer, - "StripePaymentMethod" => PaymentMethod - } + @schema_mappings [ + {"User", User}, + {"Org", User}, + {"GithubUser", User}, + {"Account", Identity}, + {"OrgMember", Member}, + {"Task", Ticket}, + {"GithubIssue", nil}, + {"GithubPullRequest", nil}, + {"Bounty", Bounty}, + {"Reward", nil}, + {"Attempt", Attempt}, + {"Claim", Claim}, + {"BountyCharge", Transaction}, + {"BountyTransfer", Transaction}, + {"GithubInstallation", Installation}, + {"StripeAccount", Account}, + {"StripeCustomer", Customer}, + {"StripePaymentMethod", PaymentMethod} + ] @backfilled_tables [ "accounts", @@ -71,9 +71,9 @@ defmodule DatabaseMigration do "users" ] - @relevant_tables Map.keys(@schema_mappings) + defp relevant_tables, do: @schema_mappings |> Enum.map(fn {k, _v} -> k end) |> Enum.dedup() - defp transform("Task", row, db) do + defp transform({"Task", Ticket}, row, db) do if row["forge"] != "github" do raise "Unknown forge: #{row["forge"]}" end @@ -133,7 +133,7 @@ defmodule DatabaseMigration do row end - defp transform("User", row, db) do + defp transform({"User", User}, row, db) do # TODO: reenable # if !row["\"emailVerified\""] || String.length(row["\"emailVerified\""]) < 10 do # raise "Email not verified: #{inspect(row)}" @@ -189,7 +189,7 @@ defmodule DatabaseMigration do } end - defp transform("Org", row, _db) do + defp transform({"Org", User}, row, _db) do %{ "id" => row["id"], "provider" => row["github_handle"] && "github", @@ -238,7 +238,7 @@ defmodule DatabaseMigration do } end - defp transform("GithubUser", %{user_id: nil} = row, _db) do + defp transform({"GithubUser", User}, %{user_id: nil} = row, _db) do if row["type"] != "User" do raise "GithubUser is not a User: #{inspect(row)}" end @@ -291,9 +291,9 @@ defmodule DatabaseMigration do } end - defp transform("GithubUser", _row, _db), do: nil + defp transform({"GithubUser", User}, _row, _db), do: nil - defp transform("Account", row, _db) do + defp transform({"Account", Identity}, row, _db) do %{ "id" => row["id"], "user_id" => row["\"userId\""], @@ -308,7 +308,7 @@ defmodule DatabaseMigration do } end - defp transform("OrgMember", row, _db) do + defp transform({"OrgMember", Member}, row, _db) do %{ "id" => row["id"], "org_id" => row["org_id"], @@ -319,7 +319,7 @@ defmodule DatabaseMigration do } end - defp transform("Bounty", row, db) do + defp transform({"Bounty", Bounty}, row, db) do reward = db |> Map.get("Reward", []) |> Enum.find(&(&1["bounty_id"] == row["id"])) amount = if reward, do: Money.from_integer(String.to_integer(reward["amount"]), reward["currency"]) @@ -335,7 +335,7 @@ defmodule DatabaseMigration do } end - defp transform("Attempt", row, db) do + defp transform({"Attempt", Attempt}, row, db) do bounty = db |> Map.get("Bounty", []) |> Enum.find(&(&1["id"] == row["bounty_id"])) github_user = db |> Map.get("GithubUser", []) |> Enum.find(&(&1["id"] == row["github_user_id"])) @@ -361,7 +361,7 @@ defmodule DatabaseMigration do } end - defp transform("Claim", row, db) do + defp transform({"Claim", Claim}, row, db) do bounty = db |> Map.get("Bounty", []) |> Enum.find(&(&1["id"] == row["bounty_id"])) task = db |> Map.get("Task", []) |> Enum.find(&(&1["id"] == bounty["task_id"])) @@ -393,7 +393,7 @@ defmodule DatabaseMigration do } end - defp transform("BountyCharge", row, db) do + defp transform({"BountyCharge", Transaction}, row, db) do user = db |> Map.get("Org", []) |> Enum.find(&(&1["id"] == row["org_id"])) amount = Money.from_integer(String.to_integer(row["amount"]), row["currency"]) @@ -438,7 +438,7 @@ defmodule DatabaseMigration do } end - defp transform("BountyTransfer", row, db) do + defp transform({"BountyTransfer", Transaction}, row, db) do claim = db |> Map.get("Claim", []) |> Enum.find(&(&1["id"] == row["claim_id"])) github_user = db |> Map.get("GithubUser", []) |> Enum.find(&(&1["id"] == claim["github_user_id"])) @@ -487,7 +487,7 @@ defmodule DatabaseMigration do } end - defp transform("GithubInstallation", row, _db) do + defp transform({"GithubInstallation", Installation}, row, _db) do %{ "id" => row["id"], "provider" => "github", @@ -503,7 +503,7 @@ defmodule DatabaseMigration do } end - defp transform("StripeAccount", row, _db) do + defp transform({"StripeAccount", Account}, row, _db) do %{ "id" => row["id"], "provider" => "stripe", @@ -526,7 +526,7 @@ defmodule DatabaseMigration do } end - defp transform("StripeCustomer", row, _db) do + defp transform({"StripeCustomer", Customer}, row, _db) do %{ "id" => row["id"], "provider" => "stripe", @@ -539,7 +539,7 @@ defmodule DatabaseMigration do } end - defp transform("StripePaymentMethod", row, db) do + defp transform({"StripePaymentMethod", PaymentMethod}, row, db) do customer = db |> Map.get("StripeCustomer", []) |> Enum.find(&(&1["id"] == row["customer_id"])) if !customer do @@ -586,12 +586,13 @@ defmodule DatabaseMigration do &collect_after_fun/1 ) |> Enum.reduce(%{}, fn - {table, data}, acc when table in @relevant_tables -> - parsed_data = parse_copy_data(data) - Map.put(acc, table, parsed_data) - - _, acc -> - acc + {table, data}, acc -> + if table in relevant_tables() do + parsed_data = parse_copy_data(data) + Map.put(acc, table, parsed_data) + else + acc + end end) end @@ -628,24 +629,25 @@ defmodule DatabaseMigration do defp collect_after_fun({table, acc}), do: {:cont, {table, Enum.reverse(acc)}, nil} defp process_chunk(chunk, db) do - case_result = - case extract_copy_section(chunk) do - %{table: table} = section when table in @relevant_tables -> - transform_section(section, db) - - _ -> - nil - end + case extract_copy_section(chunk) do + %{table: table} = section -> + @schema_mappings + |> Enum.filter(fn {k, _v} -> k == table end) + |> Enum.map(fn {_k, v} -> transform_section(section, v, db) end) + |> Enum.reject(&is_nil/1) + |> Enum.map(&load_copy_section/1) - load_copy_section(case_result) + _ -> + [] + end end - defp transform_section(%{table: table, columns: _columns, data: data}, db) do + defp transform_section(%{table: table, columns: _columns, data: data}, schema, db) do transformed_data = data |> Enum.map(fn row -> # try do - transform(table, row, db) + transform({table, schema}, row, db) # rescue # e -> # IO.puts("Error transforming row in table #{table}: #{inspect(row)}") @@ -654,9 +656,9 @@ defmodule DatabaseMigration do # end end) |> Enum.reject(&is_nil/1) - |> Enum.map(&post_transform(table, &1)) + |> Enum.map(&post_transform(schema, &1)) - transformed_table_name = transform_table_name(table) + transformed_table_name = schema.__schema__(:source) if Enum.empty?(transformed_data) do nil @@ -666,11 +668,7 @@ defmodule DatabaseMigration do end end - defp transform_table_name(table_name), do: if(schema = @schema_mappings[table_name], do: schema.__schema__(:source)) - - defp post_transform(table_name, row) do - schema = @schema_mappings[table_name] - + defp post_transform(schema, row) do default_fields = schema.__struct__() |> Map.from_struct() @@ -732,8 +730,6 @@ defmodule DatabaseMigration do new_handle end - defp load_copy_section(nil), do: [] - defp load_copy_section(%{table: table_name, columns: columns, data: data}) do copy_statement = "COPY #{table_name} (#{Enum.join(columns, ", ")}) FROM stdin;\n" From 3efc3ea97a85d53d6de76759846c7e8b653d6850 Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 07:35:38 +0300 Subject: [PATCH 23/51] add transform for {"BountyTransfer", Tip} --- scripts/database_migration.exs | 41 ++++++++++++++++++++++++++++++++-- scripts/v2-progress.yaml | 18 +++++++-------- 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index 2fd93e638..cff5cd439 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -24,6 +24,7 @@ defmodule DatabaseMigration do alias Algora.Bounties.Attempt alias Algora.Bounties.Bounty alias Algora.Bounties.Claim + alias Algora.Bounties.Tip alias Algora.Organizations.Member alias Algora.Payments.Account alias Algora.Payments.Customer @@ -49,6 +50,7 @@ defmodule DatabaseMigration do {"Claim", Claim}, {"BountyCharge", Transaction}, {"BountyTransfer", Transaction}, + {"BountyTransfer", Tip}, {"GithubInstallation", Installation}, {"StripeAccount", Account}, {"StripeCustomer", Customer}, @@ -65,6 +67,7 @@ defmodule DatabaseMigration do "claims", "attempts", "bounties", + "tips", "tickets", "members", "identities", @@ -487,6 +490,40 @@ defmodule DatabaseMigration do } end + defp transform({"BountyTransfer", Tip}, row, db) do + claim = db |> Map.get("Claim", []) |> Enum.find(&(&1["id"] == row["claim_id"])) + + github_user = db |> Map.get("GithubUser", []) |> Enum.find(&(&1["id"] == row["github_user_id"])) + + user = db |> Map.get("User", []) |> Enum.find(&(&1["id"] == github_user["user_id"])) + + bounty = db |> Map.get("Bounty", []) |> Enum.find(&(&1["id"] == claim["bounty_id"])) + + amount = Money.from_integer(String.to_integer(row["amount"]), row["currency"]) + + if !bounty do + raise "Bounty not found: #{inspect(row)}" + end + + if !user do + raise "User not found: #{inspect(row)}" + end + + if bounty["type"] == "tip" do + %{ + "id" => row["id"], + "amount" => amount, + "status" => nil, + "ticket_id" => bounty["task_id"], + "owner_id" => bounty["org_id"], + "creator_id" => bounty["poster_id"], + "recipient_id" => user["id"], + "inserted_at" => bounty["created_at"], + "updated_at" => bounty["updated_at"] + } + end + end + defp transform({"GithubInstallation", Installation}, row, _db) do %{ "id" => row["id"], @@ -928,8 +965,8 @@ defmodule DatabaseMigration do output_file = ".local/prod_db_new.sql" if File.exists?(input_file) or File.exists?(output_file) do - # IO.puts("Processing dump...") - # :ok = process_dump(input_file, output_file) + IO.puts("Processing dump...") + :ok = process_dump(input_file, output_file) IO.puts("Clearing tables...") :ok = clear_tables!() diff --git a/scripts/v2-progress.yaml b/scripts/v2-progress.yaml index 92e09114f..47310ba8b 100644 --- a/scripts/v2-progress.yaml +++ b/scripts/v2-progress.yaml @@ -205,15 +205,15 @@ - inserted_at: -1 - updated_at: -1 - tips: - - id: 0 - - amount: 0 - - status: 0 - - ticket_id: 0 - - owner_id: 0 - - creator_id: 0 - - recipient_id: 0 - - inserted_at: 0 - - updated_at: 0 + - id: 1 + - amount: 1 + - status: -2 + - ticket_id: 1 + - owner_id: 1 + - creator_id: 1 + - recipient_id: 1 + - inserted_at: 1 + - updated_at: 1 - transactions: - id: 1 - provider: 1 From 97db61573c1ae522d753531d54d2dadfe2e6e419 Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 07:42:13 +0300 Subject: [PATCH 24/51] track tip transfers accurately --- scripts/database_migration.exs | 88 ++++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 37 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index cff5cd439..c0acae406 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -49,8 +49,8 @@ defmodule DatabaseMigration do {"Attempt", Attempt}, {"Claim", Claim}, {"BountyCharge", Transaction}, - {"BountyTransfer", Transaction}, {"BountyTransfer", Tip}, + {"BountyTransfer", Transaction}, {"GithubInstallation", Installation}, {"StripeAccount", Account}, {"StripeCustomer", Customer}, @@ -441,21 +441,61 @@ defmodule DatabaseMigration do } end + defp transform({"BountyTransfer", Tip}, row, db) do + claim = db |> Map.get("Claim", []) |> Enum.find(&(&1["id"] == row["claim_id"])) + + github_user = db |> Map.get("GithubUser", []) |> Enum.find(&(&1["id"] == row["github_user_id"])) + + user = db |> Map.get("User", []) |> Enum.find(&(&1["id"] == github_user["user_id"])) + + bounty = db |> Map.get("Bounty", []) |> Enum.find(&(&1["id"] == claim["bounty_id"])) + + amount = Money.from_integer(String.to_integer(row["amount"]), row["currency"]) + + if !bounty do + raise "Bounty not found: #{inspect(row)}" + end + + if !user do + raise "User not found: #{inspect(row)}" + end + + if bounty["type"] == "tip" do + %{ + "id" => bounty["id"] <> user["id"], + "amount" => amount, + "status" => nil, + "ticket_id" => bounty["task_id"], + "owner_id" => bounty["org_id"], + "creator_id" => bounty["poster_id"], + "recipient_id" => user["id"], + "inserted_at" => bounty["created_at"], + "updated_at" => bounty["updated_at"] + } + end + end + defp transform({"BountyTransfer", Transaction}, row, db) do claim = db |> Map.get("Claim", []) |> Enum.find(&(&1["id"] == row["claim_id"])) + bounty = db |> Map.get("Bounty", []) |> Enum.find(&(&1["id"] == claim["bounty_id"])) + github_user = db |> Map.get("GithubUser", []) |> Enum.find(&(&1["id"] == claim["github_user_id"])) user = db |> Map.get("User", []) |> Enum.find(&(&1["id"] == github_user["user_id"])) amount = Money.from_integer(String.to_integer(row["amount"]), row["currency"]) - if !claim || !user do - raise "Claim or User not found: #{inspect(row)}" + if !bounty do + raise "Bounty not found: #{inspect(row)}" + end + + if !user do + raise "User not found: #{inspect(row)}" end # TODO: add corresponding credit & debit transactions - %{ + row = %{ "id" => row["id"], "provider" => "stripe", "provider_id" => row["transfer_id"], @@ -481,46 +521,20 @@ defmodule DatabaseMigration do "contract_id" => nil, "original_contract_id" => nil, "timesheet_id" => nil, - "bounty_id" => claim["bounty_id"], + "bounty_id" => nil, "tip_id" => nil, "linked_transaction_id" => nil, "inserted_at" => row["created_at"], "updated_at" => row["updated_at"], - "claim_id" => claim["id"] + "claim_id" => nil } - end - - defp transform({"BountyTransfer", Tip}, row, db) do - claim = db |> Map.get("Claim", []) |> Enum.find(&(&1["id"] == row["claim_id"])) - - github_user = db |> Map.get("GithubUser", []) |> Enum.find(&(&1["id"] == row["github_user_id"])) - - user = db |> Map.get("User", []) |> Enum.find(&(&1["id"] == github_user["user_id"])) - - bounty = db |> Map.get("Bounty", []) |> Enum.find(&(&1["id"] == claim["bounty_id"])) - - amount = Money.from_integer(String.to_integer(row["amount"]), row["currency"]) - - if !bounty do - raise "Bounty not found: #{inspect(row)}" - end - - if !user do - raise "User not found: #{inspect(row)}" - end if bounty["type"] == "tip" do - %{ - "id" => row["id"], - "amount" => amount, - "status" => nil, - "ticket_id" => bounty["task_id"], - "owner_id" => bounty["org_id"], - "creator_id" => bounty["poster_id"], - "recipient_id" => user["id"], - "inserted_at" => bounty["created_at"], - "updated_at" => bounty["updated_at"] - } + Map.put(row, "tip_id", bounty["id"] <> user["id"]) + else + row + |> Map.put("bounty_id", claim["bounty_id"]) + |> Map.put("claim_id", claim["id"]) end end From 234589a33ee5bdebf8370ddce3b24301c9bc9ae9 Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 08:13:41 +0300 Subject: [PATCH 25/51] remove obsolete function --- scripts/database_migration.exs | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index c0acae406..af0422748 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -737,7 +737,6 @@ defmodule DatabaseMigration do |> Enum.reject(fn {_, v} -> v == "\\N" end) |> Enum.reject(fn {_, v} -> v == nil end) |> Map.new(fn {k, v} -> {k, v} end) - |> conditionally_rename_created_at() |> Map.take(Enum.map(Map.keys(default_fields), &Atom.to_string/1)) |> Map.new(fn {k, v} -> {String.to_existing_atom(k), v} end) @@ -747,18 +746,6 @@ defmodule DatabaseMigration do Map.merge(default_fields, fields) end - defp conditionally_rename_created_at(fields) do - case {Map.get(fields, "inserted_at"), Map.get(fields, "created_at")} do - {nil, created_at} when not is_nil(created_at) -> - fields - |> Map.put("inserted_at", created_at) - |> Map.delete("created_at") - - _ -> - fields - end - end - defp ensure_unique_handle(fields) do case fields[:handle] do nil -> From 4e775c74db1a5ab53be2e243a901ccf7ef61a3bf Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 08:24:42 +0300 Subject: [PATCH 26/51] add transform for {"Bounty", CommandResponse} --- scripts/database_migration.exs | 21 ++++++++++++++++++++- scripts/v1-progress.yaml | 4 ++-- scripts/v2-progress.yaml | 20 ++++++++++---------- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index af0422748..b4c842d71 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -30,6 +30,7 @@ defmodule DatabaseMigration do alias Algora.Payments.Customer alias Algora.Payments.PaymentMethod alias Algora.Payments.Transaction + alias Algora.Workspace.CommandResponse alias Algora.Workspace.Installation alias Algora.Workspace.Ticket @@ -45,6 +46,7 @@ defmodule DatabaseMigration do {"GithubIssue", nil}, {"GithubPullRequest", nil}, {"Bounty", Bounty}, + {"Bounty", CommandResponse}, {"Reward", nil}, {"Attempt", Attempt}, {"Claim", Claim}, @@ -338,6 +340,23 @@ defmodule DatabaseMigration do } end + defp transform({"Bounty", CommandResponse}, row, _db) do + if row["github_res_comment_id"] != nil do + %{ + "id" => row["id"], + "provider" => "github", + "provider_meta" => nil, + "provider_command_id" => row["github_req_comment_id"], + "provider_response_id" => row["github_res_comment_id"], + "command_source" => "comment", + "command_type" => "bounty", + "ticket_id" => row["task_id"], + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"] + } + end + end + defp transform({"Attempt", Attempt}, row, db) do bounty = db |> Map.get("Bounty", []) |> Enum.find(&(&1["id"] == row["bounty_id"])) @@ -740,7 +759,7 @@ defmodule DatabaseMigration do |> Map.take(Enum.map(Map.keys(default_fields), &Atom.to_string/1)) |> Map.new(fn {k, v} -> {String.to_existing_atom(k), v} end) - # Ensure handle is unique + # TODO: do we need this? fields = ensure_unique_handle(fields) Map.merge(default_fields, fields) diff --git a/scripts/v1-progress.yaml b/scripts/v1-progress.yaml index 7190701f3..dd00626fe 100644 --- a/scripts/v1-progress.yaml +++ b/scripts/v1-progress.yaml @@ -72,8 +72,8 @@ - org_id: 1 - task_id: 1 - status: 0 - - github_req_comment_id: 0 - - github_res_comment_id: 0 + - github_req_comment_id: 1 + - github_res_comment_id: 1 - type: 0 - kind: 0 - visibility: 0 diff --git a/scripts/v2-progress.yaml b/scripts/v2-progress.yaml index 47310ba8b..304c45da5 100644 --- a/scripts/v2-progress.yaml +++ b/scripts/v2-progress.yaml @@ -54,16 +54,16 @@ - inserted_at: 1 - updated_at: 1 - command_responses: - - id: 0 - - provider: 0 - - provider_meta: 0 - - provider_command_id: 0 - - provider_response_id: 0 - - command_source: 0 - - command_type: 0 - - ticket_id: 0 - - inserted_at: 0 - - updated_at: 0 + - id: 1 + - provider: 1 + - provider_meta: -1 + - provider_command_id: 1 + - provider_response_id: 1 + - command_source: 1 + - command_type: 1 + - ticket_id: 1 + - inserted_at: 1 + - updated_at: 1 - comment_cursors: - id: -1 - provider: -1 From 3f1dc749f41dbbaf9207ff6f783840043b850741 Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 08:53:20 +0300 Subject: [PATCH 27/51] add missing transaction data --- scripts/database_migration.exs | 188 ++++++++++++++++++++++++--------- scripts/v1-progress.yaml | 8 +- scripts/v2-progress.yaml | 18 ++-- 3 files changed, 151 insertions(+), 63 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index b4c842d71..c7b7a2130 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -445,7 +445,7 @@ defmodule DatabaseMigration do "status" => if(row["succeeded_at"] == nil, do: :initialized, else: :succeeded), "succeeded_at" => row["succeeded_at"], "reversed_at" => nil, - "group_id" => nil, + "group_id" => row["id"], ## TODO: this might be null but shouldn't "user_id" => user["id"], "contract_id" => nil, @@ -503,7 +503,9 @@ defmodule DatabaseMigration do user = db |> Map.get("User", []) |> Enum.find(&(&1["id"] == github_user["user_id"])) - amount = Money.from_integer(String.to_integer(row["amount"]), row["currency"]) + org = db |> Map.get("Org", []) |> Enum.find(&(&1["id"] == bounty["org_id"])) + + bounty_charge = db |> Map.get("BountyCharge", []) |> Enum.find(&(&1["id"] == row["bounty_charge_id"])) if !bounty do raise "Bounty not found: #{inspect(row)}" @@ -513,48 +515,37 @@ defmodule DatabaseMigration do raise "User not found: #{inspect(row)}" end - # TODO: add corresponding credit & debit transactions - row = %{ - "id" => row["id"], - "provider" => "stripe", - "provider_id" => row["transfer_id"], - "provider_charge_id" => nil, - "provider_payment_intent_id" => nil, - "provider_transfer_id" => row["transfer_id"], - "provider_invoice_id" => nil, - "provider_balance_transaction_id" => nil, - "provider_meta" => nil, - "gross_amount" => amount, - "net_amount" => amount, - "total_fee" => Money.zero(:USD), - "provider_fee" => nil, - "line_items" => nil, - "type" => "transfer", - # TODO: only add debit & credit transactions if not succeeded - "status" => if(row["succeeded_at"] == nil, do: :initialized, else: :succeeded), - "succeeded_at" => row["succeeded_at"], - "reversed_at" => nil, - "group_id" => nil, - ## TODO: this might be null but shouldn't - "user_id" => user["id"], - "contract_id" => nil, - "original_contract_id" => nil, - "timesheet_id" => nil, - "bounty_id" => nil, - "tip_id" => nil, - "linked_transaction_id" => nil, - "inserted_at" => row["created_at"], - "updated_at" => row["updated_at"], - "claim_id" => nil - } + if !org do + raise "Org not found: #{inspect(row)}" + end - if bounty["type"] == "tip" do - Map.put(row, "tip_id", bounty["id"] <> user["id"]) - else - row - |> Map.put("bounty_id", claim["bounty_id"]) - |> Map.put("claim_id", claim["id"]) + if !bounty_charge do + raise "BountyCharge not found: #{inspect(row)}" end + + [ + maybe_create_transaction("debit", %{ + bounty_charge: bounty_charge, + bounty_transfer: row, + bounty: bounty, + claim: claim, + user: org + }), + maybe_create_transaction("credit", %{ + bounty_charge: bounty_charge, + bounty_transfer: row, + bounty: bounty, + claim: claim, + user: user + }), + maybe_create_transaction("transfer", %{ + bounty_charge: bounty_charge, + bounty_transfer: row, + bounty: bounty, + claim: claim, + user: user + }) + ] end defp transform({"GithubInstallation", Installation}, row, _db) do @@ -566,7 +557,7 @@ defmodule DatabaseMigration do "avatar_url" => nil, "repository_selection" => nil, "owner_id" => nil, - "connected_user_id" => nil, + "connected_user_id" => row["org_id"], "inserted_at" => row["created_at"], "updated_at" => row["updated_at"], "provider_user_id" => nil @@ -631,6 +622,102 @@ defmodule DatabaseMigration do defp transform(_, _row, _db), do: nil + defp maybe_create_transaction(type, %{ + bounty_charge: bounty_charge, + bounty_transfer: bounty_transfer, + bounty: bounty, + claim: claim, + user: user + }) do + amount = Money.from_integer(String.to_integer(bounty_transfer["amount"]), bounty_transfer["currency"]) + + res = %{ + "id" => String.slice(type, 0, 2) <> "_" <> bounty_transfer["id"], + "provider" => "stripe", + "provider_id" => nil, + "provider_charge_id" => bounty_charge["charge_id"], + "provider_payment_intent_id" => nil, + "provider_transfer_id" => bounty_transfer["transfer_id"], + "provider_invoice_id" => nil, + "provider_balance_transaction_id" => nil, + "provider_meta" => nil, + "gross_amount" => amount, + "net_amount" => amount, + "total_fee" => Money.zero(:USD), + "provider_fee" => nil, + "line_items" => nil, + "type" => type, + "status" => nil, + "succeeded_at" => nil, + "reversed_at" => nil, + "group_id" => bounty_charge["id"], + ## TODO: this might be null but shouldn't + "user_id" => user["id"], + "contract_id" => nil, + "original_contract_id" => nil, + "timesheet_id" => nil, + "bounty_id" => nil, + "tip_id" => nil, + "linked_transaction_id" => nil, + "inserted_at" => nil, + "updated_at" => nil, + "claim_id" => nil + } + + res = + if bounty["type"] == "tip" do + Map.put(res, "tip_id", bounty["id"] <> user["id"]) + else + res + |> Map.put("bounty_id", claim["bounty_id"]) + |> Map.put("claim_id", claim["id"]) + end + + res = + case type do + "transfer" -> + Map.put(res, "provider_id", bounty_transfer["transfer_id"]) + + "debit" -> + Map.put(res, "linked_transaction_id", "cr_" <> bounty_transfer["id"]) + + "credit" -> + Map.put(res, "linked_transaction_id", "de_" <> bounty_transfer["id"]) + + _ -> + res + end + + cond do + type == "transfer" && bounty_transfer["succeeded_at"] != nil -> + Map.merge(res, %{ + "status" => :succeeded, + "succeeded_at" => bounty_transfer["succeeded_at"], + "inserted_at" => bounty_transfer["created_at"], + "updated_at" => bounty_transfer["updated_at"] + }) + + type == "debit" && bounty_charge["succeeded_at"] != nil -> + Map.merge(res, %{ + "status" => :succeeded, + "succeeded_at" => bounty_charge["succeeded_at"], + "inserted_at" => bounty_charge["succeeded_at"], + "updated_at" => bounty_charge["succeeded_at"] + }) + + type == "credit" && bounty_charge["succeeded_at"] != nil -> + Map.merge(res, %{ + "status" => :succeeded, + "succeeded_at" => bounty_charge["succeeded_at"], + "inserted_at" => bounty_charge["succeeded_at"], + "updated_at" => bounty_charge["succeeded_at"] + }) + + true -> + nil + end + end + def process_dump(input_file, output_file) do db = collect_data(input_file) @@ -715,9 +802,14 @@ defmodule DatabaseMigration do defp transform_section(%{table: table, columns: _columns, data: data}, schema, db) do transformed_data = data - |> Enum.map(fn row -> + |> Enum.flat_map(fn row -> # try do - transform({table, schema}, row, db) + case transform({table, schema}, row, db) do + nil -> [] + xs when is_list(xs) -> xs + x -> [x] + end + # rescue # e -> # IO.puts("Error transforming row in table #{table}: #{inspect(row)}") @@ -725,16 +817,12 @@ defmodule DatabaseMigration do # nil # end end) - |> Enum.reject(&is_nil/1) |> Enum.map(&post_transform(schema, &1)) - transformed_table_name = schema.__schema__(:source) - if Enum.empty?(transformed_data) do nil else - transformed_columns = Map.keys(hd(transformed_data)) - %{table: transformed_table_name, columns: transformed_columns, data: transformed_data} + %{table: schema.__schema__(:source), columns: Map.keys(hd(transformed_data)), data: transformed_data} end end diff --git a/scripts/v1-progress.yaml b/scripts/v1-progress.yaml index dd00626fe..773460a22 100644 --- a/scripts/v1-progress.yaml +++ b/scripts/v1-progress.yaml @@ -90,7 +90,7 @@ - updated_at: 1 - succeeded_at: 1 - charge_id: 1 - - user_id: 0 + - user_id: -1 - org_id: 1 - amount: 1 - currency: 1 @@ -113,7 +113,7 @@ - created_at: 1 - updated_at: 1 - succeeded_at: 1 - - bounty_charge_id: 0 + - bounty_charge_id: 1 - claim_id: 1 - transfer_id: 1 - amount: 1 @@ -203,8 +203,8 @@ - created_at: 1 - updated_at: 1 - github_id: 1 - - org_id: 0 - - github_org_handle: 0 + - org_id: 1 + - github_org_handle: -1 - "GithubIssue": - id: 1 - url: -1 diff --git a/scripts/v2-progress.yaml b/scripts/v2-progress.yaml index 304c45da5..cbbdb6111 100644 --- a/scripts/v2-progress.yaml +++ b/scripts/v2-progress.yaml @@ -45,7 +45,7 @@ - id: 1 - status: 0 - type: 0 - - url: 0 + - url: 1 - group_id: 0 - group_share: 0 - source_id: 1 @@ -125,11 +125,11 @@ - provider_meta: -1 - avatar_url: -1 - repository_selection: -1 - - owner_id: 0 - - connected_user_id: 0 + - owner_id: -2 + - connected_user_id: 1 - inserted_at: 1 - updated_at: 1 - - provider_user_id: 0 + - provider_user_id: -1 - members: - id: 1 - role: 1 @@ -194,7 +194,7 @@ - description: 1 - number: 1 - url: 1 - - repository_id: 0 + - repository_id: -1 - inserted_at: 1 - updated_at: 1 - timesheets: @@ -233,14 +233,14 @@ - status: 1 - succeeded_at: 1 - reversed_at: -1 - - group_id: 0 - - user_id: 0 + - group_id: 1 + - user_id: 1 - contract_id: -1 - original_contract_id: -1 - timesheet_id: -1 - bounty_id: 1 - - tip_id: 0 - - linked_transaction_id: 0 + - tip_id: 1 + - linked_transaction_id: 1 - inserted_at: 1 - updated_at: 1 - claim_id: 1 From a7dba3f21bee3f4a5f1e7531b67bdbf397579c53 Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 09:24:47 +0300 Subject: [PATCH 28/51] add missing claim fields --- scripts/database_migration.exs | 9 +++++++-- scripts/v2-progress.yaml | 6 +++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index c7b7a2130..9eed4d5ed 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -403,9 +403,14 @@ defmodule DatabaseMigration do %{ "id" => row["id"], "status" => nil, - "type" => nil, + "type" => + cond do + row["github_pull_request_id"] != nil -> "pull_request" + String.match?(row["github_url"] || "", ~r{^https?://(?:www\.)?figma\.com/}) -> "design" + true -> "pull_request" + end, "url" => row["github_url"], - "group_id" => nil, + "group_id" => row["id"], "group_share" => nil, "source_id" => github_pull_request["task_id"], "target_id" => task["id"], diff --git a/scripts/v2-progress.yaml b/scripts/v2-progress.yaml index cbbdb6111..b41fbf4ce 100644 --- a/scripts/v2-progress.yaml +++ b/scripts/v2-progress.yaml @@ -44,10 +44,10 @@ - claims: - id: 1 - status: 0 - - type: 0 + - type: 1 - url: 1 - - group_id: 0 - - group_share: 0 + - group_id: 1 + - group_share: -1 - source_id: 1 - target_id: 1 - user_id: 1 From 24630db0456ecff42e5c0ee5e78ba45eda402f76 Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 09:24:55 +0300 Subject: [PATCH 29/51] skip bonuses for now --- scripts/v1-progress.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/v1-progress.yaml b/scripts/v1-progress.yaml index 773460a22..706c86690 100644 --- a/scripts/v1-progress.yaml +++ b/scripts/v1-progress.yaml @@ -45,17 +45,17 @@ - currency: -1 - status: -1 - "Bonus": - - id: 0 - - created_at: 0 - - updated_at: 0 - - amount: 0 - - currency: 0 - - poster_id: 0 - - org_id: 0 - - claim_id: 0 - - github_req_comment_id: 0 - - github_res_comment_id: 0 - - type: 0 + - id: -1 + - created_at: -1 + - updated_at: -1 + - amount: -1 + - currency: -1 + - poster_id: -1 + - org_id: -1 + - claim_id: -1 + - github_req_comment_id: -1 + - github_res_comment_id: -1 + - type: -1 - "BotMessage": - id: -1 - created_at: -1 From 8e7b2ea10150272e58623e40fe0d0d2b27c5ad50 Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 09:32:46 +0300 Subject: [PATCH 30/51] update progress --- scripts/analyze_progress.exs | 73 +++++++++++++++++++++++++++++------- scripts/v1-progress.yaml | 2 +- 2 files changed, 61 insertions(+), 14 deletions(-) diff --git a/scripts/analyze_progress.exs b/scripts/analyze_progress.exs index 83ec5a73d..a62191cd2 100644 --- a/scripts/analyze_progress.exs +++ b/scripts/analyze_progress.exs @@ -1,44 +1,64 @@ defmodule ProgressAnalyzer do @moduledoc false - def analyze_file(path) do + require Logger + + def analyze_file(path, verbose \\ false) do stats = path |> File.read!() |> YamlElixir.read_from_string!() |> count_statuses() - display_stats(stats) + display_stats(stats, verbose) end defp count_statuses(yaml) do # Initialize counters - initial_counts = %{completed: 0, skipped: 0, remaining: 0, undecided: 0} + initial_counts = %{completed: 0, skipped: 0, remaining: 0, undecided: 0, remaining_columns: []} # Flatten and count all column statuses Enum.reduce(yaml, initial_counts, fn table, acc -> columns = table |> Map.values() |> List.first() Enum.reduce(columns, acc, fn column, inner_acc -> - {_col, status} = Enum.at(Map.to_list(column), 0) + {col, status} = Enum.at(Map.to_list(column), 0) case status do - 1 -> Map.update!(inner_acc, :completed, &(&1 + 1)) - -2 -> Map.update!(inner_acc, :undecided, &(&1 + 1)) - -1 -> Map.update!(inner_acc, :skipped, &(&1 + 1)) - 0 -> Map.update!(inner_acc, :remaining, &(&1 + 1)) + 1 -> + Map.update!(inner_acc, :completed, &(&1 + 1)) + + -2 -> + Map.update!(inner_acc, :undecided, &(&1 + 1)) + + -1 -> + Map.update!(inner_acc, :skipped, &(&1 + 1)) + + 0 -> + inner_acc + |> Map.update!(:remaining, &(&1 + 1)) + |> Map.update!(:remaining_columns, &(&1 ++ ["#{table |> Map.keys() |> List.first()}.#{col}"])) end end) end) end - defp display_stats(%{completed: done, skipped: skipped, remaining: todo, undecided: undecided}) do + defp display_stats( + %{ + completed: done, + skipped: skipped, + remaining: todo, + undecided: undecided, + remaining_columns: remaining_columns + }, + verbose + ) do total = done + skipped + todo + undecided done_pct = percentage(done, total) skipped_pct = percentage(skipped, total) todo_pct = percentage(todo, total) undecided_pct = percentage(undecided, total) - IO.puts(""" + base_output = """ Migration Progress Report ======================== @@ -56,7 +76,22 @@ defmodule ProgressAnalyzer do Progress: --------- [#{progress_bar(done_pct, skipped_pct, todo_pct, undecided_pct)}] - """) + """ + + if_result = + if verbose do + base_output <> + """ + + Remaining Columns: + ------------------ + #{Enum.join(remaining_columns, "\n")} + """ + else + base_output + end + + IO.puts(if_result) end defp percentage(part, total) when total > 0 do @@ -77,10 +112,22 @@ defmodule ProgressAnalyzer do end case System.argv() do + [filename, "-v"] -> + ProgressAnalyzer.analyze_file(filename, true) + + ["-v", filename] -> + ProgressAnalyzer.analyze_file(filename, true) + [filename] -> - ProgressAnalyzer.analyze_file(filename) + ProgressAnalyzer.analyze_file(filename, false) _ -> - IO.puts("Usage: elixir analyze_progress.exs ") + IO.puts(""" + Usage: elixir analyze_progress.exs [-v] + + Options: + -v Show detailed remaining columns list + """) + System.halt(1) end diff --git a/scripts/v1-progress.yaml b/scripts/v1-progress.yaml index 706c86690..68a924005 100644 --- a/scripts/v1-progress.yaml +++ b/scripts/v1-progress.yaml @@ -137,7 +137,7 @@ - bounty_charge_id: -1 - github_res_comment_id: 0 - info: 0 - - share: 0 + - share: -1 - "Company": - id: -1 - created_at: -1 From f01a3b5c2c8f7d054d417de796e71dcb6069d537 Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 18:13:56 +0300 Subject: [PATCH 31/51] fix miscellaneous issues --- lib/algora/payments/schemas/account.ex | 4 +- .../workspace/schemas/command_response.ex | 2 +- lib/algora/workspace/schemas/installation.ex | 4 +- ...0208173814_make_provider_meta_nullable.exs | 69 ++++++++++ scripts/database_migration.exs | 127 ++++++++++-------- 5 files changed, 148 insertions(+), 58 deletions(-) create mode 100644 priv/repo/migrations/20250208173814_make_provider_meta_nullable.exs diff --git a/lib/algora/payments/schemas/account.ex b/lib/algora/payments/schemas/account.ex index ec6de5847..07805fef5 100644 --- a/lib/algora/payments/schemas/account.ex +++ b/lib/algora/payments/schemas/account.ex @@ -9,7 +9,7 @@ defmodule Algora.Payments.Account do typed_schema "accounts" do field :provider, :string, null: false field :provider_id, :string, null: false - field :provider_meta, :map, null: false + field :provider_meta, :map field :name, :string field :details_submitted, :boolean, default: false, null: false @@ -19,7 +19,7 @@ defmodule Algora.Payments.Account do field :payout_speed, :integer field :default_currency, :string field :service_agreement, :string - field :country, :string, null: false + field :country, :string field :type, Ecto.Enum, values: [:standard, :express], null: false field :stale, :boolean, default: false, null: false diff --git a/lib/algora/workspace/schemas/command_response.ex b/lib/algora/workspace/schemas/command_response.ex index d4f4b173d..25192b4de 100644 --- a/lib/algora/workspace/schemas/command_response.ex +++ b/lib/algora/workspace/schemas/command_response.ex @@ -7,7 +7,7 @@ defmodule Algora.Workspace.CommandResponse do typed_schema "command_responses" do field :provider, :string, null: false - field :provider_meta, :map, null: false + field :provider_meta, :map field :provider_command_id, :string field :provider_response_id, :string, null: false field :command_source, Ecto.Enum, values: [:ticket, :comment], null: false diff --git a/lib/algora/workspace/schemas/installation.ex b/lib/algora/workspace/schemas/installation.ex index 33878269b..ec164c0f2 100644 --- a/lib/algora/workspace/schemas/installation.ex +++ b/lib/algora/workspace/schemas/installation.ex @@ -9,8 +9,8 @@ defmodule Algora.Workspace.Installation do typed_schema "installations" do field :provider, :string, null: false field :provider_id, :string, null: false - field :provider_meta, :map, null: false - field :provider_user_id, :string, null: false + field :provider_meta, :map + field :provider_user_id, :string field :avatar_url, :string field :repository_selection, :string diff --git a/priv/repo/migrations/20250208173814_make_provider_meta_nullable.exs b/priv/repo/migrations/20250208173814_make_provider_meta_nullable.exs new file mode 100644 index 000000000..846f82000 --- /dev/null +++ b/priv/repo/migrations/20250208173814_make_provider_meta_nullable.exs @@ -0,0 +1,69 @@ +defmodule Algora.Repo.Migrations.MakeProviderMetaNullable do + use Ecto.Migration + + def up do + alter table(:command_responses) do + modify :provider_meta, :map, null: true + end + + alter table(:installations) do + modify :provider_meta, :map, null: true + modify :provider_user_id, :string, null: true + end + + alter table(:customers) do + modify :provider_meta, :map, null: true + end + + alter table(:payment_methods) do + modify :provider_meta, :map, null: true + end + + alter table(:accounts) do + modify :provider_meta, :map, null: true + modify :country, :string, null: true + end + + alter table(:identities) do + modify :provider_meta, :map, null: true + modify :provider_login, :string, null: true + end + + alter table(:claims) do + modify :source_id, :string, null: true + end + end + + def down do + alter table(:command_responses) do + modify :provider_meta, :map, null: false + end + + alter table(:installations) do + modify :provider_meta, :map, null: false + modify :provider_user_id, :string, null: false + end + + alter table(:customers) do + modify :provider_meta, :map, null: false + end + + alter table(:payment_methods) do + modify :provider_meta, :map, null: false + end + + alter table(:accounts) do + modify :provider_meta, :map, null: false + modify :country, :string, null: false + end + + alter table(:identities) do + modify :provider_meta, :map, null: false + modify :provider_login, :string, null: false + end + + alter table(:claims) do + modify :source_id, :string, null: false + end + end +end diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index 9eed4d5ed..c8d157507 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -130,8 +130,8 @@ defmodule DatabaseMigration do "description" => row["body"], "number" => row["number"], "url" => "https://github.com/#{row["repo_owner"]}/#{row["repo_name"]}/issues/#{row["number"]}", - "inserted_at" => row["created_at"], - "updated_at" => row["updated_at"] + "inserted_at" => "1970-01-01 00:00:00", + "updated_at" => "1970-01-01 00:00:00" } end @@ -298,13 +298,19 @@ defmodule DatabaseMigration do defp transform({"GithubUser", User}, _row, _db), do: nil - defp transform({"Account", Identity}, row, _db) do + defp transform({"Account", Identity}, row, db) do + user = db |> Map.get("User", []) |> Enum.find(&(&1["id"] == row["\"userId\""])) + + if !user do + raise "User not found: #{inspect(row)}" + end + %{ "id" => row["id"], - "user_id" => row["\"userId\""], + "user_id" => user["id"], "provider" => row["provider"], "provider_token" => row["access_token"], - "provider_email" => nil, + "provider_email" => user["email"], "provider_login" => nil, "provider_id" => row["\"providerAccountId\""], "provider_meta" => nil, @@ -341,7 +347,11 @@ defmodule DatabaseMigration do end defp transform({"Bounty", CommandResponse}, row, _db) do - if row["github_res_comment_id"] != nil do + if row["task_id"] == "clo0q1x540000mj0ghdgmezqw" do + IO.inspect(row, label: "transform(Bounty -> CommandResponse)") + end + + if !nullish?(row["github_res_comment_id"]) do %{ "id" => row["id"], "provider" => "github", @@ -362,7 +372,7 @@ defmodule DatabaseMigration do github_user = db |> Map.get("GithubUser", []) |> Enum.find(&(&1["id"] == row["github_user_id"])) - user = db |> Map.get("User", []) |> Enum.find(&(&1["id"] == github_user["user_id"])) + user = db |> Map.get("User", []) |> Enum.find(&(&1["id"] == github_user["user_id"] || github_user["id"])) if !bounty do raise "Bounty not found: #{inspect(row)}" @@ -390,7 +400,7 @@ defmodule DatabaseMigration do github_user = db |> Map.get("GithubUser", []) |> Enum.find(&(&1["id"] == row["github_user_id"])) - user = db |> Map.get("User", []) |> Enum.find(&(&1["id"] == github_user["user_id"])) + user = db |> Map.get("User", []) |> Enum.find(&(&1["id"] == github_user["user_id"] || github_user["id"])) # TODO: this might be null github_pull_request = @@ -402,10 +412,11 @@ defmodule DatabaseMigration do %{ "id" => row["id"], - "status" => nil, + # TODO: + "status" => :pending, "type" => cond do - row["github_pull_request_id"] != nil -> "pull_request" + !nullish?(row["github_pull_request_id"]) -> "pull_request" String.match?(row["github_url"] || "", ~r{^https?://(?:www\.)?figma\.com/}) -> "design" true -> "pull_request" end, @@ -468,9 +479,9 @@ defmodule DatabaseMigration do defp transform({"BountyTransfer", Tip}, row, db) do claim = db |> Map.get("Claim", []) |> Enum.find(&(&1["id"] == row["claim_id"])) - github_user = db |> Map.get("GithubUser", []) |> Enum.find(&(&1["id"] == row["github_user_id"])) + github_user = db |> Map.get("GithubUser", []) |> Enum.find(&(&1["id"] == claim["github_user_id"])) - user = db |> Map.get("User", []) |> Enum.find(&(&1["id"] == github_user["user_id"])) + user = db |> Map.get("User", []) |> Enum.find(&(&1["id"] == github_user["user_id"] || github_user["id"])) bounty = db |> Map.get("Bounty", []) |> Enum.find(&(&1["id"] == claim["bounty_id"])) @@ -506,7 +517,7 @@ defmodule DatabaseMigration do github_user = db |> Map.get("GithubUser", []) |> Enum.find(&(&1["id"] == claim["github_user_id"])) - user = db |> Map.get("User", []) |> Enum.find(&(&1["id"] == github_user["user_id"])) + user = db |> Map.get("User", []) |> Enum.find(&(&1["id"] == github_user["user_id"] || github_user["id"])) org = db |> Map.get("Org", []) |> Enum.find(&(&1["id"] == bounty["org_id"])) @@ -528,29 +539,32 @@ defmodule DatabaseMigration do raise "BountyCharge not found: #{inspect(row)}" end - [ - maybe_create_transaction("debit", %{ - bounty_charge: bounty_charge, - bounty_transfer: row, - bounty: bounty, - claim: claim, - user: org - }), - maybe_create_transaction("credit", %{ - bounty_charge: bounty_charge, - bounty_transfer: row, - bounty: bounty, - claim: claim, - user: user - }), - maybe_create_transaction("transfer", %{ - bounty_charge: bounty_charge, - bounty_transfer: row, - bounty: bounty, - claim: claim, - user: user - }) - ] + Enum.reject( + [ + maybe_create_transaction("debit", %{ + bounty_charge: bounty_charge, + bounty_transfer: row, + bounty: bounty, + claim: claim, + user: org + }), + maybe_create_transaction("credit", %{ + bounty_charge: bounty_charge, + bounty_transfer: row, + bounty: bounty, + claim: claim, + user: user + }), + maybe_create_transaction("transfer", %{ + bounty_charge: bounty_charge, + bounty_transfer: row, + bounty: bounty, + claim: claim, + user: user + }) + ], + &is_nil/1 + ) end defp transform({"GithubInstallation", Installation}, row, _db) do @@ -573,7 +587,7 @@ defmodule DatabaseMigration do %{ "id" => row["id"], "provider" => "stripe", - "provider_id" => row["stripe_id"], + "provider_id" => row["id"], "provider_meta" => nil, "name" => nil, "details_submitted" => row["details_submitted"], @@ -606,7 +620,7 @@ defmodule DatabaseMigration do end defp transform({"StripePaymentMethod", PaymentMethod}, row, db) do - customer = db |> Map.get("StripeCustomer", []) |> Enum.find(&(&1["id"] == row["customer_id"])) + customer = db |> Map.get("StripeCustomer", []) |> Enum.find(&(&1["org_id"] == row["org_id"])) if !customer do raise "StripeCustomer not found: #{inspect(row)}" @@ -619,7 +633,7 @@ defmodule DatabaseMigration do "provider_meta" => nil, "provider_customer_id" => customer["stripe_id"], "is_default" => row["is_default"], - "customer_id" => row["customer_id"], + "customer_id" => customer["id"], "inserted_at" => row["created_at"], "updated_at" => row["updated_at"] } @@ -694,7 +708,7 @@ defmodule DatabaseMigration do end cond do - type == "transfer" && bounty_transfer["succeeded_at"] != nil -> + type == "transfer" && !nullish?(bounty_transfer["succeeded_at"]) -> Map.merge(res, %{ "status" => :succeeded, "succeeded_at" => bounty_transfer["succeeded_at"], @@ -702,7 +716,7 @@ defmodule DatabaseMigration do "updated_at" => bounty_transfer["updated_at"] }) - type == "debit" && bounty_charge["succeeded_at"] != nil -> + type == "debit" && !nullish?(bounty_charge["succeeded_at"]) -> Map.merge(res, %{ "status" => :succeeded, "succeeded_at" => bounty_charge["succeeded_at"], @@ -710,7 +724,7 @@ defmodule DatabaseMigration do "updated_at" => bounty_charge["succeeded_at"] }) - type == "credit" && bounty_charge["succeeded_at"] != nil -> + type == "credit" && !nullish?(bounty_charge["succeeded_at"]) -> Map.merge(res, %{ "status" => :succeeded, "succeeded_at" => bounty_charge["succeeded_at"], @@ -1037,6 +1051,8 @@ defmodule DatabaseMigration do defp deserialize_value(value), do: value + defp nullish?(value), do: is_nil(deserialize_value(value)) + defp clear_tables! do commands = [ @@ -1073,24 +1089,29 @@ defmodule DatabaseMigration do end end + defp time_step(description, function) do + IO.puts("\n#{description}...") + {time, result} = :timer.tc(function) + IO.puts("✓ #{description} completed in #{time / 1_000_000} seconds") + result + end + def run! do input_file = ".local/prod_db.sql" output_file = ".local/prod_db_new.sql" if File.exists?(input_file) or File.exists?(output_file) do - IO.puts("Processing dump...") - :ok = process_dump(input_file, output_file) - - IO.puts("Clearing tables...") - :ok = clear_tables!() - - IO.puts("Importing new data...") - {:ok, _} = psql(["-f", output_file]) + IO.puts("\nStarting migration...") - IO.puts("Backfilling repositories...") - :ok = Algora.Admin.backfill_repos!() + {total_time, _} = + :timer.tc(fn -> + :ok = time_step("Processing dump", fn -> process_dump(input_file, output_file) end) + :ok = time_step("Clearing tables", fn -> clear_tables!() end) + {:ok, _} = time_step("Importing new data", fn -> psql(["-f", output_file]) end) + :ok = time_step("Backfilling repositories", fn -> Algora.Admin.backfill_repos!() end) + end) - IO.puts("Migration completed successfully") + IO.puts("\n✓ Migration completed successfully in #{total_time / 1_000_000} seconds") end end end From 37e08b084a06cc68a015309e3fa52dcfd1f96ecb Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 18:49:34 +0300 Subject: [PATCH 32/51] fix miscellaneous issues --- ...0208173814_make_provider_meta_nullable.exs | 35 +-- scripts/database_migration.exs | 226 +++++++++--------- 2 files changed, 117 insertions(+), 144 deletions(-) diff --git a/priv/repo/migrations/20250208173814_make_provider_meta_nullable.exs b/priv/repo/migrations/20250208173814_make_provider_meta_nullable.exs index 846f82000..869ff1a33 100644 --- a/priv/repo/migrations/20250208173814_make_provider_meta_nullable.exs +++ b/priv/repo/migrations/20250208173814_make_provider_meta_nullable.exs @@ -1,7 +1,7 @@ defmodule Algora.Repo.Migrations.MakeProviderMetaNullable do use Ecto.Migration - def up do + def change do alter table(:command_responses) do modify :provider_meta, :map, null: true end @@ -32,38 +32,9 @@ defmodule Algora.Repo.Migrations.MakeProviderMetaNullable do alter table(:claims) do modify :source_id, :string, null: true end - end - - def down do - alter table(:command_responses) do - modify :provider_meta, :map, null: false - end - - alter table(:installations) do - modify :provider_meta, :map, null: false - modify :provider_user_id, :string, null: false - end - - alter table(:customers) do - modify :provider_meta, :map, null: false - end - alter table(:payment_methods) do - modify :provider_meta, :map, null: false - end - - alter table(:accounts) do - modify :provider_meta, :map, null: false - modify :country, :string, null: false - end - - alter table(:identities) do - modify :provider_meta, :map, null: false - modify :provider_login, :string, null: false - end - - alter table(:claims) do - modify :source_id, :string, null: false + alter table(:tips) do + modify :creator_id, :string, null: true end end end diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index c8d157507..eada06363 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -243,61 +243,61 @@ defmodule DatabaseMigration do } end - defp transform({"GithubUser", User}, %{user_id: nil} = row, _db) do + defp transform({"GithubUser", User}, row, _db) do if row["type"] != "User" do - raise "GithubUser is not a User: #{inspect(row)}" + Logger.warning("GithubUser is not a User: #{inspect(row["login"])}") end - %{ - "id" => row["id"], - "provider" => "github", - "provider_id" => row["id"], - "provider_login" => row["login"], - "provider_meta" => deserialize_value(row), - "email" => nil, - "display_name" => row["name"], - "handle" => nil, - "avatar_url" => row["avatar_url"], - "external_homepage_url" => nil, - "type" => "individual", - "bio" => row["bio"], - "location" => row["location"], - "country" => nil, - "timezone" => nil, - "stargazers_count" => nil, - "domain" => nil, - "tech_stack" => nil, - "featured" => nil, - "priority" => nil, - "fee_pct" => nil, - "seeded" => nil, - "activated" => nil, - "max_open_attempts" => nil, - "manual_assignment" => nil, - "bounty_mode" => nil, - "hourly_rate_min" => nil, - "hourly_rate_max" => nil, - "hours_per_week" => nil, - "website_url" => nil, - "twitter_url" => row["twitter_username"] && "https://www.twitter.com/#{row["twitter_username"]}", - "github_url" => nil, - "youtube_url" => nil, - "twitch_url" => nil, - "discord_url" => nil, - "slack_url" => nil, - "linkedin_url" => nil, - "og_title" => nil, - "og_image_url" => nil, - "last_context" => nil, - "need_avatar" => nil, - "inserted_at" => row["retrieved_at"], - "updated_at" => row["retrieved_at"], - "is_admin" => nil - } + if nullish?(row["user_id"]) do + %{ + "id" => row["id"], + "provider" => "github", + "provider_id" => row["id"], + "provider_login" => row["login"], + "provider_meta" => deserialize_value(row), + "email" => nil, + "display_name" => row["name"], + "handle" => nil, + "avatar_url" => row["avatar_url"], + "external_homepage_url" => nil, + "type" => "individual", + "bio" => row["bio"], + "location" => row["location"], + "country" => nil, + "timezone" => nil, + "stargazers_count" => nil, + "domain" => nil, + "tech_stack" => nil, + "featured" => nil, + "priority" => nil, + "fee_pct" => nil, + "seeded" => nil, + "activated" => nil, + "max_open_attempts" => nil, + "manual_assignment" => nil, + "bounty_mode" => nil, + "hourly_rate_min" => nil, + "hourly_rate_max" => nil, + "hours_per_week" => nil, + "website_url" => nil, + "twitter_url" => row["twitter_username"] && "https://www.twitter.com/#{row["twitter_username"]}", + "github_url" => nil, + "youtube_url" => nil, + "twitch_url" => nil, + "discord_url" => nil, + "slack_url" => nil, + "linkedin_url" => nil, + "og_title" => nil, + "og_image_url" => nil, + "last_context" => nil, + "need_avatar" => nil, + "inserted_at" => row["retrieved_at"], + "updated_at" => row["retrieved_at"], + "is_admin" => nil + } + end end - defp transform({"GithubUser", User}, _row, _db), do: nil - defp transform({"Account", Identity}, row, db) do user = db |> Map.get("User", []) |> Enum.find(&(&1["id"] == row["\"userId\""])) @@ -347,10 +347,6 @@ defmodule DatabaseMigration do end defp transform({"Bounty", CommandResponse}, row, _db) do - if row["task_id"] == "clo0q1x540000mj0ghdgmezqw" do - IO.inspect(row, label: "transform(Bounty -> CommandResponse)") - end - if !nullish?(row["github_res_comment_id"]) do %{ "id" => row["id"], @@ -372,13 +368,13 @@ defmodule DatabaseMigration do github_user = db |> Map.get("GithubUser", []) |> Enum.find(&(&1["id"] == row["github_user_id"])) - user = db |> Map.get("User", []) |> Enum.find(&(&1["id"] == github_user["user_id"] || github_user["id"])) + user_id = or_else(github_user["user_id"], github_user["id"]) if !bounty do raise "Bounty not found: #{inspect(row)}" end - if !user do + if nullish?(user_id) do raise "User not found: #{inspect(row)}" end @@ -387,7 +383,7 @@ defmodule DatabaseMigration do "status" => row["status"], "warnings_count" => row["warnings_count"], "ticket_id" => bounty["task_id"], - "user_id" => user["id"], + "user_id" => user_id, "inserted_at" => row["created_at"], "updated_at" => row["updated_at"] } @@ -400,14 +396,18 @@ defmodule DatabaseMigration do github_user = db |> Map.get("GithubUser", []) |> Enum.find(&(&1["id"] == row["github_user_id"])) - user = db |> Map.get("User", []) |> Enum.find(&(&1["id"] == github_user["user_id"] || github_user["id"])) + user_id = or_else(github_user["user_id"], github_user["id"]) # TODO: this might be null github_pull_request = db |> Map.get("GithubPullRequest", []) |> Enum.find(&(&1["id"] == row["github_pull_request_id"])) - if !task || !user do - raise "Task or User not found: #{inspect(row)}" + if !task do + raise "Task not found: #{inspect(row)}" + end + + if nullish?(user_id) do + raise "User not found: #{inspect(row)}" end %{ @@ -417,15 +417,15 @@ defmodule DatabaseMigration do "type" => cond do !nullish?(row["github_pull_request_id"]) -> "pull_request" - String.match?(row["github_url"] || "", ~r{^https?://(?:www\.)?figma\.com/}) -> "design" + String.match?(row["github_url"], ~r{^https?://(?:www\.)?figma\.com/}) -> "design" true -> "pull_request" end, - "url" => row["github_url"], + "url" => or_else(row["github_url"], "https://algora.io"), "group_id" => row["id"], "group_share" => nil, "source_id" => github_pull_request["task_id"], "target_id" => task["id"], - "user_id" => user["id"], + "user_id" => user_id, "inserted_at" => row["created_at"], "updated_at" => row["updated_at"] } @@ -436,44 +436,45 @@ defmodule DatabaseMigration do amount = Money.from_integer(String.to_integer(row["amount"]), row["currency"]) - if !user || row["succeeded_at"] == nil do + if !user do raise "User not found: #{inspect(row)}" end - %{ - "id" => row["id"], - "provider" => "stripe", - "provider_id" => row["charge_id"], - "provider_charge_id" => row["charge_id"], - "provider_payment_intent_id" => nil, - "provider_transfer_id" => nil, - "provider_invoice_id" => nil, - "provider_balance_transaction_id" => nil, - "provider_meta" => nil, - # TODO: incorrect - "gross_amount" => amount, - "net_amount" => amount, - # TODO: incorrect - "total_fee" => Money.zero(:USD), - "provider_fee" => nil, - "line_items" => nil, - "type" => "charge", - "status" => if(row["succeeded_at"] == nil, do: :initialized, else: :succeeded), - "succeeded_at" => row["succeeded_at"], - "reversed_at" => nil, - "group_id" => row["id"], - ## TODO: this might be null but shouldn't - "user_id" => user["id"], - "contract_id" => nil, - "original_contract_id" => nil, - "timesheet_id" => nil, - "bounty_id" => nil, - "tip_id" => nil, - "linked_transaction_id" => nil, - "inserted_at" => row["created_at"], - "updated_at" => row["updated_at"], - "claim_id" => nil - } + if !nullish?(row["succeeded_at"]) do + %{ + "id" => row["id"], + "provider" => "stripe", + "provider_id" => row["charge_id"], + "provider_charge_id" => row["charge_id"], + "provider_payment_intent_id" => nil, + "provider_transfer_id" => nil, + "provider_invoice_id" => nil, + "provider_balance_transaction_id" => nil, + "provider_meta" => nil, + # TODO: incorrect + "gross_amount" => amount, + "net_amount" => amount, + # TODO: incorrect + "total_fee" => Money.zero(:USD), + "provider_fee" => nil, + "line_items" => nil, + "type" => "charge", + "status" => if(nullish?(row["succeeded_at"]), do: :initialized, else: :succeeded), + "succeeded_at" => row["succeeded_at"], + "reversed_at" => nil, + "group_id" => row["id"], + "user_id" => user["id"], + "contract_id" => nil, + "original_contract_id" => nil, + "timesheet_id" => nil, + "bounty_id" => nil, + "tip_id" => nil, + "linked_transaction_id" => nil, + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"], + "claim_id" => nil + } + end end defp transform({"BountyTransfer", Tip}, row, db) do @@ -481,7 +482,7 @@ defmodule DatabaseMigration do github_user = db |> Map.get("GithubUser", []) |> Enum.find(&(&1["id"] == claim["github_user_id"])) - user = db |> Map.get("User", []) |> Enum.find(&(&1["id"] == github_user["user_id"] || github_user["id"])) + user_id = or_else(github_user["user_id"], github_user["id"]) bounty = db |> Map.get("Bounty", []) |> Enum.find(&(&1["id"] == claim["bounty_id"])) @@ -491,19 +492,19 @@ defmodule DatabaseMigration do raise "Bounty not found: #{inspect(row)}" end - if !user do + if nullish?(user_id) do raise "User not found: #{inspect(row)}" end if bounty["type"] == "tip" do %{ - "id" => bounty["id"] <> user["id"], + "id" => bounty["id"] <> user_id, "amount" => amount, "status" => nil, "ticket_id" => bounty["task_id"], "owner_id" => bounty["org_id"], "creator_id" => bounty["poster_id"], - "recipient_id" => user["id"], + "recipient_id" => user_id, "inserted_at" => bounty["created_at"], "updated_at" => bounty["updated_at"] } @@ -517,7 +518,7 @@ defmodule DatabaseMigration do github_user = db |> Map.get("GithubUser", []) |> Enum.find(&(&1["id"] == claim["github_user_id"])) - user = db |> Map.get("User", []) |> Enum.find(&(&1["id"] == github_user["user_id"] || github_user["id"])) + user_id = or_else(github_user["user_id"], github_user["id"]) org = db |> Map.get("Org", []) |> Enum.find(&(&1["id"] == bounty["org_id"])) @@ -527,7 +528,7 @@ defmodule DatabaseMigration do raise "Bounty not found: #{inspect(row)}" end - if !user do + if nullish?(user_id) do raise "User not found: #{inspect(row)}" end @@ -546,21 +547,21 @@ defmodule DatabaseMigration do bounty_transfer: row, bounty: bounty, claim: claim, - user: org + user_id: org["id"] }), maybe_create_transaction("credit", %{ bounty_charge: bounty_charge, bounty_transfer: row, bounty: bounty, claim: claim, - user: user + user_id: user_id }), maybe_create_transaction("transfer", %{ bounty_charge: bounty_charge, bounty_transfer: row, bounty: bounty, claim: claim, - user: user + user_id: user_id }) ], &is_nil/1 @@ -646,7 +647,7 @@ defmodule DatabaseMigration do bounty_transfer: bounty_transfer, bounty: bounty, claim: claim, - user: user + user_id: user_id }) do amount = Money.from_integer(String.to_integer(bounty_transfer["amount"]), bounty_transfer["currency"]) @@ -670,8 +671,7 @@ defmodule DatabaseMigration do "succeeded_at" => nil, "reversed_at" => nil, "group_id" => bounty_charge["id"], - ## TODO: this might be null but shouldn't - "user_id" => user["id"], + "user_id" => user_id, "contract_id" => nil, "original_contract_id" => nil, "timesheet_id" => nil, @@ -685,10 +685,10 @@ defmodule DatabaseMigration do res = if bounty["type"] == "tip" do - Map.put(res, "tip_id", bounty["id"] <> user["id"]) + Map.put(res, "tip_id", bounty["id"] <> user_id) else res - |> Map.put("bounty_id", claim["bounty_id"]) + |> Map.put("bounty_id", bounty["id"]) |> Map.put("claim_id", claim["id"]) end @@ -1053,6 +1053,8 @@ defmodule DatabaseMigration do defp nullish?(value), do: is_nil(deserialize_value(value)) + defp or_else(value, default), do: if(nullish?(value), do: default, else: value) + defp clear_tables! do commands = [ From 62975d33784827012db0fc56ff79f236d01cedbc Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 19:19:44 +0300 Subject: [PATCH 33/51] fix more issues --- lib/algora/bounties/schemas/bounty.ex | 1 + ...0208173814_make_provider_meta_nullable.exs | 7 +++ scripts/database_migration.exs | 48 ++++++++++++------- scripts/v1-progress.yaml | 2 +- 4 files changed, 39 insertions(+), 19 deletions(-) diff --git a/lib/algora/bounties/schemas/bounty.ex b/lib/algora/bounties/schemas/bounty.ex index 97346f085..b6ad925d8 100644 --- a/lib/algora/bounties/schemas/bounty.ex +++ b/lib/algora/bounties/schemas/bounty.ex @@ -9,6 +9,7 @@ defmodule Algora.Bounties.Bounty do typed_schema "bounties" do field :amount, Algora.Types.Money field :status, Ecto.Enum, values: [:open, :cancelled, :paid] + field :number, :integer, default: 0 belongs_to :ticket, Algora.Workspace.Ticket belongs_to :owner, User diff --git a/priv/repo/migrations/20250208173814_make_provider_meta_nullable.exs b/priv/repo/migrations/20250208173814_make_provider_meta_nullable.exs index 869ff1a33..a7a073931 100644 --- a/priv/repo/migrations/20250208173814_make_provider_meta_nullable.exs +++ b/priv/repo/migrations/20250208173814_make_provider_meta_nullable.exs @@ -36,5 +36,12 @@ defmodule Algora.Repo.Migrations.MakeProviderMetaNullable do alter table(:tips) do modify :creator_id, :string, null: true end + + alter table(:bounties) do + add :number, :integer, null: false, default: 0 + end + + drop unique_index(:bounties, [:ticket_id, :owner_id]) + create unique_index(:bounties, [:ticket_id, :owner_id, :number]) end end diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index eada06363..8b745e430 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -244,10 +244,6 @@ defmodule DatabaseMigration do end defp transform({"GithubUser", User}, row, _db) do - if row["type"] != "User" do - Logger.warning("GithubUser is not a User: #{inspect(row["login"])}") - end - if nullish?(row["user_id"]) do %{ "id" => row["id"], @@ -335,15 +331,18 @@ defmodule DatabaseMigration do amount = if reward, do: Money.from_integer(String.to_integer(reward["amount"]), reward["currency"]) - %{ - "id" => row["id"], - "amount" => amount, - "ticket_id" => row["task_id"], - "owner_id" => row["org_id"], - "creator_id" => row["poster_id"], - "inserted_at" => row["created_at"], - "updated_at" => row["updated_at"] - } + if row["type"] != "tip" do + %{ + "id" => row["id"], + "amount" => amount, + "ticket_id" => row["task_id"], + "owner_id" => row["org_id"], + "creator_id" => row["poster_id"], + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"], + "number" => row["number"] + } + end end defp transform({"Bounty", CommandResponse}, row, _db) do @@ -547,13 +546,15 @@ defmodule DatabaseMigration do bounty_transfer: row, bounty: bounty, claim: claim, - user_id: org["id"] + org: org, + user_id: user_id }), maybe_create_transaction("credit", %{ bounty_charge: bounty_charge, bounty_transfer: row, bounty: bounty, claim: claim, + org: org, user_id: user_id }), maybe_create_transaction("transfer", %{ @@ -561,6 +562,7 @@ defmodule DatabaseMigration do bounty_transfer: row, bounty: bounty, claim: claim, + org: org, user_id: user_id }) ], @@ -647,6 +649,7 @@ defmodule DatabaseMigration do bounty_transfer: bounty_transfer, bounty: bounty, claim: claim, + org: org, user_id: user_id }) do amount = Money.from_integer(String.to_integer(bounty_transfer["amount"]), bounty_transfer["currency"]) @@ -671,7 +674,7 @@ defmodule DatabaseMigration do "succeeded_at" => nil, "reversed_at" => nil, "group_id" => bounty_charge["id"], - "user_id" => user_id, + "user_id" => nil, "contract_id" => nil, "original_contract_id" => nil, "timesheet_id" => nil, @@ -695,13 +698,22 @@ defmodule DatabaseMigration do res = case type do "transfer" -> - Map.put(res, "provider_id", bounty_transfer["transfer_id"]) + Map.merge(res, %{ + "user_id" => user_id, + "provider_id" => bounty_transfer["transfer_id"] + }) "debit" -> - Map.put(res, "linked_transaction_id", "cr_" <> bounty_transfer["id"]) + Map.merge(res, %{ + "user_id" => org["id"], + "linked_transaction_id" => "cr_" <> bounty_transfer["id"] + }) "credit" -> - Map.put(res, "linked_transaction_id", "de_" <> bounty_transfer["id"]) + Map.merge(res, %{ + "user_id" => user_id, + "linked_transaction_id" => "de_" <> bounty_transfer["id"] + }) _ -> res diff --git a/scripts/v1-progress.yaml b/scripts/v1-progress.yaml index 68a924005..364ace109 100644 --- a/scripts/v1-progress.yaml +++ b/scripts/v1-progress.yaml @@ -77,7 +77,7 @@ - type: 0 - kind: 0 - visibility: 0 - - number: 0 + - number: 1 - deleted: 0 - reward_type: 0 - autopay_disabled: 0 From 42719c4c45f63bffb6437380c5888cb74918cec0 Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 19:42:05 +0300 Subject: [PATCH 34/51] fix more issues --- scripts/database_migration.exs | 54 ++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index 8b745e430..d52d84b05 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -610,36 +610,40 @@ defmodule DatabaseMigration do end defp transform({"StripeCustomer", Customer}, row, _db) do - %{ - "id" => row["id"], - "provider" => "stripe", - "provider_id" => row["stripe_id"], - "provider_meta" => nil, - "name" => row["name"], - "user_id" => row["org_id"], - "inserted_at" => row["created_at"], - "updated_at" => row["updated_at"] - } + if row["org_id"] not in ["clfqtao4h0001mo0gkp9az0bn", "cm251pvg40007ld031q5t2hj2", "cljo6j981000el60f1k1cvtns"] do + %{ + "id" => row["id"], + "provider" => "stripe", + "provider_id" => row["stripe_id"], + "provider_meta" => nil, + "name" => row["name"], + "user_id" => row["org_id"], + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"] + } + end end defp transform({"StripePaymentMethod", PaymentMethod}, row, db) do - customer = db |> Map.get("StripeCustomer", []) |> Enum.find(&(&1["org_id"] == row["org_id"])) + if row["org_id"] not in ["clfqtao4h0001mo0gkp9az0bn", "cm251pvg40007ld031q5t2hj2", "cljo6j981000el60f1k1cvtns"] do + customer = db |> Map.get("StripeCustomer", []) |> Enum.find(&(&1["org_id"] == row["org_id"])) - if !customer do - raise "StripeCustomer not found: #{inspect(row)}" - end + if !customer do + raise "StripeCustomer not found: #{inspect(row)}" + end - %{ - "id" => row["id"], - "provider" => "stripe", - "provider_id" => row["stripe_id"], - "provider_meta" => nil, - "provider_customer_id" => customer["stripe_id"], - "is_default" => row["is_default"], - "customer_id" => customer["id"], - "inserted_at" => row["created_at"], - "updated_at" => row["updated_at"] - } + %{ + "id" => row["id"], + "provider" => "stripe", + "provider_id" => row["stripe_id"], + "provider_meta" => nil, + "provider_customer_id" => customer["stripe_id"], + "is_default" => row["is_default"], + "customer_id" => customer["id"], + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"] + } + end end defp transform(_, _row, _db), do: nil From 5307c563d5a0cddd4540051c83a9e1300a90aa2e Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 19:54:24 +0300 Subject: [PATCH 35/51] speed up retrievals by building indexes upfront --- scripts/database_migration.exs | 124 +++++++++++++++++++-------------- 1 file changed, 73 insertions(+), 51 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index d52d84b05..54812011f 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -76,6 +76,20 @@ defmodule DatabaseMigration do "users" ] + @index_fields [ + {"GithubUser", ["id", "user_id"]}, + {"User", ["id"]}, + {"Org", ["id"]}, + {"Bounty", ["id", "task_id"]}, + {"Task", ["id"]}, + {"Claim", ["id", "bounty_id"]}, + {"BountyCharge", ["id"]}, + {"StripeCustomer", ["org_id"]}, + {"GithubIssue", ["id"]}, + {"GithubPullRequest", ["id", "task_id"]}, + {"Reward", ["bounty_id"]} + ] + defp relevant_tables, do: @schema_mappings |> Enum.map(fn {k, _v} -> k end) |> Enum.dedup() defp transform({"Task", Ticket}, row, db) do @@ -83,9 +97,8 @@ defmodule DatabaseMigration do raise "Unknown forge: #{row["forge"]}" end - github_issue = db |> Map.get("GithubIssue", []) |> Enum.find(&(&1["id"] == row["issue_id"])) - - github_pull_request = db |> Map.get("GithubPullRequest", []) |> Enum.find(&(&1["id"] == row["pull_request_id"])) + github_issue = find_by_index(db, "GithubIssue", "id", row["issue_id"]) + github_pull_request = find_by_index(db, "GithubPullRequest", "id", row["pull_request_id"]) row = cond do @@ -144,7 +157,7 @@ defmodule DatabaseMigration do # raise "Email not verified: #{inspect(row)}" # end - github_user = db |> Map.get("GithubUser", []) |> Enum.find(&(&1["user_id"] == row["id"])) + github_user = find_by_index(db, "GithubUser", "user_id", row["id"]) %{ "id" => row["id"], @@ -295,7 +308,7 @@ defmodule DatabaseMigration do end defp transform({"Account", Identity}, row, db) do - user = db |> Map.get("User", []) |> Enum.find(&(&1["id"] == row["\"userId\""])) + user = find_by_index(db, "User", "id", row["\"userId\""]) if !user do raise "User not found: #{inspect(row)}" @@ -327,7 +340,7 @@ defmodule DatabaseMigration do end defp transform({"Bounty", Bounty}, row, db) do - reward = db |> Map.get("Reward", []) |> Enum.find(&(&1["bounty_id"] == row["id"])) + reward = find_by_index(db, "Reward", "bounty_id", row["id"]) amount = if reward, do: Money.from_integer(String.to_integer(reward["amount"]), reward["currency"]) @@ -363,9 +376,8 @@ defmodule DatabaseMigration do end defp transform({"Attempt", Attempt}, row, db) do - bounty = db |> Map.get("Bounty", []) |> Enum.find(&(&1["id"] == row["bounty_id"])) - - github_user = db |> Map.get("GithubUser", []) |> Enum.find(&(&1["id"] == row["github_user_id"])) + bounty = find_by_index(db, "Bounty", "id", row["bounty_id"]) + github_user = find_by_index(db, "GithubUser", "id", row["github_user_id"]) user_id = or_else(github_user["user_id"], github_user["id"]) @@ -389,18 +401,13 @@ defmodule DatabaseMigration do end defp transform({"Claim", Claim}, row, db) do - bounty = db |> Map.get("Bounty", []) |> Enum.find(&(&1["id"] == row["bounty_id"])) - - task = db |> Map.get("Task", []) |> Enum.find(&(&1["id"] == bounty["task_id"])) - - github_user = db |> Map.get("GithubUser", []) |> Enum.find(&(&1["id"] == row["github_user_id"])) + bounty = find_by_index(db, "Bounty", "id", row["bounty_id"]) + task = find_by_index(db, "Task", "id", bounty["task_id"]) + github_user = find_by_index(db, "GithubUser", "id", row["github_user_id"]) + github_pull_request = find_by_index(db, "GithubPullRequest", "id", row["github_pull_request_id"]) user_id = or_else(github_user["user_id"], github_user["id"]) - # TODO: this might be null - github_pull_request = - db |> Map.get("GithubPullRequest", []) |> Enum.find(&(&1["id"] == row["github_pull_request_id"])) - if !task do raise "Task not found: #{inspect(row)}" end @@ -431,7 +438,7 @@ defmodule DatabaseMigration do end defp transform({"BountyCharge", Transaction}, row, db) do - user = db |> Map.get("Org", []) |> Enum.find(&(&1["id"] == row["org_id"])) + user = find_by_index(db, "Org", "id", row["org_id"]) amount = Money.from_integer(String.to_integer(row["amount"]), row["currency"]) @@ -477,14 +484,11 @@ defmodule DatabaseMigration do end defp transform({"BountyTransfer", Tip}, row, db) do - claim = db |> Map.get("Claim", []) |> Enum.find(&(&1["id"] == row["claim_id"])) - - github_user = db |> Map.get("GithubUser", []) |> Enum.find(&(&1["id"] == claim["github_user_id"])) + claim = find_by_index(db, "Claim", "id", row["claim_id"]) + github_user = find_by_index(db, "GithubUser", "id", claim["github_user_id"]) + bounty = find_by_index(db, "Bounty", "id", claim["bounty_id"]) user_id = or_else(github_user["user_id"], github_user["id"]) - - bounty = db |> Map.get("Bounty", []) |> Enum.find(&(&1["id"] == claim["bounty_id"])) - amount = Money.from_integer(String.to_integer(row["amount"]), row["currency"]) if !bounty do @@ -511,18 +515,14 @@ defmodule DatabaseMigration do end defp transform({"BountyTransfer", Transaction}, row, db) do - claim = db |> Map.get("Claim", []) |> Enum.find(&(&1["id"] == row["claim_id"])) - - bounty = db |> Map.get("Bounty", []) |> Enum.find(&(&1["id"] == claim["bounty_id"])) - - github_user = db |> Map.get("GithubUser", []) |> Enum.find(&(&1["id"] == claim["github_user_id"])) + claim = find_by_index(db, "Claim", "id", row["claim_id"]) + bounty = find_by_index(db, "Bounty", "id", claim["bounty_id"]) + github_user = find_by_index(db, "GithubUser", "id", claim["github_user_id"]) + org = find_by_index(db, "Org", "id", bounty["org_id"]) + bounty_charge = find_by_index(db, "BountyCharge", "id", row["bounty_charge_id"]) user_id = or_else(github_user["user_id"], github_user["id"]) - org = db |> Map.get("Org", []) |> Enum.find(&(&1["id"] == bounty["org_id"])) - - bounty_charge = db |> Map.get("BountyCharge", []) |> Enum.find(&(&1["id"] == row["bounty_charge_id"])) - if !bounty do raise "Bounty not found: #{inspect(row)}" end @@ -626,7 +626,7 @@ defmodule DatabaseMigration do defp transform({"StripePaymentMethod", PaymentMethod}, row, db) do if row["org_id"] not in ["clfqtao4h0001mo0gkp9az0bn", "cm251pvg40007ld031q5t2hj2", "cljo6j981000el60f1k1cvtns"] do - customer = db |> Map.get("StripeCustomer", []) |> Enum.find(&(&1["org_id"] == row["org_id"])) + customer = find_by_index(db, "StripeCustomer", "org_id", row["org_id"]) if !customer do raise "StripeCustomer not found: #{inspect(row)}" @@ -770,22 +770,44 @@ defmodule DatabaseMigration do end defp collect_data(input_file) do - input_file - |> File.stream!() - |> Stream.chunk_while( - nil, - &collect_chunk_fun/2, - &collect_after_fun/1 - ) - |> Enum.reduce(%{}, fn - {table, data}, acc -> - if table in relevant_tables() do - parsed_data = parse_copy_data(data) - Map.put(acc, table, parsed_data) - else - acc - end - end) + db = + input_file + |> File.stream!() + |> Stream.chunk_while( + nil, + &collect_chunk_fun/2, + &collect_after_fun/1 + ) + |> Enum.reduce(%{}, fn + {table, data}, acc -> + if table in relevant_tables() do + parsed_data = parse_copy_data(data) + Map.put(acc, table, parsed_data) + else + acc + end + end) + + indexes = + Enum.reduce(@index_fields, %{}, fn {table, columns}, acc -> + table_indexes = Map.new(columns, fn column -> {column, index_by_field(db[table], column)} end) + Map.put(acc, table, table_indexes) + end) + + Map.put(db, :indexes, indexes) + end + + defp index_by_field(data, field) do + data + |> Enum.group_by(&Map.get(&1, field)) + |> Map.new(fn {k, v} -> {k, List.first(v)} end) + end + + defp find_by_index(db, table, field, value) do + case get_in(db, [:indexes, table, field]) do + nil -> raise "Index not found for table #{table}.#{field}" + index -> index[value] + end end defp parse_copy_data([header | data]) do From 2f86af7bdb4bbae64e1945485a7db9c943755b26 Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 20:01:46 +0300 Subject: [PATCH 36/51] fix more issues --- lib/algora/bounties/schemas/bounty.ex | 2 +- .../migrations/20250208173814_make_provider_meta_nullable.exs | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/algora/bounties/schemas/bounty.ex b/lib/algora/bounties/schemas/bounty.ex index b6ad925d8..5de8e5bdf 100644 --- a/lib/algora/bounties/schemas/bounty.ex +++ b/lib/algora/bounties/schemas/bounty.ex @@ -34,7 +34,7 @@ defmodule Algora.Bounties.Bounty do |> foreign_key_constraint(:ticket) |> foreign_key_constraint(:owner) |> foreign_key_constraint(:creator) - |> unique_constraint([:ticket_id, :owner_id]) + |> unique_constraint([:ticket_id, :owner_id, :number]) |> Algora.Validations.validate_money_positive(:amount) end diff --git a/priv/repo/migrations/20250208173814_make_provider_meta_nullable.exs b/priv/repo/migrations/20250208173814_make_provider_meta_nullable.exs index a7a073931..3e42c4f54 100644 --- a/priv/repo/migrations/20250208173814_make_provider_meta_nullable.exs +++ b/priv/repo/migrations/20250208173814_make_provider_meta_nullable.exs @@ -43,5 +43,8 @@ defmodule Algora.Repo.Migrations.MakeProviderMetaNullable do drop unique_index(:bounties, [:ticket_id, :owner_id]) create unique_index(:bounties, [:ticket_id, :owner_id, :number]) + + drop unique_index(:command_responses, [:ticket_id, :command_type]) + create unique_index(:command_responses, [:provider, :provider_command_id]) end end From b4388a4912a33789337f6c215f934490118ba719 Mon Sep 17 00:00:00 2001 From: zafer Date: Sat, 8 Feb 2025 20:22:18 +0300 Subject: [PATCH 37/51] derive backfilled_tables from schema_mappings --- scripts/database_migration.exs | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index 54812011f..9d876a1e0 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -59,23 +59,6 @@ defmodule DatabaseMigration do {"StripePaymentMethod", PaymentMethod} ] - @backfilled_tables [ - "accounts", - "payment_methods", - "customers", - "installations", - "repositories", - "transactions", - "claims", - "attempts", - "bounties", - "tips", - "tickets", - "members", - "identities", - "users" - ] - @index_fields [ {"GithubUser", ["id", "user_id"]}, {"User", ["id"]}, @@ -90,7 +73,20 @@ defmodule DatabaseMigration do {"Reward", ["bounty_id"]} ] - defp relevant_tables, do: @schema_mappings |> Enum.map(fn {k, _v} -> k end) |> Enum.dedup() + defp relevant_tables do + @schema_mappings + |> Enum.map(fn {k, _v} -> k end) + |> Enum.dedup() + end + + defp backfilled_tables do + @schema_mappings + |> Enum.map(fn {_, v} -> v end) + |> Enum.reject(&is_nil/1) + |> Enum.dedup() + |> Enum.reverse() + |> Enum.map(& &1.__schema__(:source)) + end defp transform({"Task", Ticket}, row, db) do if row["forge"] != "github" do @@ -1098,7 +1094,7 @@ defmodule DatabaseMigration do [ "BEGIN TRANSACTION;", "SET CONSTRAINTS ALL DEFERRED;", - Enum.map(@backfilled_tables, &"TRUNCATE TABLE #{&1} CASCADE;"), + Enum.map(backfilled_tables(), &"TRUNCATE TABLE #{&1} CASCADE;"), "SET CONSTRAINTS ALL IMMEDIATE;", "COMMIT;" ] From ee071525c6bbfb68347ab59ec878e13dec4ad4a4 Mon Sep 17 00:00:00 2001 From: zafer Date: Sun, 9 Feb 2025 17:37:43 +0300 Subject: [PATCH 38/51] in midst of merging some user records --- scripts/database_migration.exs | 232 ++++++++++++++++++++++----------- 1 file changed, 157 insertions(+), 75 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index 9d876a1e0..5dd536625 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -60,9 +60,8 @@ defmodule DatabaseMigration do ] @index_fields [ - {"GithubUser", ["id", "user_id"]}, {"User", ["id"]}, - {"Org", ["id"]}, + {"GithubUser", ["id", "user_id"]}, {"Bounty", ["id", "task_id"]}, {"Task", ["id"]}, {"Claim", ["id", "bounty_id"]}, @@ -203,53 +202,57 @@ defmodule DatabaseMigration do } end - defp transform({"Org", User}, row, _db) do - %{ - "id" => row["id"], - "provider" => row["github_handle"] && "github", - "provider_id" => row["github_id"], - "provider_login" => row["github_handle"], - "provider_meta" => row["github_data"] && deserialize_value(row["github_data"]), - "email" => nil, - "display_name" => row["name"], - "handle" => row["handle"], - "avatar_url" => update_url(row["avatar_url"]), - "external_homepage_url" => nil, - "type" => "organization", - "bio" => row["description"], - "location" => nil, - "country" => nil, - "timezone" => nil, - "stargazers_count" => row["stargazers_count"], - "domain" => row["domain"], - "tech_stack" => row["tech"], - "featured" => row["featured"], - "priority" => row["priority"], - "fee_pct" => row["fee_pct"], - "seeded" => row["seeded"], - "activated" => row["active"], - "max_open_attempts" => row["max_open_attempts"], - "manual_assignment" => row["manual_assignment"], - "bounty_mode" => nil, - "hourly_rate_min" => nil, - "hourly_rate_max" => nil, - "hours_per_week" => nil, - "website_url" => row["website_url"], - "twitter_url" => row["twitter_url"], - "github_url" => nil, - "youtube_url" => row["youtube_url"], - "twitch_url" => nil, - "discord_url" => row["discord_url"], - "slack_url" => row["slack_url"], - "linkedin_url" => nil, - "og_title" => nil, - "og_image_url" => nil, - "last_context" => nil, - "need_avatar" => nil, - "inserted_at" => row["created_at"], - "updated_at" => row["updated_at"], - "is_admin" => false - } + defp transform({"Org", User}, row, db) do + merged_user = find_by_index(db, "_MergedUser", "id", row["id"]) + + if not user?(merged_user) do + %{ + "id" => row["id"], + "provider" => row["github_handle"] && "github", + "provider_id" => row["github_id"], + "provider_login" => row["github_handle"], + "provider_meta" => row["github_data"] && deserialize_value(row["github_data"]), + "email" => nil, + "display_name" => row["name"], + "handle" => row["handle"], + "avatar_url" => update_url(row["avatar_url"]), + "external_homepage_url" => nil, + "type" => "organization", + "bio" => row["description"], + "location" => nil, + "country" => nil, + "timezone" => nil, + "stargazers_count" => row["stargazers_count"], + "domain" => row["domain"], + "tech_stack" => row["tech"], + "featured" => row["featured"], + "priority" => row["priority"], + "fee_pct" => row["fee_pct"], + "seeded" => row["seeded"], + "activated" => row["active"], + "max_open_attempts" => row["max_open_attempts"], + "manual_assignment" => row["manual_assignment"], + "bounty_mode" => nil, + "hourly_rate_min" => nil, + "hourly_rate_max" => nil, + "hours_per_week" => nil, + "website_url" => row["website_url"], + "twitter_url" => row["twitter_url"], + "github_url" => nil, + "youtube_url" => row["youtube_url"], + "twitch_url" => nil, + "discord_url" => row["discord_url"], + "slack_url" => row["slack_url"], + "linkedin_url" => nil, + "og_title" => nil, + "og_image_url" => nil, + "last_context" => nil, + "need_avatar" => nil, + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"], + "is_admin" => false + } + end end defp transform({"GithubUser", User}, row, _db) do @@ -324,10 +327,16 @@ defmodule DatabaseMigration do } end - defp transform({"OrgMember", Member}, row, _db) do + defp transform({"OrgMember", Member}, row, db) do + owner = find_by_index(db, "_MergedUser", "id", row["org_id"]) + + if !owner do + raise "Owner not found: #{inspect(row)}" + end + %{ "id" => row["id"], - "org_id" => row["org_id"], + "org_id" => owner["id"], "role" => row["role"], "user_id" => row["user_id"], "inserted_at" => row["created_at"], @@ -337,15 +346,20 @@ defmodule DatabaseMigration do defp transform({"Bounty", Bounty}, row, db) do reward = find_by_index(db, "Reward", "bounty_id", row["id"]) + owner = find_by_index(db, "_MergedUser", "id", row["org_id"]) amount = if reward, do: Money.from_integer(String.to_integer(reward["amount"]), reward["currency"]) + if !owner do + raise "Owner not found: #{inspect(row)}" + end + if row["type"] != "tip" do %{ "id" => row["id"], "amount" => amount, "ticket_id" => row["task_id"], - "owner_id" => row["org_id"], + "owner_id" => owner["id"], "creator_id" => row["poster_id"], "inserted_at" => row["created_at"], "updated_at" => row["updated_at"], @@ -434,7 +448,7 @@ defmodule DatabaseMigration do end defp transform({"BountyCharge", Transaction}, row, db) do - user = find_by_index(db, "Org", "id", row["org_id"]) + user = find_by_index(db, "_MergedUser", "id", row["org_id"]) amount = Money.from_integer(String.to_integer(row["amount"]), row["currency"]) @@ -483,7 +497,7 @@ defmodule DatabaseMigration do claim = find_by_index(db, "Claim", "id", row["claim_id"]) github_user = find_by_index(db, "GithubUser", "id", claim["github_user_id"]) bounty = find_by_index(db, "Bounty", "id", claim["bounty_id"]) - + owner = find_by_index(db, "_MergedUser", "id", bounty["org_id"]) user_id = or_else(github_user["user_id"], github_user["id"]) amount = Money.from_integer(String.to_integer(row["amount"]), row["currency"]) @@ -495,13 +509,17 @@ defmodule DatabaseMigration do raise "User not found: #{inspect(row)}" end + if !owner do + raise "Owner not found: #{inspect(row)}" + end + if bounty["type"] == "tip" do %{ "id" => bounty["id"] <> user_id, "amount" => amount, "status" => nil, "ticket_id" => bounty["task_id"], - "owner_id" => bounty["org_id"], + "owner_id" => owner["id"], "creator_id" => bounty["poster_id"], "recipient_id" => user_id, "inserted_at" => bounty["created_at"], @@ -514,7 +532,7 @@ defmodule DatabaseMigration do claim = find_by_index(db, "Claim", "id", row["claim_id"]) bounty = find_by_index(db, "Bounty", "id", claim["bounty_id"]) github_user = find_by_index(db, "GithubUser", "id", claim["github_user_id"]) - org = find_by_index(db, "Org", "id", bounty["org_id"]) + org = find_by_index(db, "_MergedUser", "id", bounty["org_id"]) bounty_charge = find_by_index(db, "BountyCharge", "id", row["bounty_charge_id"]) user_id = or_else(github_user["user_id"], github_user["id"]) @@ -566,7 +584,13 @@ defmodule DatabaseMigration do ) end - defp transform({"GithubInstallation", Installation}, row, _db) do + defp transform({"GithubInstallation", Installation}, row, db) do + connected_user = find_by_index(db, "_MergedUser", "id", row["org_id"]) + + if !connected_user do + raise "Connected user not found: #{inspect(row)}" + end + %{ "id" => row["id"], "provider" => "github", @@ -575,7 +599,7 @@ defmodule DatabaseMigration do "avatar_url" => nil, "repository_selection" => nil, "owner_id" => nil, - "connected_user_id" => row["org_id"], + "connected_user_id" => connected_user["id"], "inserted_at" => row["created_at"], "updated_at" => row["updated_at"], "provider_user_id" => nil @@ -605,15 +629,21 @@ defmodule DatabaseMigration do } end - defp transform({"StripeCustomer", Customer}, row, _db) do - if row["org_id"] not in ["clfqtao4h0001mo0gkp9az0bn", "cm251pvg40007ld031q5t2hj2", "cljo6j981000el60f1k1cvtns"] do + defp transform({"StripeCustomer", Customer}, row, db) do + owner = find_by_index(db, "_MergedUser", "id", row["org_id"]) + + if !owner do + raise "Owner not found: #{inspect(row)}" + end + + if owner["id"] not in ["clfqtao4h0001mo0gkp9az0bn", "cm251pvg40007ld031q5t2hj2", "cljo6j981000el60f1k1cvtns"] do %{ "id" => row["id"], "provider" => "stripe", "provider_id" => row["stripe_id"], "provider_meta" => nil, "name" => row["name"], - "user_id" => row["org_id"], + "user_id" => owner["id"], "inserted_at" => row["created_at"], "updated_at" => row["updated_at"] } @@ -621,13 +651,19 @@ defmodule DatabaseMigration do end defp transform({"StripePaymentMethod", PaymentMethod}, row, db) do - if row["org_id"] not in ["clfqtao4h0001mo0gkp9az0bn", "cm251pvg40007ld031q5t2hj2", "cljo6j981000el60f1k1cvtns"] do - customer = find_by_index(db, "StripeCustomer", "org_id", row["org_id"]) + owner = find_by_index(db, "_MergedUser", "id", row["org_id"]) - if !customer do - raise "StripeCustomer not found: #{inspect(row)}" - end + if !owner do + raise "Owner not found: #{inspect(row)}" + end + + customer = find_by_index(db, "StripeCustomer", "org_id", row["org_id"]) + + if !customer do + raise "StripeCustomer not found: #{inspect(row)}" + end + if owner["id"] not in ["clfqtao4h0001mo0gkp9az0bn", "cm251pvg40007ld031q5t2hj2", "cljo6j981000el60f1k1cvtns"] do %{ "id" => row["id"], "provider" => "stripe", @@ -765,6 +801,8 @@ defmodule DatabaseMigration do |> Stream.run() end + defp user?(row), do: not nullish?(row["email"]) + defp collect_data(input_file) do db = input_file @@ -790,7 +828,49 @@ defmodule DatabaseMigration do Map.put(acc, table, table_indexes) end) - Map.put(db, :indexes, indexes) + db = Map.put(db, :indexes, indexes) + + put_in(db, [:indexes, "_MergedUser"], %{"id" => index_merged_users(db)}) + end + + defp index_merged_users(db) do + (db["User"] ++ db["Org"]) + |> Enum.group_by(fn row -> + if user?(row) do + github_user = find_by_index(db, "GithubUser", "user_id", row["id"]) + + if is_nil(github_user) or nullish?(github_user["login"]) do + "algora_" <> row["id"] + else + "github_" <> github_user["login"] + end + else + if nullish?(row["github_handle"]) do + "algora_" <> row["id"] + else + "github_" <> row["github_handle"] + end + end + end) + |> Enum.flat_map(fn {_k, entities} -> + case Enum.find(entities, &user?/1) do + nil -> + case entities do + [user] -> [{user["id"], user}] + _ -> raise "Unexpected number of users for #{inspect(entities)}" + end + + user -> + Enum.map(entities, fn row -> + if row["id"] != user["id"] do + Logger.info("#{row["handle"]} -> #{user["handle"]}") + end + + {row["id"], user} + end) + end + end) + |> Map.new(fn {k, v} -> {k, v} end) end defp index_by_field(data, field) do @@ -907,13 +987,11 @@ defmodule DatabaseMigration do end defp ensure_unique_handle(fields) do - case fields[:handle] do - nil -> - fields - - handle -> - new_handle = get_unique_handle(handle) - Map.put(fields, :handle, new_handle) + if nullish?(fields[:handle]) do + fields + else + new_handle = get_unique_handle(fields[:handle]) + Map.put(fields, :handle, new_handle) end end @@ -925,6 +1003,10 @@ defmodule DatabaseMigration do new_handle = if count > 0, do: "#{handle}#{count + 1}", else: handle Process.put(:handles, Map.put(handles, downcased_handle, count + 1)) + if count > 0 do + Logger.warning("Unique handle collision: #{handle} -> #{new_handle}") + end + new_handle end @@ -1144,7 +1226,7 @@ defmodule DatabaseMigration do :ok = time_step("Processing dump", fn -> process_dump(input_file, output_file) end) :ok = time_step("Clearing tables", fn -> clear_tables!() end) {:ok, _} = time_step("Importing new data", fn -> psql(["-f", output_file]) end) - :ok = time_step("Backfilling repositories", fn -> Algora.Admin.backfill_repos!() end) + # :ok = time_step("Backfilling repositories", fn -> Algora.Admin.backfill_repos!() end) end) IO.puts("\n✓ Migration completed successfully in #{total_time / 1_000_000} seconds") From d5c40c62e5e2da5ee5b160a0169ff8c733ed1a56 Mon Sep 17 00:00:00 2001 From: zafer Date: Sun, 9 Feb 2025 18:26:58 +0300 Subject: [PATCH 39/51] merge related user records --- scripts/database_migration.exs | 102 +++++++++++++++++++++++---------- 1 file changed, 71 insertions(+), 31 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index 5dd536625..23c6d9f5f 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -834,43 +834,83 @@ defmodule DatabaseMigration do end defp index_merged_users(db) do - (db["User"] ++ db["Org"]) - |> Enum.group_by(fn row -> - if user?(row) do - github_user = find_by_index(db, "GithubUser", "user_id", row["id"]) - - if is_nil(github_user) or nullish?(github_user["login"]) do - "algora_" <> row["id"] + entities = + (db["User"] ++ db["Org"]) + |> Enum.group_by(fn row -> + if user?(row) do + github_user = find_by_index(db, "GithubUser", "user_id", row["id"]) + + if is_nil(github_user) or nullish?(github_user["login"]) do + "algora_" <> row["id"] + else + "github_" <> github_user["login"] + end else - "github_" <> github_user["login"] + if nullish?(row["github_handle"]) do + "algora_" <> row["id"] + else + "github_" <> row["github_handle"] + end end - else - if nullish?(row["github_handle"]) do - "algora_" <> row["id"] - else - "github_" <> row["github_handle"] + end) + |> Enum.flat_map(fn {_k, entities} -> + case entities do + [user] -> + [{:unmerged, user["id"], user}] + + entities -> + case Enum.find(entities, &user?/1) do + nil -> + raise "Unexpected number of users for #{inspect(entities)}" + + user -> + Enum.map(entities, fn row -> + # if row["id"] != user["id"], do: Logger.info("[same github user] #{row["handle"]} -> #{user["handle"]}") + {:merged, row["id"], user} + end) + end end - end - end) - |> Enum.flat_map(fn {_k, entities} -> - case Enum.find(entities, &user?/1) do - nil -> - case entities do - [user] -> [{user["id"], user}] - _ -> raise "Unexpected number of users for #{inspect(entities)}" - end + end) + |> Enum.group_by(fn {type, _id, _user} -> type end) + + merged1 = + entities + |> Map.get(:merged, []) + |> Map.new(fn {_type, id, user} -> {id, user} end) + + merged2 = + entities + |> Map.get(:unmerged, []) + |> Enum.map(fn {_type, _id, row} -> row end) + |> Enum.group_by(fn row -> row["handle"] end) + |> Enum.flat_map(fn {handle, entities} -> + case entities do + [entity] -> + [{entity["id"], entity}] + + [_entity1, _entity2] -> + user = Enum.find(entities, &user?/1) + org = Enum.find(entities, &(not user?(&1))) + + if is_nil(user) or is_nil(org) do + raise "User or org not found for handle #{handle}: #{inspect(entities)}" + end - user -> - Enum.map(entities, fn row -> - if row["id"] != user["id"] do - Logger.info("#{row["handle"]} -> #{user["handle"]}") + if org["creator_id"] == user["id"] do + # Logger.info("[same handle] #{org["handle"]} -> #{user["handle"]}") + Enum.map(entities, fn row -> {row["id"], user} end) + else + Logger.warning("Org #{org["handle"]} was not created by user #{user["handle"]}") + Enum.map(entities, fn row -> {row["id"], row} end) end - {row["id"], user} - end) - end - end) - |> Map.new(fn {k, v} -> {k, v} end) + _ -> + raise "Unexpected number of entities for handle #{handle}: #{inspect(entities)}" + end + end) + |> Map.new() + + Map.merge(merged1, merged2) end defp index_by_field(data, field) do From 914b91ee154769c4312d76c5280fafb9859966f6 Mon Sep 17 00:00:00 2001 From: zafer Date: Sun, 9 Feb 2025 18:27:27 +0300 Subject: [PATCH 40/51] remove unused indexes --- scripts/database_migration.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index 23c6d9f5f..86229f5c1 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -62,9 +62,9 @@ defmodule DatabaseMigration do @index_fields [ {"User", ["id"]}, {"GithubUser", ["id", "user_id"]}, - {"Bounty", ["id", "task_id"]}, + {"Bounty", ["id"]}, {"Task", ["id"]}, - {"Claim", ["id", "bounty_id"]}, + {"Claim", ["id"]}, {"BountyCharge", ["id"]}, {"StripeCustomer", ["org_id"]}, {"GithubIssue", ["id"]}, From 324b1340a6c7653ab4fee9e231d703144cea2bb3 Mon Sep 17 00:00:00 2001 From: zafer Date: Sun, 9 Feb 2025 18:28:46 +0300 Subject: [PATCH 41/51] enforce unique result for each index --- scripts/database_migration.exs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index 86229f5c1..17e8598e2 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -824,7 +824,7 @@ defmodule DatabaseMigration do indexes = Enum.reduce(@index_fields, %{}, fn {table, columns}, acc -> - table_indexes = Map.new(columns, fn column -> {column, index_by_field(db[table], column)} end) + table_indexes = Map.new(columns, fn column -> {column, index_by_field(db, table, column)} end) Map.put(acc, table, table_indexes) end) @@ -913,10 +913,17 @@ defmodule DatabaseMigration do Map.merge(merged1, merged2) end - defp index_by_field(data, field) do - data + defp index_by_field(db, table, field) do + db[table] + |> Enum.reject(fn row -> table == "StripeCustomer" and row["region"] == "EU" end) |> Enum.group_by(&Map.get(&1, field)) - |> Map.new(fn {k, v} -> {k, List.first(v)} end) + |> Map.new(fn {k, v} -> + {k, + case v do + [v] -> v + v -> raise "Unexpected number of entities for #{table}.#{field}: #{inspect(v)}" + end} + end) end defp find_by_index(db, table, field, value) do From d1680a31d3034898e91be39bc5631ee6280ab9ec Mon Sep 17 00:00:00 2001 From: zafer Date: Sun, 9 Feb 2025 18:28:59 +0300 Subject: [PATCH 42/51] remove nullish keys in indexes --- scripts/database_migration.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index 17e8598e2..118f86399 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -917,6 +917,7 @@ defmodule DatabaseMigration do db[table] |> Enum.reject(fn row -> table == "StripeCustomer" and row["region"] == "EU" end) |> Enum.group_by(&Map.get(&1, field)) + |> Enum.reject(fn {k, _v} -> nullish?(k) end) |> Map.new(fn {k, v} -> {k, case v do From ed1f36270169d4a8e03e9d97e2efd6b2a1a64b31 Mon Sep 17 00:00:00 2001 From: zafer Date: Mon, 10 Feb 2025 18:54:11 +0300 Subject: [PATCH 43/51] more fixes --- scripts/database_migration.exs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index 118f86399..55e44c464 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -334,14 +334,16 @@ defmodule DatabaseMigration do raise "Owner not found: #{inspect(row)}" end - %{ - "id" => row["id"], - "org_id" => owner["id"], - "role" => row["role"], - "user_id" => row["user_id"], - "inserted_at" => row["created_at"], - "updated_at" => row["updated_at"] - } + if owner["id"] != row["user_id"] do + %{ + "id" => row["id"], + "org_id" => owner["id"], + "role" => row["role"], + "user_id" => row["user_id"], + "inserted_at" => row["created_at"], + "updated_at" => row["updated_at"] + } + end end defp transform({"Bounty", Bounty}, row, db) do @@ -1263,8 +1265,8 @@ defmodule DatabaseMigration do end def run! do - input_file = ".local/prod_db.sql" - output_file = ".local/prod_db_new.sql" + input_file = ".local/db/v1-data-2025-02-10.sql" + output_file = ".local/db/v2-data-2025-02-10.sql" if File.exists?(input_file) or File.exists?(output_file) do IO.puts("\nStarting migration...") From 6828afb60ddbc87feb4fb80085d993e771275acc Mon Sep 17 00:00:00 2001 From: zafer Date: Mon, 10 Feb 2025 18:54:24 +0300 Subject: [PATCH 44/51] add sanity checks --- lib/algora/admin/migration.ex | 124 ++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 lib/algora/admin/migration.ex diff --git a/lib/algora/admin/migration.ex b/lib/algora/admin/migration.ex new file mode 100644 index 000000000..0887612a4 --- /dev/null +++ b/lib/algora/admin/migration.ex @@ -0,0 +1,124 @@ +defmodule Algora.Admin.Migration do + @moduledoc false + import Ecto.Query + + alias Algora.Accounts.User + alias Algora.Payments.Transaction + alias Algora.Repo + + @balances_file ".local/db/balance-2025-02-10.json" + + def get_actual_balances(type) do + with {:ok, content} <- File.read(@balances_file), + {:ok, data} <- Jason.decode(content) do + Enum.map(data[Atom.to_string(type)], fn %{"provider_login" => login, "balance" => balance} -> + %{ + provider_login: login, + balance: Money.new!(:USD, Decimal.new(balance)) + } + end) + else + error -> raise "Failed to load balances: #{inspect(error)}" + end + end + + def get_balances(type) do + Enum.filter(get_balances(), &(&1.type == type)) + end + + def get_balances do + user_txs = + from t in Transaction, + group_by: [t.user_id, t.type], + select: %{ + user_id: t.user_id, + type: t.type, + net_amount: sum(t.net_amount) + } + + user_balances = + from ut in subquery(user_txs), + group_by: ut.user_id, + select: %{ + user_id: ut.user_id, + balance: + sum( + fragment( + """ + CASE + WHEN ? = 'credit' THEN ? + WHEN ? = 'debit' THEN money_negate(?) + WHEN ? = 'charge' THEN ? + WHEN ? = 'transfer' THEN money_negate(?) + ELSE ('USD', 0)::money_with_currency + END + """, + ut.type, + ut.net_amount, + ut.type, + ut.net_amount, + ut.type, + ut.net_amount, + ut.type, + ut.net_amount + ) + ) + } + + query = + from ub in subquery(user_balances), + join: u in User, + on: u.id == ub.user_id, + where: ub.balance != fragment("('USD', 0)::money_with_currency"), + order_by: [desc: u.type, desc: ub.balance, asc: u.provider_login], + select: %{ + type: u.type, + provider_login: u.provider_login, + balance: ub.balance + } + + query + |> Repo.all() + |> Enum.map(fn + user -> + {currency, amount} = user.balance + %{user | balance: Money.new!(currency, amount)} + end) + end + + def diff_balances(type) do + actual = Enum.map(get_actual_balances(type), &Map.take(&1, [:provider_login, :balance])) + current = Enum.map(get_balances(type), &Map.take(&1, [:provider_login, :balance])) + + actual_map = Map.new(actual, &{&1.provider_login, &1.balance}) + current_map = Map.new(current, &{&1.provider_login, &1.balance}) + + all_logins = + MapSet.union( + MapSet.new(Map.keys(actual_map)), + MapSet.new(Map.keys(current_map)) + ) + + differences = + Enum.reduce(all_logins, [], fn login, acc -> + actual_balance = Map.get(actual_map, login) + current_balance = Map.get(current_map, login) + + cond do + actual_balance == current_balance -> + acc + + actual_balance == nil -> + [{:extra_in_current, login, current_balance} | acc] + + current_balance == nil -> + [{:missing_in_current, login, actual_balance} | acc] + + true -> + [{:different, login, actual_balance, current_balance} | acc] + end + end) + + Enum.sort_by(differences, &elem(&1, 1)) + end +end From 13e9f8958a61bc660d4db0c926a7027f4b8d4c4d Mon Sep 17 00:00:00 2001 From: zafer Date: Mon, 10 Feb 2025 20:48:10 +0300 Subject: [PATCH 45/51] migrate bounty statuses --- scripts/database_migration.exs | 22 +++++++++++++++++++++- scripts/v2-progress.yaml | 2 +- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index 55e44c464..cfada10c6 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -356,10 +356,18 @@ defmodule DatabaseMigration do raise "Owner not found: #{inspect(row)}" end + transfer = find_by_index(db, "_BountyTransfer", "bounty_id", row["id"]) + if row["type"] != "tip" do %{ "id" => row["id"], "amount" => amount, + "status" => + cond do + not is_nil(transfer) -> :paid + row["deleted_at"] || row["status"] == "inactive" -> :cancelled + true -> :open + end, "ticket_id" => row["task_id"], "owner_id" => owner["id"], "creator_id" => row["poster_id"], @@ -832,7 +840,19 @@ defmodule DatabaseMigration do db = Map.put(db, :indexes, indexes) - put_in(db, [:indexes, "_MergedUser"], %{"id" => index_merged_users(db)}) + db + |> put_in([:indexes, "_MergedUser"], %{"id" => index_merged_users(db)}) + |> put_in([:indexes, "_BountyTransfer"], %{ + "bounty_id" => + Enum.group_by(db["BountyTransfer"], fn row -> + claim = find_by_index(db, "Claim", "id", row["claim_id"]) + charge = find_by_index(db, "BountyCharge", "id", row["bounty_charge_id"]) + + if charge["succeeded_at"] do + claim["bounty_id"] + end + end) + }) end defp index_merged_users(db) do diff --git a/scripts/v2-progress.yaml b/scripts/v2-progress.yaml index b41fbf4ce..e1956480c 100644 --- a/scripts/v2-progress.yaml +++ b/scripts/v2-progress.yaml @@ -28,7 +28,7 @@ - bounties: - id: 1 - amount: 1 - - status: 0 + - status: 1 - ticket_id: 1 - owner_id: 1 - creator_id: 1 From 5b67cc3b12b0544e086be3cbd026653dda3091f3 Mon Sep 17 00:00:00 2001 From: zafer Date: Thu, 13 Feb 2025 19:05:48 +0300 Subject: [PATCH 46/51] def money_negate op --- .../20250213155854_define_negate_operator.exs | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 priv/repo/migrations/20250213155854_define_negate_operator.exs diff --git a/priv/repo/migrations/20250213155854_define_negate_operator.exs b/priv/repo/migrations/20250213155854_define_negate_operator.exs new file mode 100644 index 000000000..5395eb2e1 --- /dev/null +++ b/priv/repo/migrations/20250213155854_define_negate_operator.exs @@ -0,0 +1,38 @@ +defmodule Algora.Repo.Migrations.DefineNegateOperator do + use Ecto.Migration + + def up do + execute(""" + CREATE OR REPLACE FUNCTION money_negate(money_1 money_with_currency) + RETURNS money_with_currency + IMMUTABLE + STRICT + LANGUAGE plpgsql + AS $$ + DECLARE + currency varchar; + addition numeric; + BEGIN + currency := currency_code(money_1); + addition := amount(money_1) * -1; + return row(currency, addition); + END; + $$; + """) + |> Money.Migration.adjust_for_type(repo()) + + execute(""" + CREATE OPERATOR - ( + rightarg = money_with_currency, + procedure = money_negate + ); + """) + |> Money.Migration.adjust_for_type(repo()) + end + + def down do + execute("DROP OPERATOR IF EXISTS - (none, money_with_currency);") + + execute("DROP FUNCTION IF EXISTS money_negate(money_with_currency);") + end +end From 7e8cae672f0b3ba219ccc963f7e2e7fd950b626e Mon Sep 17 00:00:00 2001 From: zafer Date: Thu, 13 Feb 2025 20:31:42 +0300 Subject: [PATCH 47/51] migrate org balance transactions --- scripts/database_migration.exs | 54 ++++++++++++++++++++++++++++++++-- scripts/v1-progress.yaml | 10 +++---- 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index cfada10c6..0179e5720 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -53,6 +53,7 @@ defmodule DatabaseMigration do {"BountyCharge", Transaction}, {"BountyTransfer", Tip}, {"BountyTransfer", Transaction}, + {"OrgBalanceTransaction", Transaction}, {"GithubInstallation", Installation}, {"StripeAccount", Account}, {"StripeCustomer", Customer}, @@ -594,6 +595,55 @@ defmodule DatabaseMigration do ) end + defp transform({"OrgBalanceTransaction", Transaction}, row, db) do + user = find_by_index(db, "_MergedUser", "id", row["org_id"]) + + if !user do + raise "User not found: #{inspect(row)}" + end + + amount = Money.from_integer(String.to_integer(row["amount"]), row["currency"]) + + {abs_amount, type} = + if Money.positive?(amount) do + {amount, "credit"} + else + {Money.negate!(amount), "debit"} + end + + %{ + "id" => row["id"], + "provider" => "stripe", + "provider_id" => nil, + "provider_charge_id" => nil, + "provider_payment_intent_id" => nil, + "provider_transfer_id" => nil, + "provider_invoice_id" => nil, + "provider_balance_transaction_id" => nil, + "provider_meta" => nil, + "gross_amount" => abs_amount, + "net_amount" => abs_amount, + "total_fee" => Money.zero(:USD), + "provider_fee" => nil, + "line_items" => nil, + "type" => type, + "status" => :succeeded, + "succeeded_at" => row["created_at"], + "reversed_at" => nil, + "group_id" => row["id"], + "user_id" => user["id"], + "contract_id" => nil, + "original_contract_id" => nil, + "timesheet_id" => nil, + "bounty_id" => nil, + "tip_id" => nil, + "linked_transaction_id" => nil, + "inserted_at" => row["created_at"], + "updated_at" => row["created_at"], + "claim_id" => nil + } + end + defp transform({"GithubInstallation", Installation}, row, db) do connected_user = find_by_index(db, "_MergedUser", "id", row["org_id"]) @@ -1285,8 +1335,8 @@ defmodule DatabaseMigration do end def run! do - input_file = ".local/db/v1-data-2025-02-10.sql" - output_file = ".local/db/v2-data-2025-02-10.sql" + input_file = ".local/db/v1-data-2025-02-13.sql" + output_file = ".local/db/v2-data-2025-02-13.sql" if File.exists?(input_file) or File.exists?(output_file) do IO.puts("\nStarting migration...") diff --git a/scripts/v1-progress.yaml b/scripts/v1-progress.yaml index 364ace109..b25b43b3a 100644 --- a/scripts/v1-progress.yaml +++ b/scripts/v1-progress.yaml @@ -360,11 +360,11 @@ - github_handle: 1 - active: 1 - "OrgBalanceTransaction": - - id: -1 - - created_at: -1 - - org_id: -1 - - amount: -1 - - currency: -1 + - id: 1 + - created_at: 1 + - org_id: 1 + - amount: 1 + - currency: 1 - type: -1 - effect: -1 - tier: -1 From d361ded11f378f527a6699534f8c6f6a06c934c2 Mon Sep 17 00:00:00 2001 From: zafer Date: Fri, 14 Feb 2025 19:03:29 +0300 Subject: [PATCH 48/51] update claim status --- scripts/database_migration.exs | 7 +++++-- scripts/v1-progress.yaml | 2 +- scripts/v2-progress.yaml | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index 0179e5720..d4786673e 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -439,8 +439,11 @@ defmodule DatabaseMigration do %{ "id" => row["id"], - # TODO: - "status" => :pending, + "status" => + case row["status"] do + "accepted" -> :approved + _ -> :pending + end, "type" => cond do !nullish?(row["github_pull_request_id"]) -> "pull_request" diff --git a/scripts/v1-progress.yaml b/scripts/v1-progress.yaml index b25b43b3a..7360f025d 100644 --- a/scripts/v1-progress.yaml +++ b/scripts/v1-progress.yaml @@ -123,7 +123,7 @@ - created_at: 1 - updated_at: 1 - bounty_id: 1 - - status: 0 + - status: 1 - github_user_id: 1 - github_url: 1 - charge_id: -1 diff --git a/scripts/v2-progress.yaml b/scripts/v2-progress.yaml index e1956480c..116c20a48 100644 --- a/scripts/v2-progress.yaml +++ b/scripts/v2-progress.yaml @@ -43,7 +43,7 @@ - updated_at: -1 - claims: - id: 1 - - status: 0 + - status: 1 - type: 1 - url: 1 - group_id: 1 From 42b83cafe9957f9e8a8500390912ac24f7fabf22 Mon Sep 17 00:00:00 2001 From: zafer Date: Fri, 14 Feb 2025 19:05:35 +0300 Subject: [PATCH 49/51] update bounty status --- scripts/v1-progress.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/v1-progress.yaml b/scripts/v1-progress.yaml index 7360f025d..76f6ab349 100644 --- a/scripts/v1-progress.yaml +++ b/scripts/v1-progress.yaml @@ -71,7 +71,7 @@ - poster_id: 1 - org_id: 1 - task_id: 1 - - status: 0 + - status: 1 - github_req_comment_id: 1 - github_res_comment_id: 1 - type: 0 From 320c25f3a4113e89c0da413eaa8057cac35f4080 Mon Sep 17 00:00:00 2001 From: zafer Date: Fri, 14 Feb 2025 19:13:30 +0300 Subject: [PATCH 50/51] update sanity checks --- lib/algora/admin/migration.ex | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/algora/admin/migration.ex b/lib/algora/admin/migration.ex index 0887612a4..a7fc72739 100644 --- a/lib/algora/admin/migration.ex +++ b/lib/algora/admin/migration.ex @@ -6,7 +6,7 @@ defmodule Algora.Admin.Migration do alias Algora.Payments.Transaction alias Algora.Repo - @balances_file ".local/db/balance-2025-02-10.json" + @balances_file ".local/db/balance-2025-02-13.json" def get_actual_balances(type) do with {:ok, content} <- File.read(@balances_file), @@ -23,7 +23,9 @@ defmodule Algora.Admin.Migration do end def get_balances(type) do - Enum.filter(get_balances(), &(&1.type == type)) + get_balances() + |> Enum.filter(&(&1.type == type)) + |> Enum.reject(&(&1.provider_login in ["algora-io", "Uber4Coding"])) end def get_balances do @@ -47,9 +49,9 @@ defmodule Algora.Admin.Migration do """ CASE WHEN ? = 'credit' THEN ? - WHEN ? = 'debit' THEN money_negate(?) + WHEN ? = 'debit' THEN -? WHEN ? = 'charge' THEN ? - WHEN ? = 'transfer' THEN money_negate(?) + WHEN ? = 'transfer' THEN -? ELSE ('USD', 0)::money_with_currency END """, @@ -86,6 +88,10 @@ defmodule Algora.Admin.Migration do end) end + def diff_balances do + diff_balances(:individual) ++ diff_balances(:organization) + end + def diff_balances(type) do actual = Enum.map(get_actual_balances(type), &Map.take(&1, [:provider_login, :balance])) current = Enum.map(get_balances(type), &Map.take(&1, [:provider_login, :balance])) From 10f78b05c0c02ef266251ea6a94318cd18d39a38 Mon Sep 17 00:00:00 2001 From: zafer Date: Fri, 14 Feb 2025 19:26:18 +0300 Subject: [PATCH 51/51] update migration script --- lib/algora/integrations/github/client.ex | 26 ++++++++++++------------ scripts/database_migration.exs | 12 ++++++----- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/lib/algora/integrations/github/client.ex b/lib/algora/integrations/github/client.ex index d18af5d3c..484bf294f 100644 --- a/lib/algora/integrations/github/client.ex +++ b/lib/algora/integrations/github/client.ex @@ -8,24 +8,24 @@ defmodule Algora.Github.Client do @type token :: String.t() - # TODO: move to a separate module and use only for data migration between databases def http(host, method, path, headers, body, opts \\ []) do - cache_path = ".local/github/#{path}.json" - - with :error <- read_from_cache(cache_path), - {:ok, response_body} <- do_http_request(host, method, path, headers, body, opts) do - write_to_cache(cache_path, response_body) - {:ok, response_body} + # TODO: remove after migration + if System.get_env("MIGRATION", "false") == "true" do + cache_path = ".local/github/#{path}.json" + + with :error <- read_from_cache(cache_path), + {:ok, response_body} <- do_http_request(host, method, path, headers, body, opts) do + write_to_cache(cache_path, response_body) + {:ok, response_body} + else + {:ok, cached_data} -> {:ok, cached_data} + {:error, reason} -> {:error, reason} + end else - {:ok, cached_data} -> {:ok, cached_data} - {:error, reason} -> {:error, reason} + do_http_request(host, method, path, headers, body, opts) end end - def http0(host, method, path, headers, body, opts \\ []) do - do_http_request(host, method, path, headers, body, opts) - end - defp do_http_request(host, method, path, headers, body, opts) do url = "https://#{host}#{path}" headers = [{"Content-Type", "application/json"} | headers] diff --git a/scripts/database_migration.exs b/scripts/database_migration.exs index d4786673e..8accdced3 100644 --- a/scripts/database_migration.exs +++ b/scripts/database_migration.exs @@ -1331,9 +1331,9 @@ defmodule DatabaseMigration do end defp time_step(description, function) do - IO.puts("\n#{description}...") + IO.puts("⏳ #{description}...") {time, result} = :timer.tc(function) - IO.puts("✓ #{description} completed in #{time / 1_000_000} seconds") + IO.puts("✅ #{description} completed in #{time / 1_000_000} seconds") result end @@ -1341,18 +1341,20 @@ defmodule DatabaseMigration do input_file = ".local/db/v1-data-2025-02-13.sql" output_file = ".local/db/v2-data-2025-02-13.sql" + System.put_env("MIGRATION", "true") + if File.exists?(input_file) or File.exists?(output_file) do - IO.puts("\nStarting migration...") + IO.puts("⏳ Starting migration...") {total_time, _} = :timer.tc(fn -> :ok = time_step("Processing dump", fn -> process_dump(input_file, output_file) end) :ok = time_step("Clearing tables", fn -> clear_tables!() end) {:ok, _} = time_step("Importing new data", fn -> psql(["-f", output_file]) end) - # :ok = time_step("Backfilling repositories", fn -> Algora.Admin.backfill_repos!() end) + :ok = time_step("Backfilling repositories", fn -> Algora.Admin.backfill_repos!() end) end) - IO.puts("\n✓ Migration completed successfully in #{total_time / 1_000_000} seconds") + IO.puts("✅ Migration completed successfully in #{total_time / 1_000_000} seconds") end end end