add missing transaction data

Author: zcesur

scripts/database_migration.exs

@@ -445,7 +445,7 @@ defmodule DatabaseMigration do
       "status" => if(row["succeeded_at"] == nil, do: :initialized, else: :succeeded),
       "succeeded_at" => row["succeeded_at"],
       "reversed_at" => nil,
-      "group_id" => nil,
+      "group_id" => row["id"],
       ## TODO: this might be null but shouldn't
       "user_id" => user["id"],
       "contract_id" => nil,
@@ -503,7 +503,9 @@ defmodule DatabaseMigration do
 
     user = db |> Map.get("User", []) |> Enum.find(&(&1["id"] == github_user["user_id"]))
 
-    amount = Money.from_integer(String.to_integer(row["amount"]), row["currency"])
+    org = db |> Map.get("Org", []) |> Enum.find(&(&1["id"] == bounty["org_id"]))
+
+    bounty_charge = db |> Map.get("BountyCharge", []) |> Enum.find(&(&1["id"] == row["bounty_charge_id"]))
 
     if !bounty do
       raise "Bounty not found: #{inspect(row)}"
@@ -513,48 +515,37 @@ defmodule DatabaseMigration do
       raise "User not found: #{inspect(row)}"
     end
 
-    # TODO: add corresponding credit & debit transactions
-    row = %{
-      "id" => row["id"],
-      "provider" => "stripe",
-      "provider_id" => row["transfer_id"],
-      "provider_charge_id" => nil,
-      "provider_payment_intent_id" => nil,
-      "provider_transfer_id" => row["transfer_id"],
-      "provider_invoice_id" => nil,
-      "provider_balance_transaction_id" => nil,
-      "provider_meta" => nil,
-      "gross_amount" => amount,
-      "net_amount" => amount,
-      "total_fee" => Money.zero(:USD),
-      "provider_fee" => nil,
-      "line_items" => nil,
-      "type" => "transfer",
-      # TODO: only add debit & credit transactions if not succeeded
-      "status" => if(row["succeeded_at"] == nil, do: :initialized, else: :succeeded),
-      "succeeded_at" => row["succeeded_at"],
-      "reversed_at" => nil,
-      "group_id" => nil,
-      ## TODO: this might be null but shouldn't
-      "user_id" => user["id"],
-      "contract_id" => nil,
-      "original_contract_id" => nil,
-      "timesheet_id" => nil,
-      "bounty_id" => nil,
-      "tip_id" => nil,
-      "linked_transaction_id" => nil,
-      "inserted_at" => row["created_at"],
-      "updated_at" => row["updated_at"],
-      "claim_id" => nil
-    }
+    if !org do
+      raise "Org not found: #{inspect(row)}"
+    end
 
-    if bounty["type"] == "tip" do
-      Map.put(row, "tip_id", bounty["id"] <> user["id"])
-    else
-      row
-      |> Map.put("bounty_id", claim["bounty_id"])
-      |> Map.put("claim_id", claim["id"])
+    if !bounty_charge do
+      raise "BountyCharge not found: #{inspect(row)}"
     end
+
+    [
+      maybe_create_transaction("debit", %{
+        bounty_charge: bounty_charge,
+        bounty_transfer: row,
+        bounty: bounty,
+        claim: claim,
+        user: org
+      }),
+      maybe_create_transaction("credit", %{
+        bounty_charge: bounty_charge,
+        bounty_transfer: row,
+        bounty: bounty,
+        claim: claim,
+        user: user
+      }),
+      maybe_create_transaction("transfer", %{
+        bounty_charge: bounty_charge,
+        bounty_transfer: row,
+        bounty: bounty,
+        claim: claim,
+        user: user
+      })
+    ]
   end
 
   defp transform({"GithubInstallation", Installation}, row, _db) do
@@ -566,7 +557,7 @@ defmodule DatabaseMigration do
       "avatar_url" => nil,
       "repository_selection" => nil,
       "owner_id" => nil,
-      "connected_user_id" => nil,
+      "connected_user_id" => row["org_id"],
       "inserted_at" => row["created_at"],
       "updated_at" => row["updated_at"],
       "provider_user_id" => nil
@@ -631,6 +622,90 @@ defmodule DatabaseMigration do
 
   defp transform(_, _row, _db), do: nil
 
+  defp maybe_create_transaction(type, %{
+         bounty_charge: bounty_charge,
+         bounty_transfer: bounty_transfer,
+         bounty: bounty,
+         claim: claim,
+         user: user
+       }) do
+    amount = Money.from_integer(String.to_integer(bounty_transfer["amount"]), bounty_transfer["currency"])
+
+    res = %{
+      "id" => String.slice(type, 0, 2) <> "_" <> bounty_transfer["id"],
+      "provider" => "stripe",
+      "provider_id" => nil,
+      "provider_charge_id" => bounty_charge["charge_id"],
+      "provider_payment_intent_id" => nil,
+      "provider_transfer_id" => bounty_transfer["transfer_id"],
+      "provider_invoice_id" => nil,
+      "provider_balance_transaction_id" => nil,
+      "provider_meta" => nil,
+      "gross_amount" => amount,
+      "net_amount" => amount,
+      "total_fee" => Money.zero(:USD),
+      "provider_fee" => nil,
+      "line_items" => nil,
+      "type" => type,
+      "status" => nil,
+      "succeeded_at" => nil,
+      "reversed_at" => nil,
+      "group_id" => bounty_transfer["bounty_charge_id"],
+      ## TODO: this might be null but shouldn't
+      "user_id" => user["id"],
+      "contract_id" => nil,
+      "original_contract_id" => nil,
+      "timesheet_id" => nil,
+      "bounty_id" => nil,
+      "tip_id" => nil,
+      "linked_transaction_id" => nil,
+      "inserted_at" => bounty_transfer["created_at"],
+      "updated_at" => bounty_transfer["updated_at"],
+      "claim_id" => nil
+    }
+
+    res =
+      if bounty["type"] == "tip" do
+        Map.put(res, "tip_id", bounty["id"] <> user["id"])
+      else
+        res
+        |> Map.put("bounty_id", claim["bounty_id"])
+        |> Map.put("claim_id", claim["id"])
+      end
+
+    res =
+      case type do
+        "transfer" ->
+          Map.put(res, "provider_id", bounty_transfer["transfer_id"])
+
+        "debit" ->
+          Map.put(res, "linked_transaction_id", "cr_" <> bounty_transfer["id"])
+
+        "credit" ->
+          Map.put(res, "linked_transaction_id", "de_" <> bounty_transfer["id"])
+
+        _ ->
+          res
+      end
+
+    cond do
+      bounty_charge["succeeded_at"] == nil ->
+        nil
+
+      type == "transfer" && bounty_transfer["succeeded_at"] == nil ->
+        nil
+
+      type == "transfer" ->
+        res |> Map.put("status", :succeeded) |> Map.put("succeeded_at", bounty_transfer["succeeded_at"])
+
+      type == "debit" ->
+        res |> Map.put("status", :succeeded) |> Map.put("succeeded_at", bounty_charge["succeeded_at"])
+
+      type == "credit" ->
+        res |> Map.put("status", :succeeded) |> Map.put("succeeded_at", bounty_charge["succeeded_at"])
+    end
+  end
+
   def process_dump(input_file, output_file) do
     db = collect_data(input_file)
 
@@ -715,26 +790,27 @@ defmodule DatabaseMigration do
   defp transform_section(%{table: table, columns: _columns, data: data}, schema, db) do
     transformed_data =
       data
-      |> Enum.map(fn row ->
+      |> Enum.flat_map(fn row ->
         # try do
-        transform({table, schema}, row, db)
+        case transform({table, schema}, row, db) do
+          nil -> []
+          xs when is_list(xs) -> xs
+          x -> [x]
+        end
+
         # rescue
         #   e ->
         #     IO.puts("Error transforming row in table #{table}: #{inspect(row)}")
         #     IO.puts("Error: #{inspect(e)}")
         #     nil
         # end
       end)
-      |> Enum.reject(&is_nil/1)
       |> Enum.map(&post_transform(schema, &1))
 
-    transformed_table_name = schema.__schema__(:source)
-
     if Enum.empty?(transformed_data) do
       nil
     else
-      transformed_columns = Map.keys(hd(transformed_data))
-      %{table: transformed_table_name, columns: transformed_columns, data: transformed_data}
+      %{table: schema.__schema__(:source), columns: Map.keys(hd(transformed_data)), data: transformed_data}
     end
   end
 

scripts/v1-progress.yaml

@@ -90,7 +90,7 @@
     - updated_at: 1
     - succeeded_at: 1
     - charge_id: 1
-    - user_id: 0
+    - user_id: -1
     - org_id: 1
     - amount: 1
     - currency: 1
@@ -113,7 +113,7 @@
     - created_at: 1
     - updated_at: 1
     - succeeded_at: 1
-    - bounty_charge_id: 0
+    - bounty_charge_id: 1
     - claim_id: 1
     - transfer_id: 1
     - amount: 1
@@ -203,8 +203,8 @@
     - created_at: 1
     - updated_at: 1
     - github_id: 1
-    - org_id: 0
-    - github_org_handle: 0
+    - org_id: 1
+    - github_org_handle: -1
 - "GithubIssue":
     - id: 1
     - url: -1

scripts/v2-progress.yaml

@@ -45,7 +45,7 @@
     - id: 1
     - status: 0
     - type: 0
-    - url: 0
+    - url: 1
     - group_id: 0
     - group_share: 0
     - source_id: 1
@@ -125,11 +125,11 @@
     - provider_meta: -1
     - avatar_url: -1
     - repository_selection: -1
-    - owner_id: 0
-    - connected_user_id: 0
+    - owner_id: -2
+    - connected_user_id: 1
     - inserted_at: 1
     - updated_at: 1
-    - provider_user_id: 0
+    - provider_user_id: -1
 - members:
     - id: 1
     - role: 1
@@ -194,7 +194,7 @@
     - description: 1
     - number: 1
     - url: 1
-    - repository_id: 0
+    - repository_id: -1
     - inserted_at: 1
     - updated_at: 1
 - timesheets:
@@ -233,14 +233,14 @@
     - status: 1
     - succeeded_at: 1
     - reversed_at: -1
-    - group_id: 0
-    - user_id: 0
+    - group_id: 1
+    - user_id: 1
     - contract_id: -1
     - original_contract_id: -1
     - timesheet_id: -1
     - bounty_id: 1
-    - tip_id: 0
-    - linked_transaction_id: 0
+    - tip_id: 1
+    - linked_transaction_id: 1
     - inserted_at: 1
     - updated_at: 1
     - claim_id: 1