new validations & set default configs

Arp-G · Arp-G · commit 7300c30c06a8 · 2022-08-18T19:48:05.000+05:30
diff --git a/apps/csv2sql/lib/csv2sql/config/loader.ex b/apps/csv2sql/lib/csv2sql/config/loader.ex
@@ -30,6 +30,40 @@ defmodule Csv2sql.Config.Loader do
   @insertion_chunk_size 100
   @min_insertion_chunk_size 1
 
+  @spec get_defaults() :: map()
+  def get_defaults do
+    %{
+      date_patterns: [%{id: 1, pattern: @date_pattern}],
+      datetime_patterns: [%{id: 1, pattern: @datetime_pattern}],
+      db_type: @db_type,
+      drop_existing_tables: false,
+      insert_data: false,
+      insert_schema: true,
+      log: false,
+      ordered: false,
+      parse_datetime: true,
+      remove_illegal_characters: false,
+      schema_infer_chunk_size: @schema_infer_chunk_size,
+      db_worker_count: @default_db_worker_count,
+      varchar_limit: @varchar_limit,
+      insertion_chunk_size: @insertion_chunk_size
+    }
+  end
+
+  @spec get_constraints() :: map()
+  def get_constraints do
+    %{
+      schema_infer_chunk_size: %{
+        min: @min_schema_infer_chunk_size,
+        max: @max_schema_infer_chunk_size
+      },
+      varchar_limit: %{min: @min_varchar_limit, max: @max_varchar_limit},
+      worker_count: %{min: @min_worker_count, max: @max_worker_count},
+      db_worker_count: %{min: @min_db_worker_count, max: @max_db_worker_count},
+      insertion_chunk_size: %{min: @min_insertion_chunk_size, max: @max_insertion_chunk_size}
+    }
+  end
+
   @spec load(map) :: :ok
   def load(args), do: Application.put_env(:csv2sql, :config, prepare_configs(args))
 
diff --git a/apps/dashboard/assets/js/app.js b/apps/dashboard/assets/js/app.js
@@ -15,8 +15,7 @@ let hooks = {
       new bootstrap.Tooltip(this.el);
 
       this.handleEvent("save-config", ({ config }) => {
-        console.log("write to local", JSON.stringify(config))
-        localStorage.setItem("config", JSON.stringify(config))
+        localStorage.setItem("config", JSON.stringify(config));
       }
       )
     }
diff --git a/apps/dashboard/lib/dashboard/config.ex b/apps/dashboard/lib/dashboard/config.ex
@@ -9,6 +9,7 @@ end
 defmodule DashBoard.Config do
   use Ecto.Schema
   import Ecto.Changeset
+  import ShorterMaps
 
   @primary_key false
   embedded_schema do
@@ -46,21 +47,8 @@ defmodule DashBoard.Config do
   end
 
   def get_defaults do
-    %__MODULE__{
-      db_type: :mysql,
-      db_worker_count: 10,
-      drop_existing_tables: false,
-      insert_data: true,
-      insert_schema: true,
-      insertion_chunk_size: 100,
-      log: false,
-      ordered: false,
-      parse_datetime: true,
-      remove_illegal_characters: false,
-      schema_infer_chunk_size: 100,
-      varchar_limit: 200,
-      worker_count: System.schedulers_online()
-    }
+    Csv2sql.Config.Loader.get_defaults()
+    |> Map.put(:source_directory, get_source_directory())
   end
 
   def changeset(params), do: changeset(%__MODULE__{}, params)
@@ -78,6 +66,7 @@ defmodule DashBoard.Config do
     changeset =
       config
       |> cast(params, attrs_to_cast)
+      |> validate_limit_constraints(Csv2sql.Config.Loader.get_constraints())
       |> validate_source_directory()
       |> validate_path(:schema_path, true)
       |> cast_embed(:db_attrs)
@@ -88,6 +77,21 @@ defmodule DashBoard.Config do
     %{changeset | action: :insert}
   end
 
+  defp validate_limit_constraints(changeset, constants) do
+    Enum.reduce(
+      constants,
+      changeset,
+      fn {key, ~M{min, max}}, changeset ->
+        validate_number(
+          changeset,
+          key,
+          greater_than_or_equal_to: min,
+          less_than_or_equal_to: max
+        )
+      end
+    )
+  end
+
   defp validate_source_directory(changeset) do
     changeset
     |> validate_path(:source_directory)
@@ -109,25 +113,40 @@ defmodule DashBoard.Config do
     end
   end
 
-  defp add_csv_count(%Ecto.Changeset{valid?: true} = changeset) do
-    csv_count =
+  defp add_csv_count(changeset) do
+    source_directory = get_change(changeset, :source_directory)
+
+    if File.dir?(source_directory) do
+      csv_count =
+        source_directory
+        |> File.ls!()
+        |> Enum.filter(&is_csv?/1)
+        |> Enum.count()
+
+      if csv_count == 0,
+        do: add_error(changeset, :source_directory, "No CSVs found at path"),
+        else: put_change(changeset, :csv_count, csv_count)
+    else
       changeset
-      |> get_change(:source_directory)
-      |> File.ls!()
-      |> Enum.filter(&is_csv?/1)
-      |> Enum.count()
-
-    if csv_count == 0,
-      do: add_error(changeset, :source_directory, "No CSVs found at path"),
-      else: put_change(changeset, :csv_count, csv_count)
+    end
   end
 
-  defp add_csv_count(changeset), do: changeset
-
   defp is_csv?(filepath) do
     filepath
     |> String.trim()
     |> String.slice(-4..-1)
     |> String.downcase() == ".csv"
   end
+
+  defp get_source_directory() do
+    case :os.type() do
+      {:win32, _} ->
+        username = System.shell("echo %USERNAME%") |> elem(0) |> String.trim()
+        "C:/Users/#{username}/Desktop"
+
+      {:unix, _} ->
+        {op, _exit_code} = System.cmd("eval", ["echo ~$USER"])
+        String.trim(op)
+    end
+  end
 end
diff --git a/apps/dashboard/lib/dashboard/date_pattern.ex b/apps/dashboard/lib/dashboard/date_pattern.ex
@@ -8,6 +8,8 @@ defmodule DashBoard.DatePattern do
     field(:pattern, :string)
   end
 
+  def changeset(params), do: changeset(%__MODULE__{}, params)
+
   def changeset(struct, params) do
     struct
     |> cast(params, [:id, :pattern])
diff --git a/apps/dashboard/lib/dashboard/date_time_pattern.ex b/apps/dashboard/lib/dashboard/date_time_pattern.ex
@@ -8,6 +8,8 @@ defmodule DashBoard.DateTimePattern do
     field(:pattern, :string)
   end
 
+  def changeset(params), do: changeset(%__MODULE__{}, params)
+
   def changeset(struct, params) do
     struct
     |> cast(params, [:id, :pattern])
diff --git a/apps/dashboard/lib/dashboard_web/live/config_live.ex b/apps/dashboard/lib/dashboard_web/live/config_live.ex
@@ -53,7 +53,9 @@ defmodule DashboardWeb.Live.ConfigLive do
               tooltip="TODO: popup title"
               placeholder="Worker count">
               <:input let={f}>
-                <%= text_input f, :worker_count, type: "number", class: "form-control", placeholder: "Worker count" %>
+                <%= text_input f, :worker_count, type: "number", class: "form-control", placeholder: "Worker count",
+                    max: @constraints.worker_count.max, min: @constraints.worker_count.min %>
+                <%= error_tag(f, :worker_count) %>
               </:input>
             </.config_item>
 
@@ -64,7 +66,9 @@ defmodule DashboardWeb.Live.ConfigLive do
               tooltip="TODO: popup title"
               placeholder="Chunk Size">
               <:input let={f}>
-                <%= text_input f, :schema_infer_chunk_size, type: "number", class: "form-control", placeholder: "Chunk Size" %>
+                <%= text_input f, :schema_infer_chunk_size, type: "number", class: "form-control", placeholder: "Chunk Size",
+                    max: @constraints.schema_infer_chunk_size.max, min: @constraints.schema_infer_chunk_size.min  %>
+                <%= error_tag(f, :schema_infer_chunk_size) %>
               </:input>
             </.config_item>
 
@@ -104,7 +108,7 @@ defmodule DashboardWeb.Live.ConfigLive do
         </div>
 
         <!-- Database Configurations -->
-        <div class="card m-4 d-flex w-75 m-auto mt-4">
+        <div class={"card m-4 d-flex w-75 m-auto mt-4 #{unless(db_needed(@changeset), do: "invisible")}"}>
           <header>
             <IconSvg.db_settings class="m-2"/>
             <span class="font-monospace small fst-italic fw-bold"> Database Configurations </span>
@@ -123,7 +127,7 @@ defmodule DashboardWeb.Live.ConfigLive do
                 <:input let={f}>
                   <div class="d-flex flex-column">
                     <div class="form-check">
-                      <%= radio_button f, :db_type, :mysql, class: "form-check-input mt-4", id: "mysql_db_type", checked: "checked" %>
+                      <%= radio_button f, :db_type, :mysql, class: "form-check-input mt-4", id: "mysql_db_type", checked: "checked", disable: !db_needed(@changeset) %>
                       <label class="form-check-label mysql-label" for="mysql_db_type">
                         <IconSvg.mysql_icon {%{width: 100, height: 100}} />
                       </label>
@@ -237,7 +241,9 @@ defmodule DashboardWeb.Live.ConfigLive do
                 tooltip="TODO: popup title"
                 placeholder="Varchar size">
                 <:input let={f}>
-                  <%= text_input f, :varchar_limit, type: "number", class: "form-control", placeholder: "Varchar size" %>
+                  <%= text_input f, :varchar_limit, type: "number", class: "form-control", placeholder: "Varchar size",
+                      max: @constraints.varchar_limit.max, min: @constraints.varchar_limit.min  %>
+                  <%= error_tag(f, :varchar_limit) %>
                 </:input>
               </.config_item>
 
@@ -249,7 +255,9 @@ defmodule DashboardWeb.Live.ConfigLive do
                 tooltip="TODO: popup title"
                 placeholder="DB Worker count">
                 <:input let={f}>
-                  <%= text_input f, :db_worker_count, type: "number", class: "form-control", placeholder: "DB Worker count" %>
+                  <%= text_input f, :db_worker_count, type: "number", class: "form-control", placeholder: "DB Worker count",
+                      max: @constraints.db_worker_count.max, min: @constraints.db_worker_count.min  %>
+                  <%= error_tag(f, :db_worker_count) %>
                 </:input>
               </.config_item>
 
@@ -260,7 +268,9 @@ defmodule DashboardWeb.Live.ConfigLive do
                 tooltip="TODO: popup title"
                 placeholder="Insertion Chunk Size">
                 <:input let={f}>
-                  <%= text_input f, :insertion_chunk_size, type: "number", class: "form-control", placeholder: "Insertion Chunk size" %>
+                  <%= text_input f, :insertion_chunk_size, type: "number", class: "form-control", placeholder: "Insertion Chunk size",
+                      max: @constraints.insertion_chunk_size.max, min: @constraints.insertion_chunk_size.min  %>
+                  <%= error_tag(f, :insertion_chunk_size) %>
                 </:input>
               </.config_item>
             </div>
@@ -296,4 +306,9 @@ defmodule DashboardWeb.Live.ConfigLive do
       </.form>
     """
   end
+
+  defp db_needed(changeset) do
+    Ecto.Changeset.get_field(changeset, :insert_schema) ||
+      Ecto.Changeset.get_field(changeset, :insert_data)
+  end
 end
diff --git a/apps/dashboard/lib/dashboard_web/live/main_live.ex b/apps/dashboard/lib/dashboard_web/live/main_live.ex
@@ -8,7 +8,11 @@ defmodule DashboardWeb.Live.MainLive do
 
   @impl true
   def mount(_params, _session, socket) do
-    localstorage_config = (get_connect_params(socket) || %{}) |> Map.get("localConfig", %{})
+    local_storage_config = (get_connect_params(socket) || %{}) |> Map.get("localConfig", %{})
+
+    local_storage_config =
+      for {key, val} <- local_storage_config, into: %{}, do: {String.to_existing_atom(key), val}
+
     # Check for DB connection on config load from local storage
     timer_ref = Process.send_after(self(), :check_db_connection, @debounce_time)
 
@@ -19,8 +23,9 @@ defmodule DashboardWeb.Live.MainLive do
        path_validator_debouncer: nil,
        db_connection_debouncer: timer_ref,
        db_connection_established: false,
-       changeset: Config.get_defaults() |> Config.changeset(localstorage_config),
-       matching_date_time: nil
+       changeset: Config.get_defaults() |> Map.merge(local_storage_config) |> Config.changeset(),
+       matching_date_time: nil,
+       constraints: Csv2sql.Config.Loader.get_constraints()
      )}
   end
 
diff --git a/apps/dashboard/lib/dashboard_web/live/modals/date_time_patterns_modal.ex b/apps/dashboard/lib/dashboard_web/live/modals/date_time_patterns_modal.ex
@@ -49,14 +49,16 @@ defmodule DashboardWeb.Live.Modal.DateTimePatternsModal do
                         <div class="d-flex flex-row">
                           <div class="input-group mb-3">
                             <span class="input-group-text fw-bold pe-2 pt-1"><%= index %>.</span>
-                            <%= text_input date_patterns_form, :pattern, class: input_class(index, :date, @matching_date_time), placeholder: "Pattern", list: "date-patterns-suggestions" %>
+                            <%= text_input date_patterns_form, :pattern, disabled: index == 1, class: input_class(index, :date, @matching_date_time), placeholder: "Pattern", list: "date-patterns-suggestions" %>
                           </div>
 
                           <%= hidden_input(date_patterns_form, :id) %>
 
-                          <div role="button" phx-click="remove-date-pattern" phx-value-attrid={Ecto.Changeset.get_field(date_patterns_form.source, :id)}>
-                            <IconSvg.remove_icon class="ms-2 pt-1"/>
-                          </div>
+                          <%= unless index == 1 do %>
+                            <div role="button" phx-click="remove-date-pattern" phx-value-attrid={Ecto.Changeset.get_field(date_patterns_form.source, :id)}>
+                              <IconSvg.remove_icon class="ms-2 pt-1"/>
+                            </div>
+                          <% end %>
                         </div>
                       <% end %>
                     <% else %>
@@ -78,14 +80,16 @@ defmodule DashboardWeb.Live.Modal.DateTimePatternsModal do
                         <div class="d-flex flex-row">
                           <div class="input-group mb-3">
                             <span class="input-group-text fw-bold pe-2 pt-1"><%= index %>.</span>
-                            <%= text_input date_time_patterns_form, :pattern, class: input_class(index, :date_time, @matching_date_time), placeholder: "Pattern", list: "date-time-patterns-suggestions" %>
+                            <%= text_input date_time_patterns_form, :pattern, disabled: index == 1, class: input_class(index, :date_time, @matching_date_time), placeholder: "Pattern", list: "date-time-patterns-suggestions" %>
                           </div>
 
                           <%= hidden_input(date_time_patterns_form, :id) %>
 
-                          <div role="button" phx-click="remove-date-time-pattern" phx-value-attrid={Ecto.Changeset.get_field(date_time_patterns_form.source, :id)}>
-                            <IconSvg.remove_icon class="ms-2 pt-1"/>
-                          </div>
+                          <%= unless index == 1 do %>
+                            <div role="button" phx-click="remove-date-time-pattern" phx-value-attrid={Ecto.Changeset.get_field(date_time_patterns_form.source, :id)}>
+                              <IconSvg.remove_icon class="ms-2 pt-1"/>
+                            </div>
+                          <% end %>
                         </div>
                       <% end %>
                     <% else %>

Original file line number	Diff line number	Diff line change
`@@ -15,8 +15,7 @@ let hooks = {`
`15`	`15`	`new bootstrap.Tooltip(this.el);`
`16`	`16`
`17`	`17`	`this.handleEvent("save-config", ({ config }) => {`
`18`		`- console.log("write to local", JSON.stringify(config))`
`19`		`- localStorage.setItem("config", JSON.stringify(config))`
	`18`	`+ localStorage.setItem("config", JSON.stringify(config));`
`20`	`19`	`}`
`21`	`20`	`)`
`22`	`21`	`}`