Add various performance fixes for API endpoints

- Remove preloads from views and write explicitly in controllers
- Reduce id filter repetition
- Ensure id query filters are run on every controller that has an index action
- Add missing indexes

Author: joshsmith

lib/code_corps/helpers/query.ex

@@ -36,14 +36,6 @@ defmodule CodeCorps.Helpers.Query do
 
   # end task queries
 
-  # def comment queries
-
-  def task_filter(query, task_id) do
-    query |> where([object], object.task_id == ^task_id)
-  end
-
-  # end comment queries
-
   # user queries
 
   def user_filter(query, %{"query" => query_string}) do
@@ -61,8 +53,6 @@ defmodule CodeCorps.Helpers.Query do
 
   # sorting
 
-  def sort_by_newest_first(query), do: query |> order_by([desc: :inserted_at])
-
   def sort_by_order(query), do: query |> order_by([asc: :order])
 
   # end sorting

lib/code_corps/task/query.ex

@@ -19,44 +19,49 @@ defmodule CodeCorps.Task.Query do
   @spec list(map) :: list(Project.t)
   def list(%{} = params) do
     Task
+    |> Helpers.Query.id_filter(params)
     |> apply_archived_status(params)
+    |> apply_status(params)
     |> apply_optional_filters(params)
     |> order_by([asc: :order])
-    |> Repo.all
+    |> Repo.all()
   end
 
   @spec apply_optional_filters(Queryable.t, map) :: Queryable.t
-  defp apply_optional_filters(queryable, %{"filter" => %{} = params}) do
-    queryable |> apply_optional_filters(params)
+  defp apply_optional_filters(query, %{"filter" => %{} = params}) do
+    query |> apply_optional_filters(params)
   end
-  defp apply_optional_filters(queryable, %{"project_id" => project_id} = params) do
-    queryable
+  defp apply_optional_filters(query, %{"project_id" => project_id} = params) do
+    query
     |> where(project_id: ^project_id)
     |> apply_optional_filters(params |> Map.delete("project_id"))
   end
-  defp apply_optional_filters(queryable, %{"task_list_ids" => task_list_ids} = params) do
+  defp apply_optional_filters(query, %{"task_list_ids" => task_list_ids} = params) do
     task_list_ids = task_list_ids |> Helpers.String.coalesce_id_string
 
-    queryable
+    query
     |> where([r], r.task_list_id in ^task_list_ids)
     |> apply_optional_filters(params |> Map.delete("task_list_ids"))
   end
-  defp apply_optional_filters(queryable, %{"status" => status} = params) do
-    queryable
-    |> where(status: ^status)
-    |> apply_optional_filters(params |> Map.delete("status"))
-  end
-  defp apply_optional_filters(queryable, %{}), do: queryable
+  defp apply_optional_filters(query, %{}), do: query
 
   @spec apply_archived_status(Queryable.t, map) :: Queryable.t
-  defp apply_archived_status(queryable, %{"archived" => archived}) do
-    queryable
+  defp apply_archived_status(query, %{"archived" => archived}) do
+    query
     |> where(archived: ^archived)
   end
-  defp apply_archived_status(queryable, %{}) do
-    queryable
+  defp apply_archived_status(query, %{}) do
+    query
     |> where(archived: false)
   end
+
+  @spec apply_status(Queryable.t, map) :: Queryable.t
+  defp apply_status(query, %{"status" => status}) do
+    query
+    |> where(status: ^status)
+  end
+  defp apply_status(query, %{}), do: query
+
   @doc ~S"""
   Returns a `Task` record retrived using a set of parameters.
 

lib/code_corps_web/controllers/category_controller.ex

@@ -2,21 +2,20 @@ defmodule CodeCorpsWeb.CategoryController do
   @moduledoc false
   use CodeCorpsWeb, :controller
 
-  alias CodeCorps.{Category, User, Helpers.Query}
+  alias CodeCorps.{Category, Repo, User, Helpers.Query}
 
   action_fallback CodeCorpsWeb.FallbackController
   plug CodeCorpsWeb.Plug.DataToAttributes
 
   @spec index(Conn.t, map) :: Conn.t
   def index(%Conn{} = conn, %{} = params) do
-    with categories <- Category |> Query.id_filter(params) |> Repo.all do
-      conn |> render("index.json-api", data: categories)
-    end
+    categories = Category |> Query.id_filter(params) |> Repo.all |> preload()
+    conn |> render("index.json-api", data: categories)
   end
 
   @spec show(Conn.t, map) :: Conn.t
   def show(%Conn{} = conn, %{"id" => id}) do
-    with %Category{} = category <- Category |> Repo.get(id) do
+    with %Category{} = category <- Category |> Repo.get(id) |> preload() do
       conn |> render("show.json-api", data: category)
     end
   end
@@ -25,7 +24,8 @@ defmodule CodeCorpsWeb.CategoryController do
   def create(%Conn{} = conn, %{} = params) do
     with %User{} = current_user <- conn |> Guardian.Plug.current_resource,
       {:ok, :authorized} <- current_user |> Policy.authorize(:create, %Category{}, params),
-      {:ok, %Category{} = category} <- %Category{} |> Category.create_changeset(params) |> Repo.insert
+      {:ok, %Category{} = category} <- %Category{} |> Category.create_changeset(params) |> Repo.insert,
+      category <- preload(category)
     do
       conn |> put_status(:created) |> render("show.json-api", data: category)
     end
@@ -36,9 +36,16 @@ defmodule CodeCorpsWeb.CategoryController do
     with %Category{} = category <- Category |> Repo.get(id),
       %User{} = current_user <- conn |> Guardian.Plug.current_resource,
       {:ok, :authorized} <- current_user |> Policy.authorize(:update, category),
-      {:ok, %Category{} = category} <- category |> Category.changeset(params) |> Repo.update
+      {:ok, %Category{} = category} <- category |> Category.changeset(params) |> Repo.update,
+      category <- preload(category)
     do
       conn |> render("show.json-api", data: category)
     end
   end
+
+  @preloads [:project_categories]
+
+  def preload(data) do
+    Repo.preload(data, @preloads)
+  end
 end

lib/code_corps_web/controllers/donation_goal_controller.ex

@@ -11,14 +11,18 @@ defmodule CodeCorpsWeb.DonationGoalController do
 
   @spec index(Conn.t, map) :: Conn.t
   def index(%Conn{} = conn, %{} = params) do
-    with donation_goals <- DonationGoal |> Query.id_filter(params) |> Repo.all do
-      conn |> render("index.json-api", data: donation_goals)
-    end
+    donation_goals =
+      DonationGoal
+      |> Query.id_filter(params)
+      |> Repo.all
+      |> preload()
+
+    conn |> render("index.json-api", data: donation_goals)
   end
 
   @spec show(Conn.t, map) :: Conn.t
   def show(%Conn{} = conn, %{"id" => id}) do
-    with %DonationGoal{} = donation_goal <- DonationGoal |> Repo.get(id) do
+    with %DonationGoal{} = donation_goal <- DonationGoal |> Repo.get(id) |> preload() do
       conn |> render("show.json-api", data: donation_goal)
     end
   end
@@ -27,7 +31,9 @@ defmodule CodeCorpsWeb.DonationGoalController do
   def create(%Conn{} = conn, %{} = params) do
     with %User{} = current_user <- conn |> Guardian.Plug.current_resource,
          {:ok, :authorized} <- current_user |> Policy.authorize(:create, %DonationGoal{}, params),
-         {:ok, %DonationGoal{} = donation_goal} <- DonationGoalsService.create(params) do
+         {:ok, %DonationGoal{} = donation_goal} <- DonationGoalsService.create(params),
+         donation_goal <- preload(donation_goal)
+    do
       conn |> put_status(:created) |> render("show.json-api", data: donation_goal)
     end
   end
@@ -48,8 +54,16 @@ defmodule CodeCorpsWeb.DonationGoalController do
     with %DonationGoal{} = donation_goal <- DonationGoal |> Repo.get(id),
       %User{} = current_user <- conn |> Guardian.Plug.current_resource,
       {:ok, :authorized} <- current_user |> Policy.authorize(:update, donation_goal),
-      {:ok, %DonationGoal{} = updated_donation_goal} <- donation_goal |> DonationGoalsService.update(params) do
-        conn |> render("show.json-api", data: updated_donation_goal)
+      {:ok, %DonationGoal{} = updated_donation_goal} <- donation_goal |> DonationGoalsService.update(params),
+      updated_donation_goal <- preload(updated_donation_goal)
+    do
+      conn |> render("show.json-api", data: updated_donation_goal)
     end
   end
+
+  @preloads [:project]
+
+  def preload(data) do
+    Repo.preload(data, @preloads)
+  end
 end

lib/code_corps_web/controllers/github_app_installation_controller.ex

@@ -12,14 +12,18 @@ defmodule CodeCorpsWeb.GithubAppInstallationController do
 
   @spec index(Conn.t, map) :: Conn.t
   def index(%Conn{} = conn, %{} = params) do
-    with installations <- GithubAppInstallation |> id_filter(params) |> Repo.all do
-      conn |> render("index.json-api", data: installations)
-    end
+    installations =
+      GithubAppInstallation
+      |> id_filter(params)
+      |> Repo.all()
+      |> preload()
+
+    conn |> render("index.json-api", data: installations)
   end
 
   @spec show(Conn.t, map) :: Conn.t
   def show(%Conn{} = conn, %{"id" => id}) do
-    with %GithubAppInstallation{} = installation <- GithubAppInstallation |> Repo.get(id) do
+    with %GithubAppInstallation{} = installation <- GithubAppInstallation |> Repo.get(id) |> preload() do
       conn |> render("show.json-api", data: installation)
     end
   end
@@ -28,13 +32,20 @@ defmodule CodeCorpsWeb.GithubAppInstallationController do
   def create(%Conn{} = conn, %{} = params) do
     with %User{} = current_user <- conn |> Guardian.Plug.current_resource,
          {:ok, :authorized} <- current_user |> Policy.authorize(:create, %GithubAppInstallation{}, params),
-         {:ok, %GithubAppInstallation{} = installation} <- %GithubAppInstallation{} |> GithubAppInstallation.create_changeset(params) |> Repo.insert do
-
+         {:ok, %GithubAppInstallation{} = installation} <- %GithubAppInstallation{} |> GithubAppInstallation.create_changeset(params) |> Repo.insert,
+         installation <- preload(installation)
+    do
       current_user |> track_created(installation)
       conn |> put_status(:created) |> render("show.json-api", data: installation)
     end
   end
 
+  @preloads [:github_repos, :organization_github_app_installations]
+
+  def preload(data) do
+    Repo.preload(data, @preloads)
+  end
+
   @spec track_created(User.t, GithubAppInstallation.t) :: any
   defp track_created(%User{id: user_id}, %GithubAppInstallation{} = installation) do
     user_id |> SegmentTracker.track("Created GitHub App Installation", installation)

lib/code_corps_web/controllers/organization_controller.ex

@@ -4,19 +4,24 @@ defmodule CodeCorpsWeb.OrganizationController do
 
   alias CodeCorps.{Helpers.Query, Organization, User}
 
-  action_fallback CodeCorpsWeb.FallbackController 
-  plug CodeCorpsWeb.Plug.DataToAttributes  
+  action_fallback CodeCorpsWeb.FallbackController
+  plug CodeCorpsWeb.Plug.DataToAttributes
 
   @spec index(Conn.t, map) :: Conn.t
   def index(%Conn{} = conn, %{} = params) do
-    with organizations <- Organization |> Query.id_filter(params) |> Repo.all do
-      conn |> render("index.json-api", data: organizations)
-    end
+    organizations =
+      Organization
+      |> Query.id_filter(params)
+      |> Repo.all
+      |> preload()
+
+    conn |> render("index.json-api", data: organizations)
   end
 
   @spec show(Conn.t, map) :: Conn.t
   def show(%Conn{} = conn, %{"id" => id}) do
-    with %Organization{} = organization <- Organization |> Repo.get(id) do
+    with %Organization{} = organization <- Organization |> Repo.get(id) |> preload()
+    do
       conn |> render("show.json-api", data: organization)
     end
   end
@@ -25,7 +30,9 @@ defmodule CodeCorpsWeb.OrganizationController do
   def create(%Conn{} = conn, %{} = params) do
     with %User{} = current_user <- conn |> Guardian.Plug.current_resource,
          {:ok, :authorized} <- current_user |> Policy.authorize(:create, %Organization{}, params),
-         {:ok, %Organization{} = organization} <- %Organization{} |> Organization.create_changeset(params) |> Repo.insert do
+         {:ok, %Organization{} = organization} <- %Organization{} |> Organization.create_changeset(params) |> Repo.insert,
+         organization <- preload(organization)
+    do
       conn |> put_status(:created) |> render("show.json-api", data: organization)
     end
   end
@@ -35,8 +42,16 @@ defmodule CodeCorpsWeb.OrganizationController do
     with %Organization{} = organization <- Organization |> Repo.get(id),
       %User{} = current_user <- conn |> Guardian.Plug.current_resource,
       {:ok, :authorized} <- current_user |> Policy.authorize(:update, organization),
-      {:ok, %Organization{} = organization} <- organization |> Organization.changeset(params) |> Repo.update do
+      {:ok, %Organization{} = organization} <- organization |> Organization.changeset(params) |> Repo.update,
+      organization <- preload(organization)
+    do
         conn |> render("show.json-api", data: organization)
     end
   end
+
+  @preloads [:organization_github_app_installations, :projects, :slugged_route, :stripe_connect_account]
+
+  def preload(data) do
+    Repo.preload(data, @preloads)
+  end
 end

lib/code_corps_web/controllers/project_controller.ex

@@ -9,14 +9,14 @@ defmodule CodeCorpsWeb.ProjectController do
 
   @spec index(Conn.t, map) :: Conn.t
   def index(%Conn{} = conn, %{} = params) do
-    with projects <- Project.Query.list(params) do
+    with projects <- Project.Query.list(params) |> preload do
       conn |> render("index.json-api", data: projects)
     end
   end
 
   @spec show(Conn.t, map) :: Conn.t
   def show(%Conn{} = conn, %{} = params) do
-    with %Project{} = project <- Project.Query.find(params) do
+    with %Project{} = project <- Project.Query.find(params) |> preload do
       conn |> render("show.json-api", data: project)
     end
   end
@@ -25,18 +25,32 @@ defmodule CodeCorpsWeb.ProjectController do
   def create(%Conn{} = conn, %{} = params) do
     with %User{} = current_user <- conn |> Guardian.Plug.current_resource,
          {:ok, :authorized} <- current_user |> Policy.authorize(:create, %Project{}, params),
-         {:ok, %Project{} = project} <- %Project{} |> Project.create_changeset(params) |> Repo.insert do
+         {:ok, %Project{} = project} <- %Project{} |> Project.create_changeset(params) |> Repo.insert,
+         project <- preload(project)
+    do
       conn |> put_status(:created) |> render("show.json-api", data: project)
     end
   end
 
   @spec update(Conn.t, map) :: Conn.t
   def update(%Conn{} = conn, %{} = params) do
     with %Project{} = project <- Project.Query.find(params),
-      %User{} = current_user <- conn |> Guardian.Plug.current_resource,
-      {:ok, :authorized} <- current_user |> Policy.authorize(:update, project),
-      {:ok, %Project{} = project} <- project |> Project.changeset(params) |> Repo.update do
-        conn |> render("show.json-api", data: project)
+         %User{} = current_user <- conn |> Guardian.Plug.current_resource,
+         {:ok, :authorized} <- current_user |> Policy.authorize(:update, project),
+         {:ok, %Project{} = project} <- project |> Project.changeset(params) |> Repo.update,
+         project <- preload(project)
+    do
+      conn |> render("show.json-api", data: project)
     end
   end
+
+  @preloads [
+    :donation_goals, [organization: :stripe_connect_account],
+    :project_categories, :project_github_repos, :project_skills,
+    :project_users, :stripe_connect_plan, :task_lists, :tasks
+  ]
+
+  def preload(data) do
+    Repo.preload(data, @preloads)
+  end
 end

lib/code_corps_web/controllers/role_controller.ex

@@ -10,14 +10,13 @@ defmodule CodeCorpsWeb.RoleController do
 
   @spec index(Conn.t, map) :: Conn.t
   def index(%Conn{} = conn, %{} = params) do
-    with roles <- Role |> Query.id_filter(params) |> Repo.all do
-      conn |> render("index.json-api", data: roles)
-    end
+    roles = Role |> Query.id_filter(params) |> Repo.all |> preload()
+    conn |> render("index.json-api", data: roles)
   end
 
   @spec show(Conn.t, map) :: Conn.t
   def show(%Conn{} = conn, %{"id" => id}) do
-    with %Role{} = role <- Role |> Repo.get(id) do
+    with %Role{} = role <- Role |> Repo.get(id) |> preload() do
       conn |> render("show.json-api", data: role)
     end
   end
@@ -26,8 +25,16 @@ defmodule CodeCorpsWeb.RoleController do
   def create(%Conn{} = conn, %{} = params) do
     with %User{} = current_user <- conn |> Guardian.Plug.current_resource,
          {:ok, :authorized} <- current_user |> Policy.authorize(:create, %Role{}, params),
-         {:ok, %Role{} = role} <- %Role{} |> Role.changeset(params) |> Repo.insert do
+         {:ok, %Role{} = role} <- %Role{} |> Role.changeset(params) |> Repo.insert,
+         role = preload(role)
+    do
       conn |> put_status(:created) |> render("show.json-api", data: role)
     end
   end
+
+  @preloads [:role_skills]
+
+  def preload(data) do
+    Repo.preload(data, @preloads)
+  end
 end