feat(front): use a job to fetch yamls in workflow editor (#206)

The workflow editor currently fetches yaml files in a synchronous call
before the page is rendered, which is very unstable for large
repositories and can lead to 500 errors.
This PR changes this behavior so the workflow editor starts with a zero
state screen with the spinner, and the job is started in the background
to fetch yaml files asynchronously.
Once the files are fetched successfully, the workflow editor is fully
rendered with visual and text editor components.
This behavior will remain behind a feature flag until we verify that it
works as expected at scale.

Co-authored-by: Lucas Pinheiro <[email protected]>

Author: DamjanBecirovic

front/assets/js/workflow_editor/editor.js

@@ -33,6 +33,8 @@ export class WorkflowEditor {
       workflowData: InjectedDataByBackend.WorkflowData,
       agentTypes: InjectedDataByBackend.AgentTypes,
       deploymentTargets: InjectedDataByBackend.DeploymentTargetsList,
+      checkFetchingJobPath: InjectedDataByBackend.CheckFetchingJobPath,
+      fetchingJobId: InjectedDataByBackend.FetchingJobId,
       commitInfo: {
         paths: {
           dismiss: InjectedDataByBackend.CommitForm.DismissPath,
@@ -67,20 +69,129 @@ export class WorkflowEditor {
   }
 
   constructor(config) {
-    this.config = config
+    this.config = config;
 
-    Promotion.setProjectName(this.config.projectName)
-    Promotion.setValidDeploymentTargets(this.config.deploymentTargets)
-    Secrets.setValidSecretNames(this.config.orgSecretNames, this.config.projectSecretNames)
-    Agent.setValidAgentTypes(this.config.agentTypes)
+    Promotion.setProjectName(this.config.projectName);
+    Promotion.setValidDeploymentTargets(this.config.deploymentTargets);
+    Secrets.setValidSecretNames(this.config.orgSecretNames, this.config.projectSecretNames);
+    Agent.setValidAgentTypes(this.config.agentTypes);
 
-    this.setUpModelComponentEventLoop()
+    if(Features.isEnabled("useFetchingJob")) {
+      this.registerLeavePageHandler(); 
+      
+      this.disableButtonsInHeader();
 
-    this.registerLeavePageHandler()
+      this.toggleWorkflowEditor();
 
-    this.preselectFirstBlock()
+      this.toggleZeroState();
+
+      this.waitForFetchingJob();
+    } else {
+      this.setUpModelComponentEventLoop();
+
+      this.registerLeavePageHandler();
+  
+      this.preselectFirstBlock();
+    }
+  }
+
+  disableButtonsInHeader() {
+    $("[data-action=editorDismiss]").prop("disabled", true);
+    $("[data-action=toggleCommitDialog]").prop("disabled", true);
+  }
+
+  enableButtonsInHeader() {
+    $("[data-action=editorDismiss]").prop("disabled", false);
+    $("[data-action=toggleCommitDialog]").prop("disabled", false);
+  }
+
+  toggleWorkflowEditor() {
+    $("#workflow-editor-tabs").toggle();
+    $("#workflow-editor-container").toggle();
   }
 
+  toggleZeroState() {
+    $("#zero-state-loading-message").toggle();
+  }
+
+  waitForFetchingJob() {
+    let url = this.config.checkFetchingJobPath + `?job_id=${this.config.fetchingJobId}`;
+
+    console.log(`Checking fetching job ${url}`);
+
+    fetch(url)
+    .then((res) => {
+      const contentType = res.headers.get("content-type");
+
+      if(contentType && contentType.includes("application/json")) {
+        return res.json();
+      } else {
+        throw new Error(res.statusText);
+      }
+    })
+    .then((res) => {
+      if(res.error) {
+        throw new Error(res.error);
+      } else {
+        return res;
+      }
+    })
+    .then((data) => {
+      if(!data.finished) {
+        setTimeout(this.waitForFetchingJob.bind(this), 1000);
+      } else {
+        this.fetchFilesAndShowEditor(data.signed_urls);
+      }
+    })
+    .catch(
+      (reason) => { 
+        console.log(reason); 
+        this.enableExitAndShowErrorMessage();
+      }
+    )
+  }
+
+  enableExitAndShowErrorMessage() {
+    this.disableOnLeaveConfirm();
+    $("[data-action=editorDismiss]").prop("disabled", false);
+
+    $("#zero-state-title").text('Searching git repository for .yml files has failed.');
+
+    const message1 = "There was an issue with searching your git repository for .yml files.";
+    $("#zero-state-paragraph-one").text(message1);
+    $("#zero-state-paragraph-one").addClass("red");
+
+    const message2 = 'Please try reloading the page and contact support if the issue persists.';
+    $("#zero-state-paragraph-two").text(message2);
+    $("#zero-state-paragraph-two").addClass("red");
+  }
+
+  fetchFilesAndShowEditor(urls) {
+    fetchFilesSequentially(urls)
+    .then(({ updatedFiles, errors }) => {
+      if (errors.length > 0) {
+        console.warn("Some files failed to fetch:", errors);
+
+        this.enableExitAndShowErrorMessage();
+      } else {
+        console.log("All .yml files were fetched successfully.");
+
+        this.config.workflowData.yamls = updatedFiles;
+
+        this.toggleZeroState();
+
+        this.toggleWorkflowEditor();
+
+        this.enableButtonsInHeader();
+        
+        this.setUpModelComponentEventLoop();
+    
+        this.preselectFirstBlock();
+      }
+    });
+  }
+  
+
   //
   // Sets up a model-component event loop.
   //
@@ -215,3 +326,26 @@ export class WorkflowEditor {
     SelectionRegister.setCurrentSelectionUid(uid)
   }
 }
+
+async function fetchFilesSequentially(fileMap) {
+  const errors = [];
+
+  for (const [filePath, signedUrl] of Object.entries(fileMap)) {
+    try {
+      const response = await fetch(signedUrl);
+      if (!response.ok) {
+        throw new Error(`Failed to fetch ${filePath}: ${response.statusText}`);
+      }
+      const content = await response.text();
+      fileMap[filePath] = content;
+    } catch (error) {
+      console.error(`Error fetching ${filePath}:`, error);
+      errors.push({ filePath, error });
+    }
+  }
+
+  return {
+    updatedFiles: fileMap,
+    errors,
+  };
+}

front/config/.credo.exs

@@ -35,7 +35,7 @@
         # or the `schema` macro in Ecto schemas to trigger DuplicatedCode, just
         # set the `excluded_macros` parameter to `[:schema, :setup, :test]`.
         #
-        {Credo.Check.Design.DuplicatedCode, excluded_macros: [:test], mass_threshold: 45},
+        {Credo.Check.Design.DuplicatedCode, excluded_macros: [:test], mass_threshold: 55},
         # You can also customize the exit_status of each check.
         # If you don't want TODO comments to cause `mix credo` to fail, just
         # set this value to 0 (zero).

front/lib/front/models/artifacthub.ex

@@ -46,12 +46,17 @@ defmodule Front.Models.Artifacthub do
     end)
   end
 
-  def list(project_id, source_kind, source_id, path \\ "") do
+  def list(project_id, source_kind, source_id, path \\ "", unwrap_directories \\ false) do
     Watchman.benchmark("artifacthub.list_request.duration", fn ->
       with req_path <- Resource.request_path(source_kind, source_id, path),
            {:ok, store_id} <- get_artifact_store_id(project_id),
            {:ok, channel} <- GRPC.Stub.connect(api_endpoint()),
-           request <- ListRequest.new(artifact_id: store_id, path: req_path),
+           request <-
+             ListRequest.new(
+               artifact_id: store_id,
+               path: req_path,
+               unwrap_directories: unwrap_directories
+             ),
            {:ok, response} <- Stub.list_path(channel, request, timeout: 30_000),
            {:ok, artifacts} <- parse_list_response(response, source_kind, source_id, req_path) do
         {:ok, artifacts}
@@ -119,6 +124,24 @@ defmodule Front.Models.Artifacthub do
     end)
   end
 
+  def list_and_sign_urls(project_id, source_kind, source_id, relative_path) do
+    case list(project_id, source_kind, source_id, relative_path, true) do
+      {:ok, artifacts} ->
+        Enum.reduce_while(artifacts, {:ok, %{}}, fn artifact, {_, urls} ->
+          case signed_url(project_id, source_kind, source_id, artifact.path) do
+            {:ok, url} ->
+              {:cont, {:ok, Map.put(urls, artifact.path, url)}}
+
+            e ->
+              {:halt, {:error, "error generating signed URL for #{artifact.path}: #{inspect(e)}"}}
+          end
+        end)
+
+      e ->
+        e
+    end
+  end
+
   def signed_url(project_id, source_kind, source_id, relative_path, method \\ "GET") do
     case get_artifact_store_id(project_id) do
       {:ok, store_id} ->

front/lib/front/models/fetching_job.ex

@@ -0,0 +1,122 @@
+defmodule Front.Models.FetchingJob do
+  @moduledoc """
+  Module encapsulates all functions needed to define the job used for fetching
+  the yaml files needed in workflow editor from the git repository.
+  """
+
+  alias Front.Models.OrganizationSettings
+  alias InternalApi.ServerFarm.Job.JobSpec
+  alias Front.Models.Job
+
+  require Logger
+
+  def start_fetching_job(params) do
+    with {:ok, agent} <- get_agent(params.project),
+         {:ok, job_spec} <- create_job_spec(agent, params),
+         {:ok, job} <- Job.create(job_spec, params) do
+      {:ok, job.id}
+    else
+      error ->
+        Logger.error(
+          Enum.join(
+            [
+              "Could not create fetching job",
+              "project: #{params.project.id}",
+              "branch: #{params.target_branch}",
+              "user: #{params.user_id}",
+              "error: #{inspect(error)}"
+            ],
+            ", "
+          )
+        )
+
+        {:error, :fetching_job_creation_failed}
+    end
+  end
+
+  defp get_agent(project) do
+    case OrganizationSettings.fetch(project.organization_id) do
+      {:ok, settings} -> decide_on_agent(settings)
+      error -> {:error, :fetch_agent, error}
+    end
+  end
+
+  defp decide_on_agent(%{"custom_machine_type" => type, "custom_os_image" => os_image})
+       when is_binary(type) and type != "" do
+    {:ok, %{type: type, os_image: os_image}}
+  end
+
+  defp decide_on_agent(%{"plan_machine_type" => type, "plan_os_image" => os_image})
+       when is_binary(type) and type != "" do
+    {:ok, %{type: type, os_image: os_image}}
+  end
+
+  defp decide_on_agent(_seetings), do: {:error, :settings_without_agent_def}
+
+  defp create_job_spec(agent, params) do
+    {:ok,
+     %JobSpec{
+       job_name: "Workflow editor fetching files * #{params.project.name} * #{params.hook.name}",
+       agent: %JobSpec.Agent{
+         machine: %JobSpec.Agent.Machine{
+           os_image: agent.os_image,
+           type: agent.type
+         },
+         containers: [],
+         image_pull_secrets: []
+       },
+       secrets: [],
+       env_vars: [],
+       files: [],
+       commands: generate_commands(params),
+       epilogue_always_commands: [],
+       epilogue_on_pass_commands: [],
+       epilogue_on_fail_commands: [],
+       priority: 95,
+       execution_time_limit: 10
+     }}
+  end
+
+  defp generate_commands(params) do
+    [
+      "export SEMAPHORE_GIT_DEPTH=5",
+      configure_env_vars_for_checkout(params),
+      "checkout",
+      "artifact push job .semaphore -d .workflow_editor/.semaphore"
+    ]
+    |> List.flatten()
+  end
+
+  defp configure_env_vars_for_checkout(params) do
+    case params.hook.type do
+      "branch" -> set_branch_env_vars(params.hook)
+      "tag" -> set_tag_env_vars(params.hook)
+      "pr" -> set_pr_env_vars(params.hook)
+      :skip -> []
+    end
+  end
+
+  defp set_branch_env_vars(hook) do
+    [
+      "export SEMAPHORE_GIT_REF_TYPE=branch",
+      "export SEMAPHORE_GIT_BRANCH=#{hook.branch_name}",
+      "export SEMAPHORE_GIT_SHA=#{hook.head_commit_sha}"
+    ]
+  end
+
+  defp set_tag_env_vars(hook) do
+    [
+      "export SEMAPHORE_GIT_REF_TYPE=tag",
+      "export SEMAPHORE_GIT_TAG_NAME=#{hook.tag_name}",
+      "export SEMAPHORE_GIT_SHA=#{hook.head_commit_sha}"
+    ]
+  end
+
+  defp set_pr_env_vars(hook) do
+    [
+      "export SEMAPHORE_GIT_REF_TYPE=pull-request",
+      "export SEMAPHORE_GIT_REF=refs/pull/#{hook.pr_number}/merge",
+      "export SEMAPHORE_GIT_SHA=#{hook.head_commit_sha}"
+    ]
+  end
+end