feat(front): start a job to commit changes from workflow editor (#155)

Introduces an option to commit changes made in the workflow editor
through a Semaphore job that will use git CLI for these actions. The
previous approach relies on libgit2 and API actions, which is 
unstable for large git repositories.

Author: DamjanBecirovic

front/assets/js/workflow_editor/components/commit_panel.js

@@ -1,6 +1,7 @@
 import $ from "jquery";
 
 import { CommitDialogTemplate } from "../templates/commit/dialog"
+import { Features } from "../../features";
 
 function newDialogDiv() {
   return $("<div style='display=none'>")[0]
@@ -79,7 +80,7 @@ export class CommitPanel {
 
   _commit(e) {
     let branch = null;
-    let message = null;
+    let commitMessage = null;
 
     var url = this.paths.commit;
 
@@ -88,15 +89,21 @@ export class CommitPanel {
     $("[data-action=editorCommit]").addClass("btn-working");
     $("[data-action=editorCommit]").disabled = true;
 
+    var message = 'We are committing the changes to your git repository. ';
+    message = message + 'This can take up to a few minutes.';
+    $("#workflow-editor-commit-dialog-note").text(message);
+    $("#workflow-editor-commit-dialog-note").addClass("dark-indigo");
+
     let csrf = $("meta[name='csrf-token']").attr("content")
 
     branch = $("#workflow-editor-commit-dialog-branch").val()
-    message = $("#workflow-editor-commit-dialog-summary").val()
+    commitMessage = $("#workflow-editor-commit-dialog-summary").val()
 
     var body = new FormData();
     body.append("_csrf_token", csrf)
     body.append("branch", branch)
-    body.append("commit_message", message)
+    body.append("commit_message", commitMessage)
+    body.append("initial_branch", this.initialBranch)
 
     // if this is part of project onboarding, we need to also make sure 
     // that the project onboarding finished signal is sent
@@ -146,17 +153,73 @@ export class CommitPanel {
       console.log(data)
 
       if (data.wait)
-        this.afterCommitHandler(branch, data.commit_sha);
+        if(Features.isEnabled("useCommitJob"))          
+          this.commitJobHandler(branch, data.job_id);
+        else {
+          this.afterCommitHandler(branch, data.commit_sha);
+        }
       else {
         this.dialog.innerHTML = CommitDialogTemplate.renderCommited(this.paths.dismiss);
       }
     })
     .catch(function(reason) {
       console.log(reason)
 
-      e.currentTarget.classList.remove("btn-working");
-      e.currentTarget.disabled = false;
+      this.resetButtonAndShowErrorMessage();
+    })
+  }
+
+  resetButtonAndShowErrorMessage() {
+    $("[data-action=editorCommit]").removeClass("btn-working");
+    $("[data-action=editorCommit]").disabled = false;
+
+    var message = "There was an issue with committing the changes to your git repository. "
+    message = message + 'Please try again and contact support if the issue persists.';
+    $("#workflow-editor-commit-dialog-note").text(message);
+    $("#workflow-editor-commit-dialog-note").removeClass("dark-indigo");
+    $("#workflow-editor-commit-dialog-note").addClass("red");
+  }
+
+  commitJobHandler(branch, job_id) {
+    var url   = this.paths.checkCommitJob + `?job_id=${job_id}` 
+
+    console.log(`Checking commit job ${url}`)
+
+    fetch(url)
+    .then((res) => {
+      var contentType = res.headers.get("content-type");
+
+      if(contentType && contentType.includes("application/json")) {
+        return res.json();
+      } else {
+        throw new Error(res.statusText);
+      }
+    })
+    .then((res) => {
+      if(res.error) {
+        throw new Error(res.error);
+      } else {
+        return res;
+      }
+    })
+    .then((data) => {
+      if(data.commit_sha === "") {
+        setTimeout(this.commitJobHandler.bind(this, branch, job_id), 1000)
+      } else {
+        var message = 'The changes have been successfully committed. ';
+        message = message + 'We will soon navigate you to the new workflow.';
+        $("#workflow-editor-commit-dialog-note").text(message);
+
+        this.afterCommitHandler(branch, data.commit_sha);
+      }
     })
+    .catch(
+      (reason) => { 
+        console.log(reason); 
+
+        this.resetButtonAndShowErrorMessage();
+      }
+    )
   }
 
   afterCommitHandler(branch, commitSha) {

front/assets/js/workflow_editor/editor.js

@@ -37,7 +37,8 @@ export class WorkflowEditor {
         paths: {
           dismiss: InjectedDataByBackend.CommitForm.DismissPath,
           commit: InjectedDataByBackend.CommitForm.CommitPath,
-          checkWorkflow: InjectedDataByBackend.CommitForm.CheckWorkflowPath
+          checkWorkflow: InjectedDataByBackend.CommitForm.CheckWorkflowPath,
+          checkCommitJob: InjectedDataByBackend.CommitForm.CheckCommitJobPath
         },
         pushBranch: InjectedDataByBackend.CommitForm.PushBranch,
         initialBranch: InjectedDataByBackend.CommitForm.InitialBranch,
@@ -56,6 +57,7 @@ export class WorkflowEditor {
     assertKey(config.commitInfo.paths, "dismiss", "Dismiss path is not configured")
     assertKey(config.commitInfo.paths, "commit",  "Commit path is not configured")
     assertKey(config.commitInfo.paths, "checkWorkflow", "Check Workflow path is not configured")
+    assertKey(config.commitInfo.paths, "checkCommitJob", "Check Job path is not configured")
 
     //
     // If everything is cocher, start the editor.

front/assets/js/workflow_editor/templates/commit/dialog.js

@@ -232,7 +232,7 @@ function renderCommitPanel(workflow, commiterAvatar, initialBranch, pushBranch)
     </a>
   </div>`
 
-  let commitNote = `<p class="f6 measure-narrow mb1">
+  let commitNote = `<p class="f6 w-100 w-two-thirds-m mb1" id=workflow-editor-commit-dialog-note>
     This will commit and push the configuration to repository and trigger the run on Semaphore.
   </p>`
 

front/lib/front/models/artifacthub.ex

@@ -89,10 +89,21 @@ defmodule Front.Models.Artifacthub do
     end)
   end
 
-  def signed_url(project_id, source_kind, source_id, relative_path, method \\ "GET") do
+  def fetch_file(store_id, source_kind, source_id, relative_path) do
+    with {:ok, url} <- signed_url_with_store_id(store_id, source_kind, source_id, relative_path),
+         {:ok, response} <- HTTPoison.get(url),
+         %{status_code: 200, body: content} <- response do
+      {:ok, content}
+    else
+      %{status_code: 404, body: error} -> {:error, {:not_found, error}}
+      error = {:error, _e} -> error
+      error -> {:error, error}
+    end
+  end
+
+  def signed_url_with_store_id(store_id, source_kind, source_id, relative_path, method \\ "GET") do
     Watchman.benchmark("artifacthub.get_signed_url_request.duration", fn ->
       with req_path <- Resource.request_path(source_kind, source_id, relative_path),
-           {:ok, store_id} <- get_artifact_store_id(project_id),
            {:ok, channel} <- GRPC.Stub.connect(api_endpoint()),
            request <-
              GetSignedURLRequest.new(artifact_id: store_id, path: req_path, method: method),
@@ -108,6 +119,19 @@ defmodule Front.Models.Artifacthub do
     end)
   end
 
+  def signed_url(project_id, source_kind, source_id, relative_path, method \\ "GET") do
+    case get_artifact_store_id(project_id) do
+      {:ok, store_id} ->
+        signed_url_with_store_id(store_id, source_kind, source_id, relative_path, method)
+
+      e ->
+        Watchman.increment("artifacthub.get_signed_url.failed")
+        Logger.error("Failed to get url: #{inspect(e)}")
+
+        {:error, :grpc_req_failed}
+    end
+  end
+
   defp parse_list_response(response, source_kind, source_id, path) do
     if Enum.empty?(response.items) && not Resource.root_path?(source_kind, source_id, path) do
       {:error, :non_existent_path}

front/lib/front/models/commit_job.ex

@@ -0,0 +1,176 @@
+defmodule Front.Models.CommitJob do
+  @moduledoc """
+  Module encapsulates all functions needed to define the job used for commiting
+  the changes made in workflow editor.
+  """
+
+  require Logger
+
+  alias Front.Models.User
+  alias Front.Models.OrganizationSettings
+  alias Front.Models.RepositoryIntegrator
+  alias InternalApi.ServerFarm.Job.JobSpec
+  alias InternalApi.Repository.CommitRequest.Change.Action
+
+  def get_agent(project) do
+    case OrganizationSettings.fetch(project.organization_id) do
+      {:ok, settings} -> decide_on_agent(settings)
+      error -> {:error, :fetch_agent, error}
+    end
+  end
+
+  defp decide_on_agent(%{"custom_machine_type" => type, "custom_os_image" => os_image})
+       when is_binary(type) and type != "" do
+    {:ok, %{type: type, os_image: os_image}}
+  end
+
+  defp decide_on_agent(%{"plan_machine_type" => type, "plan_os_image" => os_image})
+       when is_binary(type) and type != "" do
+    {:ok, %{type: type, os_image: os_image}}
+  end
+
+  defp decide_on_agent(_seetings), do: {:error, :settings_without_agent_def}
+
+  def get_git_credentials(project, user_id) do
+    case project.integration_type do
+      :GITHUB_OAUTH_TOKEN ->
+        get_creds_from_repository_integrator(project, user_id, "x-oauth-token")
+
+      :GITHUB_APP ->
+        get_creds_from_repository_integrator(project, user_id, "x-access-token")
+
+      :BITBUCKET ->
+        get_creds_from_user_service(project, user_id, "x-token-auth")
+
+      :GITLAB ->
+        get_creds_from_user_service(project, user_id, "oauth2")
+    end
+  end
+
+  defp get_creds_from_repository_integrator(project, user_id, git_username) do
+    case RepositoryIntegrator.get_repository_token(project, user_id) do
+      {:ok, token} -> {:ok, %{username: git_username, token: token}}
+      error -> {:error, :token_from_repo_integrator, error}
+    end
+  end
+
+  defp get_creds_from_user_service(project, user_id, git_username) do
+    case User.get_repository_token(project, user_id) do
+      {:ok, token} -> {:ok, %{username: git_username, token: token}}
+      error -> {:error, :token_from_user_svc, error}
+    end
+  end
+
+  def create_job_spec(agent, creds, params) do
+    {:ok,
+     %JobSpec{
+       job_name: "Commiting changes from workflow editor to branch #{params.target_branch}",
+       agent: %JobSpec.Agent{
+         machine: %JobSpec.Agent.Machine{
+           os_image: agent.os_image,
+           type: agent.type
+         },
+         containers: [],
+         image_pull_secrets: []
+       },
+       secrets: [],
+       env_vars: [],
+       files: files_for_creds(creds) ++ modified_files(params.changes),
+       commands: generate_commands(params),
+       epilogue_always_commands: [],
+       epilogue_on_pass_commands: [],
+       epilogue_on_fail_commands: [],
+       priority: 95,
+       execution_time_limit: 10
+     }}
+  end
+
+  defp files_for_creds(creds) do
+    [
+      %JobSpec.File{
+        path: ".workflow_editor/git_username.txt",
+        content: Base.encode64(creds.username)
+      },
+      %JobSpec.File{
+        path: ".workflow_editor/git_password.txt",
+        content: Base.encode64(creds.token)
+      }
+    ]
+  end
+
+  defp modified_files(changes) do
+    Enum.reduce(changes, [], fn change, acc ->
+      if change.action != Action.value(:DELETE_FILE) do
+        file = %JobSpec.File{
+          path: ".changed_files/#{change.file.path}",
+          content: Base.encode64(change.file.content)
+        }
+
+        acc ++ [file]
+      else
+        acc
+      end
+    end)
+  end
+
+  defp generate_commands(params) do
+    [
+      # Read git credentials from files
+      "export GIT_USERNAME=$(cat .workflow_editor/git_username.txt)",
+      "export GIT_PASSWORD=$(cat .workflow_editor/git_password.txt)",
+      # Prepare URL used for pushing with temporary user access token as a credential
+      "export GIT_REPO_URL=\"${SEMAPHORE_GIT_URL/://}\"",
+      "export GIT_REPO_URL=\"${GIT_REPO_URL/git@/https:\/\/$GIT_USERNAME:$GIT_PASSWORD@}\"",
+      # Configure the branch that checkout should use
+      "export SEMAPHORE_GIT_BRANCH=#{params.initial_branch}",
+      # Clone repository using the read-only deploy key
+      "checkout",
+      # switch to a new branch if user configured a different target branch
+      configure_target_branch(params.initial_branch, params.target_branch),
+      # Add modified files and remove deleted ones
+      add_commands_for_modified_files(params),
+      add_commands_for_deleted_files(params),
+      # Configure the user to be author of the changes
+      "git config --global user.name #{params.user.name}",
+      "git config --global user.email #{params.user.email}",
+      # Create commit with changes
+      "git add .",
+      "git commit -m \"#{params.commit_message}\"",
+      # Push commit to remote repository
+      "git push $GIT_REPO_URL HEAD",
+      # save the new commit sha in artifacts
+      "git rev-parse HEAD > commit_sha.val",
+      "artifact push job commit_sha.val -d .workflow_editor/commit_sha.val"
+    ]
+    |> List.flatten()
+    |> Enum.filter(fn elem -> elem != :skip end)
+  end
+
+  defp configure_target_branch(initial, target) do
+    if initial == target do
+      :skip
+    else
+      "git checkout -b #{target}"
+    end
+  end
+
+  defp add_commands_for_modified_files(params) do
+    Enum.reduce(params.changes, [], fn change, acc ->
+      if change.action != Action.value(:DELETE_FILE) do
+        acc ++ ["mv ../.changed_files/#{change.file.path} ./#{change.file.path}"]
+      else
+        acc
+      end
+    end)
+  end
+
+  defp add_commands_for_deleted_files(params) do
+    Enum.reduce(params.changes, [], fn change, acc ->
+      if change.action == Action.value(:DELETE_FILE) do
+        acc ++ ["rm #{change.file.path} || true"]
+      else
+        acc
+      end
+    end)
+  end
+end