Introducing Git file storage (#3056)

Author: aleDsz

.github/workflows/test.yml

@@ -41,6 +41,8 @@ jobs:
         run: mix compile --warnings-as-errors
       - name: Run tests
         run: mix test
+        env:
+          TEST_GIT_SSH_KEY: ${{ secrets.TEST_GIT_SSH_KEY }}
       - name: Install Node
         uses: actions/setup-node@v4
         with:
@@ -105,6 +107,8 @@ jobs:
         run: mix deps.get
       - name: Run tests
         run: mix test
+        env:
+          TEST_GIT_SSH_KEY: ${{ secrets.TEST_GIT_SSH_KEY }}
       - name: Build the app
         run: bash .github/scripts/app/build_windows.sh
 

Dockerfile

@@ -82,6 +82,8 @@ RUN apt-get update && apt-get upgrade -y && \
     build-essential ca-certificates libncurses5-dev \
     # In case someone uses `Mix.install/2` and point to a git repo
     git \
+    # In case someone uses the Git file storage
+    openssh-client \
     # Additional standard tools
     wget \
     # In case someone uses Torchx for Nx

lib/livebook/application.ex

@@ -58,6 +58,10 @@ defmodule Livebook.Application do
           {DynamicSupervisor, name: Livebook.HubsSupervisor, strategy: :one_for_one},
           # Run startup logic relying on the supervision tree
           {Livebook.Utils.SupervisionStep, {:boot, boot(create_teams_hub)}},
+          # Start the server responsible for initializing
+          # mountable file systems. We do it after boot, because
+          # file systems and this depends on hubs being started
+          Livebook.FileSystem.Mounter,
           # App manager supervision tree. We do it after boot, because
           # permanent apps are going to be started right away and this
           # depends on hubs being started

lib/livebook/file_system.ex

@@ -246,4 +246,16 @@ defprotocol Livebook.FileSystem do
   """
   @spec external_metadata(t()) :: %{name: String.t(), error_field: String.t()}
   def external_metadata(file_system)
+
+  @doc """
+  Mounts the given file system to be used furthermore.
+  """
+  @spec mount(t()) :: :ok | {:error, error()}
+  def mount(file_system)
+
+  @doc """
+  Unmounts the given file system from being used.
+  """
+  @spec unmount(t()) :: :ok | {:error, error()}
+  def unmount(file_system)
 end

lib/livebook/file_system/git.ex

@@ -0,0 +1,192 @@
+defmodule Livebook.FileSystem.Git do
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  alias Livebook.FileSystem
+
+  # File system backed by an Git repository.
+
+  @type t :: %__MODULE__{
+          id: String.t(),
+          repo_url: String.t(),
+          branch: String.t(),
+          key: String.t(),
+          external_id: String.t() | nil,
+          hub_id: String.t()
+        }
+
+  embedded_schema do
+    field :repo_url, :string
+    field :branch, :string
+    field :key, :string, redact: true
+    field :external_id, :string
+    field :hub_id, :string
+  end
+
+  @doc """
+  Returns an `%Ecto.Changeset{}` for tracking file system changes.
+  """
+  @spec change_file_system(t(), map()) :: Ecto.Changeset.t()
+  def change_file_system(git, attrs \\ %{}) do
+    changeset(git, attrs)
+  end
+
+  defp changeset(git, attrs) do
+    git
+    |> cast(attrs, [:repo_url, :branch, :key, :external_id, :hub_id])
+    |> validate_format(:repo_url, ~r/^git@[\w\.\-]+:(?:v\d+\/)?[\w\.\-\/]+\.git$/,
+      message: "must be a valid repo URL"
+    )
+    |> validate_required([:repo_url, :branch, :key, :hub_id])
+    |> update_change(:key, fn key ->
+      if key =~ "\n" do
+        key
+      else
+        String.replace(
+          key,
+          ~r/ (?!(?:OPENSSH |RSA |EC |DSA |PRIVATE )?(?:PRIVATE )?KEY-----)/,
+          "\n"
+        )
+      end
+    end)
+    |> put_id()
+  end
+
+  defp put_id(changeset) do
+    hub_id = get_field(changeset, :hub_id)
+    repo_url = get_field(changeset, :repo_url)
+
+    if get_field(changeset, :id) do
+      changeset
+    else
+      put_change(changeset, :id, FileSystem.Utils.id("git", hub_id, repo_url))
+    end
+  end
+
+  @doc false
+  def git_dir(%__MODULE__{id: id}), do: Path.join(Livebook.Config.tmp_path(), id)
+
+  @doc false
+  def key_path(%__MODULE__{id: id}), do: Path.join(Livebook.Config.tmp_path(), "#{id}_key")
+end
+
+defimpl Livebook.FileSystem, for: Livebook.FileSystem.Git do
+  alias Livebook.FileSystem
+  alias Livebook.FileSystem.Git
+
+  def type(_file_system) do
+    :global
+  end
+
+  def default_path(_file_system) do
+    "/"
+  end
+
+  def list(file_system, path, _recursive) do
+    with {:ok, []} <- Git.Client.list_files(file_system, path) do
+      FileSystem.Utils.posix_error(:enoent)
+    end
+  end
+
+  def read(file_system, path) do
+    Git.Client.read_file(file_system, path)
+  end
+
+  def write(_file_system, _path, _content), do: raise("not implemented")
+
+  def access(_file_system, _path) do
+    {:ok, :read}
+  end
+
+  def create_dir(_file_system, _path), do: raise("not implemented")
+
+  def remove(_file_system, _path), do: raise("not implemented")
+
+  def copy(_file_system, _source_path, _destination_path), do: raise("not implemented")
+
+  def rename(_file_system, _source_path, _destination_path), do: raise("not implemented")
+
+  def etag_for(file_system, path) do
+    FileSystem.Utils.assert_regular_path!(path)
+    Git.Client.etag(file_system, path)
+  end
+
+  def exists?(file_system, path) do
+    Git.Client.exists?(file_system, path)
+  end
+
+  def resolve_path(_file_system, dir_path, subject) do
+    FileSystem.Utils.resolve_unix_like_path(dir_path, subject)
+  end
+
+  def write_stream_init(_file_system, _path, _opts), do: raise("not implemented")
+
+  def write_stream_chunk(_file_system, _state, _chunk), do: raise("not implemented")
+
+  def write_stream_finish(_file_system, _state), do: raise("not implemented")
+
+  def write_stream_halt(_file_system, _state), do: raise("not implemented")
+
+  def read_stream_into(_file_system, _path, _collectable), do: raise("not implemented")
+
+  def load(file_system, %{"hub_id" => _} = fields) do
+    load(file_system, %{
+      id: fields["id"],
+      repo_url: fields["repo_url"],
+      branch: fields["branch"],
+      key: fields["key"],
+      external_id: fields["external_id"],
+      hub_id: fields["hub_id"]
+    })
+  end
+
+  def load(file_system, fields) do
+    %{
+      file_system
+      | id: fields.id,
+        repo_url: fields.repo_url,
+        branch: fields.branch,
+        key: fields.key,
+        external_id: fields.external_id,
+        hub_id: fields.hub_id
+    }
+  end
+
+  def dump(file_system) do
+    file_system
+    |> Map.from_struct()
+    |> Map.take([:id, :repo_url, :branch, :key, :external_id, :hub_id])
+  end
+
+  def external_metadata(file_system) do
+    %{name: file_system.repo_url, error_field: "repo_url"}
+  end
+
+  def mount(file_system) do
+    if mounted?(file_system) do
+      FileSystem.Git.Client.fetch(file_system)
+    else
+      FileSystem.Git.Client.init(file_system)
+    end
+  end
+
+  def unmount(file_system) do
+    path = FileSystem.Git.git_dir(file_system)
+    key_path = FileSystem.Git.key_path(file_system)
+
+    if File.exists?(key_path) do
+      File.rm!(key_path)
+    end
+
+    case File.rm_rf(path) do
+      {:ok, _} -> :ok
+      {:error, reason, _file} -> FileSystem.Utils.posix_error(reason)
+    end
+  end
+
+  defp mounted?(file_system) do
+    file_system
+    |> FileSystem.Git.git_dir()
+    |> File.exists?()
+  end
+end