refactor: rework file transfer to use presigned URLs (#1255)

- In the previous code, file uploads and downloads were handled by the
backend. Now, the backend only generates presigned URLs for uploads and downloads,
and the actual file transfer will be done directly between the client
and the storage

Signed-off-by: ArnelaL <arnela.lisic@secomind.com>

Author: ArnelaL

backend/config/runtime.exs

@@ -1,7 +1,7 @@
 #
 # This file is part of Edgehog.
 #
-# Copyright 2021 - 2025 SECO Mind Srl
+# Copyright 2021 - 2026 SECO Mind Srl
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -140,6 +140,17 @@ if config_env() in [:prod, :test] do
       :prod -> edgehog_enable_s3_storage?
     end
 
+  # Parse the public/external S3 host from S3_ASSET_HOST so that presigned URL
+  # generation can embed the correct external address. The old Waffle upload path
+  # continues to use :ex_aws config
+  # (S3_HOST / S3_SCHEME / S3_PORT, i.e. the internal address) unaffected.
+  s3_presign_host_config =
+    if storage_type != "azure" do
+      asset_host = s3.asset_host || "http://localhost:9000"
+      uri = URI.parse(asset_host)
+      %{scheme: uri.scheme <> "://", host: uri.host, port: uri.port || 80}
+    end
+
   config :azurex, Azurex.Blob.Config,
     api_url: azure_api_url,
     default_container: azure.container,
@@ -149,7 +160,10 @@ if config_env() in [:prod, :test] do
   # Enable uploaders only when the S3 storage has been configured
   config :edgehog,
     enable_s3_storage?: edgehog_enable_s3_storage?,
-    max_upload_size_bytes: max_upload_size_bytes
+    max_upload_size_bytes: max_upload_size_bytes,
+    storage_type: String.to_atom(storage_type),
+    storage_bucket: waffle_bucket,
+    s3_presign_host_config: s3_presign_host_config
 
   config :ex_aws, :s3,
     scheme: s3.scheme,

backend/config/test.exs

@@ -132,8 +132,6 @@ config :edgehog, :astarte_trigger_data_layer, Edgehog.Astarte.Trigger.MockDataLa
 config :edgehog, :astarte_wifi_scan_result_module, Edgehog.Astarte.Device.WiFiScanResultMock
 config :edgehog, :base_images_storage_module, Edgehog.BaseImages.StorageMock
 config :edgehog, :container_reconciler, Edgehog.Containers.ReconcilerMock
-config :edgehog, :files_ephemeral_file_module, Edgehog.Files.EphemeralFileMock
-config :edgehog, :files_storage_module, Edgehog.Files.StorageMock
 config :edgehog, :os_management_ephemeral_image_module, Edgehog.OSManagement.EphemeralImageMock
 config :edgehog, :reconciler_module, Edgehog.Tenants.ReconcilerMock
 

backend/lib/edgehog/files/ephemeral_file/ephemeral_file.ex

@@ -18,23 +18,12 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-defmodule Edgehog.Files.File.Changes.HandleFileDeletion do
+defmodule Edgehog.Files.File.Changes.DeleteFile do
   @moduledoc """
-  Ash change to handle file deletion after a file record is deleted.
+  Deletes a file from the storage backend that was uploaded via a presigned URL.
   """
-
   use Ash.Resource.Change
 
-  alias Edgehog.Files.File.BucketStorage
-
-  require Logger
-
-  @storage_module Application.compile_env(
-                    :edgehog,
-                    :files_storage_module,
-                    BucketStorage
-                  )
-
   @impl Ash.Resource.Change
   def change(%Ash.Changeset{valid?: false} = changeset, _opts, _context), do: changeset
 
@@ -45,7 +34,13 @@ defmodule Edgehog.Files.File.Changes.HandleFileDeletion do
   end
 
   defp delete_old_file({:ok, file} = result) do
-    _ = @storage_module.delete(file)
+    tenant_id = file.tenant_id
+    repository_id = file.repository_id
+    filename = file.name
+
+    file_path = "uploads/tenants/#{tenant_id}/repositories/#{repository_id}/files/#{filename}"
+
+    _ = Edgehog.Storage.delete(file_path)
 
     result
   end

backend/lib/edgehog/files/file/bucket_storage.ex

@@ -25,6 +25,7 @@ defmodule Edgehog.Files.File do
     extensions: [AshGraphql.Resource]
 
   alias Edgehog.Files.File.Changes
+  alias Edgehog.Files.File.ManualActions
 
   graphql do
     type :file
@@ -37,20 +38,13 @@ defmodule Edgehog.Files.File do
       description "Create a new file by uploading it to the storage."
       primary? true
 
-      accept [:name, :mode, :user_id, :group_id]
+      accept [:name, :size, :digest, :mode, :user_id, :group_id, :url]
 
       argument :repository_id, :uuid do
         description "The ID of the repository this file will belong to."
         allow_nil? false
       end
 
-      argument :file, Edgehog.Types.Upload do
-        description "The file to upload, which will be stored in the bucket."
-        allow_nil? false
-      end
-
-      change Changes.HandleFileUpload
-
       change manage_relationship(:repository_id, :repository, type: :append)
     end
 
@@ -65,16 +59,28 @@ defmodule Edgehog.Files.File do
     end
 
     destroy :destroy do
-      description "Deletes a file and its stored content."
+      description "Deletes a file"
       primary? true
-
-      # Needed because HandleFileDeletion is not atomic
       require_atomic? false
 
-      change Changes.HandleFileDeletion
+      change Changes.DeleteFile
     end
 
     destroy :destroy_fixture
+
+    action :create_presigned_url, :map do
+      argument :filename, :string, allow_nil?: false
+      argument :repository_id, :uuid, allow_nil?: false
+
+      run ManualActions.CreatePresignedUrl
+    end
+
+    action :read_presigned_url, :map do
+      argument :filename, :string, allow_nil?: false
+      argument :repository_id, :uuid, allow_nil?: false
+
+      run ManualActions.ReadPresignedUrl
+    end
   end
 
   attributes do
@@ -137,6 +143,10 @@ defmodule Edgehog.Files.File do
     end
   end
 
+  identities do
+    identity :unique_repository_file, [:name, :repository_id]
+  end
+
   postgres do
     table "files"
     repo Edgehog.Repo