Merge branch 'main' into feature/enforce-credo-code-quality

Author: taras-tyshko

.cspell.json

@@ -11,7 +11,7 @@
     }
   ],
   "dictionaries": [
-    "cpp", 
+    "cpp",
     "elixir",
     "en-gb",
     "fhunleth-cspell-dictionary",
@@ -20,10 +20,13 @@
   ],
   "ignorePaths": [
     "**/*.csv",
+    "docker/*",
+    "Dockerfile",
     "config",
     "assets/{node_modules,static}",
     "mix.exs",
     "priv",
+    "rel",
     "test"
   ],
   "words": [

Dockerfile

@@ -8,9 +8,9 @@ ARG RUNNER_IMAGE="ubuntu:${DISTRO}"
 ARG DEBIAN_FRONTEND=noninteractive
 
 ###
-### First Stage - Fetch deps for building web assets
+### Fetch deps for building web assets
 ###
-FROM ${BUILDER_IMAGE} as deps
+FROM ${BUILDER_IMAGE} AS deps
 
 ENV MIX_ENV=prod
 
@@ -24,9 +24,9 @@ RUN mix deps.get --only $MIX_ENV
 
 
 ###
-### Second Stage - Build web assets
+### Build web assets
 ###
-FROM node:${NODE_VERSION} as assets
+FROM node:${NODE_VERSION} AS assets
 
 RUN mkdir -p /priv/static
 
@@ -42,13 +42,14 @@ RUN npm ci && npm cache clean --force && npm run deploy
 
 
 ###
-### Third Stage - Building the Release
+### Build the NervesHub release
 ###
-FROM ${BUILDER_IMAGE} as build
+FROM ${BUILDER_IMAGE} AS build
 
 # install dependencies
-RUN apt-get update -y && apt-get install -y build-essential git ca-certificates curl gnupg \
-    && apt-get clean && rm -f /var/lib/apt/lists/*_*
+RUN apt-get update -y && apt-get install -y build-essential git ca-certificates curl gnupg && \
+    apt-get clean && \
+    rm -f /var/lib/apt/lists/*_*
 
 WORKDIR /build
 
@@ -95,25 +96,83 @@ COPY rel rel
 RUN mix release
 
 
+###
+### Build a static FWUP
+###
+
+FROM ${RUNNER_IMAGE} AS fwup
+
+RUN apt-get update -y && \
+    apt-get upgrade -y && \
+    apt-get install -y git curl build-essential autoconf pkg-config libtool mtools unzip zip help2man libconfuse-dev libarchive-dev xdelta3 dosfstools
+
+RUN git clone https://github.com/fwup-home/fwup /tmp/fwup
+
+WORKDIR /tmp/fwup
+
+RUN git checkout v1.13.2 && \
+    ./scripts/download_deps.sh && \
+    ./scripts/build_deps.sh && \
+    ./autogen.sh && \
+    PKG_CONFIG_PATH=$PWD/build/host/deps/usr/lib/pkgconfig ./configure --enable-shared=no && \
+    make && \
+    make check && \
+    make install
+
+
+###
+### Build jemalloc - GCC 14
+###
+
+FROM ${RUNNER_IMAGE} AS jemalloc
+
+RUN apt-get update -y && \
+    apt-get upgrade -y && \
+    apt-get install -y git autoconf cmake make software-properties-common && \
+    add-apt-repository ppa:ubuntu-toolchain-r/ppa -y && \
+    apt-get update -y && \
+    apt-get install -y gcc-14 g++-14 && \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 14 --slave /usr/bin/g++ g++ /usr/bin/g++-14
+
+# Build the latest jemalloc
+
+RUN git clone https://github.com/facebook/jemalloc /tmp/jemalloc
+
+WORKDIR /tmp/jemalloc
+
+RUN autoconf && \
+    ./configure && \
+    make && \
+    make install
+
+
 ###
 ### Last Stage - Setup the Runtime Environment
 ###
 
 FROM ${RUNNER_IMAGE} AS app
 
-RUN apt-get update -y \
-    && apt-get install -y libstdc++6 openssl libncurses6 locales bash openssl curl python3 python3-pip jq xdelta3 zip unzip wget \
-    && wget https://github.com/fwup-home/fwup/releases/download/v1.13.2/fwup_1.13.2_amd64.deb \
-    && dpkg -i fwup_1.13.2_amd64.deb && rm fwup_1.13.2_amd64.deb \
-    && apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN apt-get update -y && \
+    apt-get install -y openssl ca-certificates locales bash xdelta3 && \
+    apt-get autoremove -y && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
 # Set the locale
 RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
 
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
+ENV LANG=en_US.UTF-8
+ENV LANGUAGE=en_US:en
+ENV LC_ALL=en_US.UTF-8
+
+# Use jemalloc for memory allocation
+COPY --from=jemalloc /usr/local/lib/libjemalloc.so.2 /usr/local/lib/libjemalloc.so.2
+ENV LD_PRELOAD=/usr/local/lib/libjemalloc.so.2
 
+# Copy over the statically built fwup
+COPY --from=fwup /usr/local/bin/fwup /usr/local/bin/fwup
+
+# Copy over NervesHub
 WORKDIR /app
 
 COPY --from=build /build/_build/prod/rel/nerves_hub ./
@@ -124,4 +183,5 @@ ENV SECRET_KEY_BASE=nokey
 ENV PORT=4000
 
 ENTRYPOINT ["bin/nerves_hub"]
+
 CMD ["start"]

lib/nerves_hub/firmwares.ex

@@ -421,55 +421,58 @@ defmodule NervesHub.Firmwares do
     {:ok, source_url} = firmware_upload_config().download_file(source_firmware)
     {:ok, target_url} = firmware_upload_config().download_file(target_firmware)
 
-    case update_tool().create_firmware_delta_file(source_url, target_url) do
+    case update_tool().create_firmware_delta_file(
+           {source_firmware.uuid, source_url},
+           {target_firmware.uuid, target_url}
+         ) do
       {:ok, created} ->
-        firmware_delta_filename = Path.basename(created.filepath)
-
-        result =
-          Repo.transaction(
-            fn ->
-              with upload_metadata <-
-                     firmware_upload_config().metadata(org.id, firmware_delta_filename),
-                   {:ok, firmware_delta} <-
-                     complete_firmware_delta(
-                       firmware_delta,
-                       created.tool,
-                       created.size,
-                       created.source_size,
-                       created.target_size,
-                       created.tool_metadata,
-                       upload_metadata
-                     ),
-                   {:ok, firmware_delta} <- get_firmware_delta(firmware_delta.id),
-                   :ok <-
-                     firmware_upload_config().upload_file(created.filepath, upload_metadata),
-                   :ok <- update_tool().cleanup_firmware_delta_files(created.filepath) do
-                Logger.info(
-                  "Created firmware delta successfully.",
+        Repo.transact(
+          fn ->
+            with upload_metadata <-
+                   firmware_upload_config().delta_metadata(
+                     org.id,
+                     source_firmware.uuid,
+                     target_firmware.uuid
+                   ),
+                 {:ok, firmware_delta} <-
+                   complete_firmware_delta(
+                     firmware_delta,
+                     created.tool,
+                     created.size,
+                     created.source_size,
+                     created.target_size,
+                     created.tool_metadata,
+                     upload_metadata
+                   ),
+                 {:ok, firmware_delta} <- get_firmware_delta(firmware_delta.id),
+                 :ok <-
+                   firmware_upload_config().upload_file(created.filepath, upload_metadata),
+                 :ok <- update_tool().cleanup_firmware_delta_files(created.filepath) do
+              Logger.info(
+                "Created firmware delta successfully.",
+                product_id: source_firmware.product_id,
+                source_firmware: source_firmware.uuid,
+                target_firmware: target_firmware.uuid
+              )
+
+              {:ok, firmware_delta}
+            else
+              {:error, error} ->
+                update_tool().cleanup_firmware_delta_files(created.filepath)
+
+                Logger.error(
+                  "Failed to create firmware delta: #{inspect(error)}",
                   product_id: source_firmware.product_id,
                   source_firmware: source_firmware.uuid,
                   target_firmware: target_firmware.uuid
                 )
 
-                firmware_delta
-              else
-                {:error, error} ->
-                  update_tool().cleanup_firmware_delta_files(created.filepath)
-
-                  Logger.error(
-                    "Failed to create firmware delta: #{inspect(error)}",
-                    product_id: source_firmware.product_id,
-                    source_firmware: source_firmware.uuid,
-                    target_firmware: target_firmware.uuid
-                  )
-
-                  Repo.rollback(error)
-              end
-            end,
-            timeout: 30_000
-          )
-
-        case result do
+                {:error, error}
+            end
+          end,
+          timeout: 30_000
+        )
+        |> case do
           {:ok, _delta} ->
             :ok
 

lib/nerves_hub/firmwares/update_tool.ex

@@ -89,7 +89,10 @@ defmodule NervesHub.Firmwares.UpdateTool do
   @doc """
   Called to create a firmware delta file on the local filesystem
   """
-  @callback create_firmware_delta_file(String.t(), String.t()) ::
+  @callback create_firmware_delta_file(
+              {source_id :: String.t(), source_url :: String.t()},
+              {target_id :: String.t(), target_url :: String.t()}
+            ) ::
               {:ok, delta_created()} | {:error, term()}
 
   @doc """

lib/nerves_hub/firmwares/update_tool/fwup.ex

@@ -45,9 +45,8 @@ defmodule NervesHub.Firmwares.UpdateTool.Fwup do
   end
 
   @impl NervesHub.Firmwares.UpdateTool
-  def create_firmware_delta_file(source_url, target_url) do
-    uuid = Ecto.UUID.generate()
-    work_dir = Path.join(System.tmp_dir(), uuid) |> Path.expand()
+  def create_firmware_delta_file({source_uuid, source_url}, {target_uuid, target_url}) do
+    work_dir = Path.join(System.tmp_dir(), "#{source_uuid}_#{target_uuid}")
     _ = File.mkdir_p(work_dir)
 
     try do
@@ -57,7 +56,7 @@ defmodule NervesHub.Firmwares.UpdateTool.Fwup do
       dl!(source_url, source_path)
       dl!(target_url, target_path)
 
-      case do_delta_file(source_path, target_path, work_dir) do
+      case do_delta_file({source_uuid, source_path}, {target_uuid, target_path}, work_dir) do
         {:ok, output} ->
           {:ok, output}
 
@@ -139,7 +138,7 @@ defmodule NervesHub.Firmwares.UpdateTool.Fwup do
     end)
   end
 
-  def do_delta_file(source_path, target_path, work_dir) do
+  def do_delta_file({source_uuid, source_path}, {target_uuid, target_path}, work_dir) do
     source_work_dir = Path.join(work_dir, "source")
     target_work_dir = Path.join(work_dir, "target")
     output_work_dir = Path.join(work_dir, "output")
@@ -156,7 +155,7 @@ defmodule NervesHub.Firmwares.UpdateTool.Fwup do
          {:ok, target_meta_conf} <- File.read(Path.join(target_work_dir, "meta.conf")),
          {:ok, tool_metadata} <- get_tool_metadata(Path.join(target_work_dir, "meta.conf")),
          :ok <- Confuse.Fwup.validate_delta(source_meta_conf, target_meta_conf),
-         {:ok, deltas} <- Confuse.Fwup.get_delta_files(Path.join(target_work_dir, "meta.conf")),
+         {:ok, deltas} <- Confuse.Fwup.get_delta_files_from_config(target_meta_conf),
          {:ok, all_delta_files} <- delta_files(deltas) do
       Logger.info("Generating delta for files: #{Enum.join(all_delta_files, ", ")}")
 
@@ -210,7 +209,7 @@ defmodule NervesHub.Firmwares.UpdateTool.Fwup do
             end
         end
 
-      {:ok, delta_zip_path} = Plug.Upload.random_file("generated_delta_zip_file")
+      {:ok, delta_zip_path} = Plug.Upload.random_file("#{source_uuid}_#{target_uuid}_delta")
 
       {:ok, _} =
         :zip.create(to_charlist(delta_zip_path), generate_file_list(output_work_dir),

lib/nerves_hub/firmwares/upload.ex

@@ -33,4 +33,13 @@ defmodule NervesHub.Firmwares.Upload do
   Called to generate the upload_metadata that will be persisted for later lookup.
   """
   @callback metadata(Org.id(), String.t()) :: upload_metadata()
+
+  @doc """
+  Called to generate the upload_metadata specific to delta firmwares that will be persisted for later lookup.
+  """
+  @callback delta_metadata(
+              Org.id(),
+              source_firmware_uuid :: String.t(),
+              target_firmware_uuid :: String.t()
+            ) :: upload_metadata()
 end

lib/nerves_hub/firmwares/upload/file.ex

@@ -35,8 +35,9 @@ defmodule NervesHub.Firmwares.Upload.File do
 
     config = Application.get_env(:nerves_hub, __MODULE__)
 
-    common_path = "#{org_id}"
-    local_path = Path.join([config[:local_path], common_path, filename])
+    common_path = Path.join([key_prefix(), Integer.to_string(org_id), filename])
+
+    local_path = Path.join([config[:local_path], common_path])
 
     port =
       if Enum.member?([443, 80], web_config[:url][:port]),
@@ -53,4 +54,42 @@ defmodule NervesHub.Firmwares.Upload.File do
       local_path: local_path
     }
   end
+
+  @impl NervesHub.Firmwares.Upload
+  def delta_metadata(org_id, source_firmware_uuid, target_firmware_uuid) do
+    web_config = Application.get_env(:nerves_hub, NervesHubWeb.Endpoint)
+
+    config = Application.get_env(:nerves_hub, __MODULE__)
+
+    common_path =
+      Path.join([
+        key_prefix(),
+        Integer.to_string(org_id),
+        source_firmware_uuid,
+        "#{target_firmware_uuid}.delta.fw"
+      ])
+
+    local_path = Path.join([config[:local_path], common_path])
+
+    port =
+      if Enum.member?([443, 80], web_config[:url][:port]),
+        do: "",
+        else: ":#{web_config[:url][:port]}"
+
+    public_path =
+      Path.join([
+        "#{web_config[:url][:scheme]}://#{web_config[:url][:host]}#{port}/",
+        config[:public_path],
+        common_path
+      ])
+
+    %{
+      public_path: public_path,
+      local_path: local_path
+    }
+  end
+
+  def key_prefix() do
+    "firmware"
+  end
 end