Merge pull request #2 from gsmlg-dev/develop

Author: gsmlg

lib/http.ex

@@ -0,0 +1,93 @@
+defmodule HTTP.Config do
+  @moduledoc """
+  Configuration module for HTTP fetch library.
+
+  This module centralizes configuration values that control HTTP client behavior,
+  including timeouts, streaming thresholds, and other runtime parameters.
+
+  ## Configuration Values
+
+  All configuration values are compile-time constants defined as module attributes.
+  To customize these values, modify this module and recompile the library.
+
+  ### Streaming Configuration
+
+  - `streaming_threshold/0` - Size threshold (in bytes) above which responses are automatically streamed.
+    Default: 5MB (5,000,000 bytes)
+
+  ### Timeout Configuration
+
+  - `default_request_timeout/0` - Maximum time (in milliseconds) to wait for a complete HTTP response.
+    Default: 120 seconds (120,000 ms)
+
+  - `streaming_timeout/0` - Maximum time (in milliseconds) to wait for streaming operations.
+    Default: 60 seconds (60,000 ms)
+
+  ## Usage
+
+      # Check if response should be streamed
+      if content_length > HTTP.Config.streaming_threshold() do
+        # Stream the response
+      end
+
+      # Use default timeout
+      receive do
+        {:http, response} -> handle_response(response)
+      after
+        HTTP.Config.default_request_timeout() -> :timeout
+      end
+  """
+
+  @streaming_threshold 5_000_000
+  @default_request_timeout 120_000
+  @streaming_timeout 60_000
+
+  @doc """
+  Returns the size threshold (in bytes) for automatic streaming.
+
+  Responses with Content-Length greater than this value will be automatically
+  streamed to avoid loading large files into memory.
+
+  Default: 5MB (5,000,000 bytes)
+
+  ## Examples
+
+      iex> HTTP.Config.streaming_threshold()
+      5_000_000
+  """
+  @spec streaming_threshold() :: 5_000_000
+  def streaming_threshold, do: @streaming_threshold
+
+  @doc """
+  Returns the default request timeout in milliseconds.
+
+  This is the maximum time the HTTP client will wait for a complete response
+  after sending a request.
+
+  Default: 120 seconds (120,000 milliseconds)
+
+  ## Examples
+
+      iex> HTTP.Config.default_request_timeout()
+      120_000
+  """
+  @spec default_request_timeout() :: 120_000
+  def default_request_timeout, do: @default_request_timeout
+
+  @doc """
+  Returns the streaming timeout in milliseconds.
+
+  This is the maximum time to wait for streaming operations, including:
+  - Waiting for stream chunks in `collect_stream/2`
+  - Waiting for messages in the stream loop
+
+  Default: 60 seconds (60,000 milliseconds)
+
+  ## Examples
+
+      iex> HTTP.Config.streaming_timeout()
+      60_000
+  """
+  @spec streaming_timeout() :: 60_000
+  def streaming_timeout, do: @streaming_timeout
+end

lib/http/config.ex

@@ -128,8 +128,9 @@ defmodule HTTP.FormData do
   Converts FormData to HTTP body content with appropriate encoding.
 
   Returns {:url_encoded, body} for regular forms or {:multipart, body, boundary} for multipart.
+  The multipart body is returned as iodata for memory efficiency with large file uploads.
   """
-  @spec to_body(t()) :: {:url_encoded, String.t()} | {:multipart, String.t(), String.t()}
+  @spec to_body(t()) :: {:url_encoded, String.t()} | {:multipart, iodata(), String.t()}
   def to_body(%__MODULE__{parts: parts} = form) do
     has_file? =
       Enum.any?(parts, fn
@@ -184,33 +185,38 @@ defmodule HTTP.FormData do
 
     body_parts =
       parts
-      |> Enum.map(fn
+      |> Enum.flat_map(fn
         {:field, name, value} ->
-          encode_multipart_field(boundary, name, value)
+          [encode_multipart_field(boundary, name, value), "\r\n"]
 
         {:file, name, filename, content_type, %File.Stream{} = stream} ->
-          encode_multipart_file_stream(boundary, name, filename, content_type, stream)
+          [encode_multipart_file_stream(boundary, name, filename, content_type, stream), "\r\n"]
 
         {:file, name, filename, content_type, content} ->
-          encode_multipart_file_content(boundary, name, filename, content_type, content)
+          [encode_multipart_file_content(boundary, name, filename, content_type, content), "\r\n"]
       end)
 
-    body = Enum.join(body_parts, "\r\n") <> "\r\n--" <> boundary <> "--\r\n"
+    # Build as iodata list for memory efficiency
+    body = [body_parts, "--", boundary, "--\r\n"]
 
     {:multipart, body, boundary}
   end
 
   defp encode_multipart_field(boundary, name, value) do
-    "--#{boundary}\r\n" <>
-      "Content-Disposition: form-data; name=\"#{name}\"\r\n\r\n" <>
-      "#{value}"
+    [
+      "--#{boundary}\r\n",
+      "Content-Disposition: form-data; name=\"#{name}\"\r\n\r\n",
+      value
+    ]
   end
 
   defp encode_multipart_file_content(boundary, name, filename, content_type, content) do
-    "--#{boundary}\r\n" <>
-      "Content-Disposition: form-data; name=\"#{name}\"; filename=\"#{filename}\"\r\n" <>
-      "Content-Type: #{content_type}\r\n\r\n" <>
+    [
+      "--#{boundary}\r\n",
+      "Content-Disposition: form-data; name=\"#{name}\"; filename=\"#{filename}\"\r\n",
+      "Content-Type: #{content_type}\r\n\r\n",
       content
+    ]
   end
 
   defp encode_multipart_file_stream(
@@ -220,7 +226,9 @@ defmodule HTTP.FormData do
          content_type,
          %File.Stream{} = stream
        ) do
-    content = stream |> Enum.into("")
-    encode_multipart_file_content(boundary, name, filename, content_type, content)
+    # Convert stream to list of binaries (iodata) instead of concatenating into single binary
+    # This avoids loading the entire file into memory as one large binary
+    content_chunks = Enum.to_list(stream)
+    encode_multipart_file_content(boundary, name, filename, content_type, content_chunks)
   end
 end

lib/http/form_data.ex

@@ -124,7 +124,8 @@ defmodule HTTP.Request do
               content_type = to_charlist("multipart/form-data; boundary=#{boundary}")
               # Add boundary header
               updated_headers = headers ++ [{~c"Content-Type", to_charlist(content_type)}]
-              {url, updated_headers, to_charlist(body)}
+              # Convert iodata to charlist efficiently
+              {url, updated_headers, iodata_to_charlist(body)}
           end
 
         # For regular string/charlist bodies
@@ -143,6 +144,15 @@ defmodule HTTP.Request do
   defp to_body(body) when is_list(body), do: body
   defp to_body(other), do: String.to_charlist(to_string(other))
 
+  # Efficiently converts iodata (nested list of binaries) to charlist
+  # This is used for multipart form data to minimize intermediate copies
+  # The iodata structure is flattened once instead of concatenating strings multiple times
+  defp iodata_to_charlist(iodata) do
+    iodata
+    |> IO.iodata_to_binary()
+    |> String.to_charlist()
+  end
+
   @spec add_default_user_agent([{String.t(), String.t()}]) :: [{String.t(), String.t()}]
   defp add_default_user_agent(headers) do
     # Check if user agent is already provided (case-insensitive)

lib/http/request.ex

@@ -133,8 +133,7 @@ defmodule HTTP.Response do
       {:stream_error, ^stream, _reason} ->
         acc
     after
-      # 60 second timeout
-      60_000 -> acc
+      HTTP.Config.streaming_timeout() -> acc
     end
   end
 
@@ -239,33 +238,31 @@ defmodule HTTP.Response do
   """
   @spec write_to(t(), String.t()) :: :ok | {:error, term()}
   def write_to(%__MODULE__{} = response, file_path) do
-    try do
-      # Ensure the directory exists
-      file_path
-      |> Path.dirname()
-      |> File.mkdir_p!()
-
-      case response do
-        %{body: body, stream: nil} when is_binary(body) or is_list(body) ->
-          # Non-streaming response
-          binary_body =
-            if is_list(body), do: IO.iodata_to_binary(body), else: body
-
-          File.write!(file_path, binary_body)
-          :ok
-
-        %{body: _body, stream: stream} when is_pid(stream) ->
-          # Streaming response - collect and write
-          write_stream_to_file(response, file_path)
-
-        _ ->
-          # Empty or nil body
-          File.write!(file_path, "")
-          :ok
-      end
-    rescue
-      error -> {:error, error}
+    # Ensure the directory exists
+    file_path
+    |> Path.dirname()
+    |> File.mkdir_p!()
+
+    case response do
+      %{body: body, stream: nil} when is_binary(body) or is_list(body) ->
+        # Non-streaming response
+        binary_body =
+          if is_list(body), do: IO.iodata_to_binary(body), else: body
+
+        File.write!(file_path, binary_body)
+        :ok
+
+      %{body: _body, stream: stream} when is_pid(stream) ->
+        # Streaming response - collect and write
+        write_stream_to_file(response, file_path)
+
+      _ ->
+        # Empty or nil body
+        File.write!(file_path, "")
+        :ok
     end
+  rescue
+    error -> {:error, error}
   end
 
   defp write_stream_to_file(response, file_path) do

lib/http/response.ex

@@ -17,7 +17,7 @@ defmodule HTTP.Telemetry do
 
   **`[:http_fetch, :request, :start]`** - Emitted when a request begins
 
-  - Measurements: `%{start_time: integer}` (milliseconds)
+  - Measurements: `%{start_time: integer}` (microseconds)
   - Metadata: `%{method: atom, url: URI.t(), headers: HTTP.Headers.t()}`
 
   **`[:http_fetch, :request, :stop]`** - Emitted when a request completes successfully
@@ -143,7 +143,7 @@ defmodule HTTP.Telemetry do
   """
   @spec request_start(String.t(), URI.t(), HTTP.Headers.t()) :: :ok
   def request_start(method, url, headers) do
-    measurements = %{start_time: System.system_time(:millisecond)}
+    measurements = %{start_time: System.system_time(:microsecond)}
 
     metadata = %{
       method: method,

lib/http/telemetry.ex

@@ -81,10 +81,15 @@ defmodule HTTP.FormDataTest do
         |> Map.put(:boundary, "test-boundary")
 
       assert {:multipart, body, "test-boundary"} = HTTP.FormData.to_body(form)
-      assert body =~ "Content-Disposition: form-data; name=\"name\""
-      assert body =~ "John"
-      assert body =~ "Content-Disposition: form-data; name=\"upload\"; filename=\"test.txt\""
-      assert body =~ "file content"
+      # Convert iodata to binary for testing
+      body_string = IO.iodata_to_binary(body)
+      assert body_string =~ "Content-Disposition: form-data; name=\"name\""
+      assert body_string =~ "John"
+
+      assert body_string =~
+               "Content-Disposition: form-data; name=\"upload\"; filename=\"test.txt\""
+
+      assert body_string =~ "file content"
     end
 
     test "encodes multipart with files" do
@@ -95,10 +100,15 @@ defmodule HTTP.FormDataTest do
 
       assert {:multipart, body, boundary} = HTTP.FormData.to_body(form)
       assert is_binary(boundary)
-      assert body =~ "Content-Disposition: form-data; name=\"description\""
-      assert body =~ "test file"
-      assert body =~ "Content-Disposition: form-data; name=\"upload\"; filename=\"test.txt\""
-      assert body =~ "file content"
+      # Convert iodata to binary for testing
+      body_string = IO.iodata_to_binary(body)
+      assert body_string =~ "Content-Disposition: form-data; name=\"description\""
+      assert body_string =~ "test file"
+
+      assert body_string =~
+               "Content-Disposition: form-data; name=\"upload\"; filename=\"test.txt\""
+
+      assert body_string =~ "file content"
     end
 
     test "encodes multipart with file stream" do
@@ -113,8 +123,13 @@ defmodule HTTP.FormDataTest do
         |> Map.put(:boundary, "test-boundary")
 
       assert {:multipart, body, "test-boundary"} = HTTP.FormData.to_body(form)
-      assert body =~ "Content-Disposition: form-data; name=\"upload\"; filename=\"test.txt\""
-      assert body =~ test_content
+      # Convert iodata to binary for testing
+      body_string = IO.iodata_to_binary(body)
+
+      assert body_string =~
+               "Content-Disposition: form-data; name=\"upload\"; filename=\"test.txt\""
+
+      assert body_string =~ test_content
 
       File.rm(path)
     end