Add basic HTTP client to download pretrained files (#21)

Close #20

Author: Philip Sampaio

.github/workflows/release.yml

@@ -33,14 +33,13 @@ jobs:
               os: ubuntu-20.04,
               nif: "2.16",
               use-cross: true,
-              features: "static_openssl",
             }
           - { target: aarch64-unknown-linux-gnu, os: ubuntu-20.04, nif: "2.16", use-cross: true }
-          - { target: aarch64-unknown-linux-musl, os: ubuntu-20.04, nif: "2.16", use-cross: true, features: "static_openssl" }
+          - { target: aarch64-unknown-linux-musl, os: ubuntu-20.04, nif: "2.16", use-cross: true }
           - { target: aarch64-apple-darwin, os: macos-11, nif: "2.16" }
           - { target: x86_64-apple-darwin, os: macos-11, nif: "2.16" }
           - { target: x86_64-unknown-linux-gnu, os: ubuntu-20.04, nif: "2.16" }
-          - { target: x86_64-unknown-linux-musl, os: ubuntu-20.04, nif: "2.16", use-cross: true, features: "static_openssl" }
+          - { target: x86_64-unknown-linux-musl, os: ubuntu-20.04, nif: "2.16", use-cross: true }
           - { target: x86_64-pc-windows-gnu, os: windows-2019, nif: "2.16" }
           - { target: x86_64-pc-windows-msvc, os: windows-2019, nif: "2.16" }
           # NIF version 2.15
@@ -49,14 +48,13 @@ jobs:
               os: ubuntu-20.04,
               nif: "2.15",
               use-cross: true,
-              features: "static_openssl",
             }
           - { target: aarch64-unknown-linux-gnu, os: ubuntu-20.04, nif: "2.15", use-cross: true }
-          - { target: aarch64-unknown-linux-musl, os: ubuntu-20.04, nif: "2.15", use-cross: true, features: "static_openssl" }
+          - { target: aarch64-unknown-linux-musl, os: ubuntu-20.04, nif: "2.15", use-cross: true }
           - { target: aarch64-apple-darwin, os: macos-11, nif: "2.15" }
           - { target: x86_64-apple-darwin, os: macos-11, nif: "2.15" }
           - { target: x86_64-unknown-linux-gnu, os: ubuntu-20.04, nif: "2.15" }
-          - { target: x86_64-unknown-linux-musl, os: ubuntu-20.04, nif: "2.15", use-cross: true, features: "static_openssl" }
+          - { target: x86_64-unknown-linux-musl, os: ubuntu-20.04, nif: "2.15", use-cross: true }
           - { target: x86_64-pc-windows-gnu, os: windows-2019, nif: "2.15" }
           - { target: x86_64-pc-windows-msvc, os: windows-2019, nif: "2.15" }
           # NIF version 2.14
@@ -65,14 +63,13 @@ jobs:
               os: ubuntu-20.04,
               nif: "2.14",
               use-cross: true,
-              features: "static_openssl",
             }
           - { target: aarch64-unknown-linux-gnu, os: ubuntu-20.04, nif: "2.14", use-cross: true }
-          - { target: aarch64-unknown-linux-musl, os: ubuntu-20.04, nif: "2.14", use-cross: true, features: "static_openssl" }
+          - { target: aarch64-unknown-linux-musl, os: ubuntu-20.04, nif: "2.14", use-cross: true }
           - { target: aarch64-apple-darwin, os: macos-11, nif: "2.14" }
           - { target: x86_64-apple-darwin, os: macos-11, nif: "2.14" }
           - { target: x86_64-unknown-linux-gnu, os: ubuntu-20.04, nif: "2.14" }
-          - { target: x86_64-unknown-linux-musl, os: ubuntu-20.04, nif: "2.14", use-cross: true, features: "static_openssl" }
+          - { target: x86_64-unknown-linux-musl, os: ubuntu-20.04, nif: "2.14", use-cross: true }
           - { target: x86_64-pc-windows-gnu, os: windows-2019, nif: "2.14" }
           - { target: x86_64-pc-windows-msvc, os: windows-2019, nif: "2.14" }
 
@@ -122,9 +119,9 @@ jobs:
         shell: bash
         run: |
           if [ "${{ matrix.job.use-cross }}" == "true" ]; then
-            cross build --release --target=${{ matrix.job.target }} --features=${{ matrix.job.features }}
+            cross build --release --target=${{ matrix.job.target }}
           else
-            cargo build --release --target=${{ matrix.job.target }} --features=${{ matrix.job.features }}
+            cargo build --release --target=${{ matrix.job.target }}
           fi
 
       - name: Rename lib to the final name

lib/tokenizers/http_client.ex

@@ -0,0 +1,74 @@
+defmodule Tokenizers.HTTPClient do
+  @moduledoc """
+  A simple implementation of an HTTP client.
+
+  This is using the built-in `:httpc` module, configured to use SSL.
+  The `request/1` function is similar to `Req.request/1`.
+  """
+
+  @base_url "https://huggingface.io"
+
+  @doc """
+  Make an HTTP(s) requests.
+
+  ## Options
+
+    * `:method` - An HTTP method. By default it uses the `:get` method.
+
+    * `:base_url` - The base URL to make requests. By default is #{inspect(@base_url)}. 
+
+    * `:url` - A path to a resource. By default is "".
+
+    * `:headers` - A list of tuples representing HTTP headers. By default it's empty.
+
+  """
+  def request(opts) when is_list(opts) do
+    opts = Keyword.validate!(opts, base_url: @base_url, headers: [], method: :get, url: "")
+
+    url = Path.join([opts[:base_url], opts[:url]]) |> String.to_charlist()
+    headers = Enum.map(opts[:headers], fn {key, value} -> {String.to_charlist(key), value} end)
+
+    {:ok, _} = Application.ensure_all_started(:inets)
+    {:ok, _} = Application.ensure_all_started(:ssl)
+
+    if proxy = System.get_env("HTTP_PROXY") || System.get_env("http_proxy") do
+      %{host: host, port: port} = URI.parse(proxy)
+
+      :httpc.set_options([{:proxy, {{String.to_charlist(host), port}, []}}])
+    end
+
+    proxy = System.get_env("HTTPS_PROXY") || System.get_env("https_proxy")
+
+    with true <- is_binary(proxy),
+         %{host: host, port: port} when is_binary(host) and is_integer(port) <- URI.parse(proxy) do
+      :httpc.set_options([{:https_proxy, {{String.to_charlist(host), port}, []}}])
+    end
+
+    # https://erlef.github.io/security-wg/secure_coding_and_deployment_hardening/inets
+    cacertfile = CAStore.file_path() |> String.to_charlist()
+
+    http_options = [
+      ssl: [
+        verify: :verify_peer,
+        cacertfile: cacertfile,
+        depth: 3,
+        customize_hostname_check: [
+          match_fun: :public_key.pkix_verify_hostname_match_fun(:https)
+        ]
+      ]
+    ]
+
+    options = [body_format: :binary]
+
+    case :httpc.request(opts[:method], {url, headers}, http_options, options) do
+      {:ok, {{_, status, _}, headers, body}} ->
+        {:ok, %{status: status, headers: headers, body: body}}
+
+      {:ok, {status, body}} ->
+        {:ok, %{status: status, body: body, headers: []}}
+
+      {:error, reason} ->
+        {:error, "could not make request #{url}: #{inspect(reason)}"}
+    end
+  end
+end

lib/tokenizers/native.ex

@@ -16,7 +16,6 @@ defmodule Tokenizers.Native do
   def encode(_tokenizer, _input, _add_special_tokens), do: err()
   def encode_batch(_tokenizer, _input, _add_special_tokens), do: err()
   def from_file(_path), do: err()
-  def from_pretrained(_identifier), do: err()
   def get_attention_mask(_encoding), do: err()
   def get_type_ids(_encoding), do: err()
   def get_ids(_encoding), do: err()

lib/tokenizers/tokenizer.ex

@@ -30,9 +30,68 @@ defmodule Tokenizers.Tokenizer do
 
   @doc """
   Instantiate a new tokenizer from an existing file on the Hugging Face Hub.
+
+  This is going to download a tokenizer file, save to a file and load that file.
+
+  ## Options
+
+    * `:http_client` - A tuple with a module and options. This module should implement
+      the `request/1` function, accepting a keyword list with the options for a request.
+      This is inspired by `Req.request/1`: https://hexdocs.pm/req/Req.html#request/1
+
+      The default HTTP client config is: `{Tokenizers.HTTPClient, []}`.
+      Since it's inspired by `Req`, it's possible to use that client without any adjustments.
+
+      When making request, the options `:url` and `:method` are going to be overriden.
+      `:headers` contains the "user-agent" set by default.
+
+    * `:revision` - The revision name that should be used for fetching the tokenizers
+      from Hugging Face.
+
   """
-  @spec from_pretrained(String.t()) :: {:ok, Tokenizer.t()} | {:error, term()}
-  def from_pretrained(identifier), do: Native.from_pretrained(identifier)
+  @spec from_pretrained(String.t(), Keyword.t()) :: {:ok, Tokenizer.t()} | {:error, term()}
+  def from_pretrained(identifier, opts \\ []) do
+    opts = Keyword.validate!(opts, revision: "main", http_client: {Tokenizers.HTTPClient, []})
+
+    {http_client, http_opts} = opts[:http_client]
+
+    {:ok, app_version} = :application.get_key(:tokenizers, :vsn)
+    app_version = List.to_string(app_version)
+
+    headers = [{"user-agent", "tokenizers-elixir/#{app_version}"}]
+    url = "/#{identifier}/resolve/#{opts[:revision]}/tokenizer.json"
+
+    http_opts =
+      http_opts
+      |> Keyword.put_new(:base_url, "https://huggingface.co")
+      |> Keyword.put(:url, url)
+      |> Keyword.put(:method, :get)
+      |> Keyword.update(:headers, headers, fn existing -> existing ++ headers end)
+
+    case http_client.request(http_opts) do
+      {:ok, response} ->
+        case response.status do
+          status when status in 200..299 ->
+            cache_dir = :filename.basedir(:user_cache, "tokenizers_elixir")
+            :ok = File.mkdir_p(cache_dir)
+            file_path = Path.join(cache_dir, "#{identifier}.json")
+
+            :ok = File.write(file_path, response.body)
+
+            from_file(file_path)
+
+          404 ->
+            {:error, :not_found}
+
+          other ->
+            {:error,
+             "download of pretrained file failed with status #{other}. Response: #{inspect(response.body)}"}
+        end
+
+      {:error, _} = error ->
+        error
+    end
+  end
 
   @doc """
   Instantiate a new tokenizer from the file at the given path.

mix.exs

@@ -2,7 +2,7 @@ defmodule Tokenizers.MixProject do
   use Mix.Project
 
   @source_url "https://github.com/elixir-nx/tokenizers"
-  @version "0.1.2"
+  @version "0.1.2-dev"
 
   def project do
     [
@@ -22,11 +22,14 @@ defmodule Tokenizers.MixProject do
   end
 
   def application do
-    [extra_applications: [:logger]]
+    [
+      extra_applications: [:logger, :inets, :public_key]
+    ]
   end
 
   defp deps do
     [
+      {:castore, "~> 0.1"},
       {:ex_doc, "~> 0.28", only: :docs, runtime: false},
       {:rustler, ">= 0.0.0", optional: true},
       {:rustler_precompiled, "~> 0.5"}