Make tokenizer functions immutable (#47)

Author: jonatanklosko

lib/tokenizers/native.ex

@@ -126,8 +126,6 @@ defmodule Tokenizers.Native do
   # Trainers
   def trainers_info(_trainer), do: err()
   #
-  def trainers_train(_trainer, _model), do: err()
-  #
   def trainers_bpe_trainer(_options), do: err()
   def trainers_wordpiece_trainer(_options), do: err()
   def trainers_wordlevel_trainer(_options), do: err()

lib/tokenizers/tokenizer.ex

@@ -65,31 +65,18 @@ defmodule Tokenizers.Tokenizer do
       even if `:use_cache` is false. By default it uses `:filename.basedir/3` to get
       a cache dir based in the "tokenizers_elixir" application name.
 
-    * `:additional_special_tokens` - A list of special tokens to append to the tokenizer.
-      Defaults to `[]`.
-
-    * `:padding` - Override for padding configuration. Currently the only supported
-      value is `:none` to disable padding. By default the configuration is restored
-      from the file.
-
-    * `:truncation` - Override for truncation configuration. Currently the only supported
-      value is `:none` to disable truncation. By default the configuration is restored
-      from the file.
-
   """
   @spec from_pretrained(String.t(), Keyword.t()) :: {:ok, t()} | {:error, term()}
   def from_pretrained(identifier, opts \\ []) do
     opts =
       Keyword.validate!(
         opts,
         [
-          :padding,
-          :truncation,
+          :additional_special_tokens,
           revision: "main",
           use_cache: true,
           cache_dir: :filename.basedir(:user_cache, "tokenizers_elixir"),
-          http_client: {Tokenizers.HTTPClient, []},
-          additional_special_tokens: []
+          http_client: {Tokenizers.HTTPClient, []}
         ]
       )
 
@@ -114,7 +101,7 @@ defmodule Tokenizers.Tokenizer do
       Path.join(cache_dir, entry_filename(url, etag))
     end
 
-    load_opts = Keyword.take(opts, [:additional_special_tokens, :padding, :truncation])
+    load_opts = Keyword.take(opts, [:additional_special_tokens])
 
     if opts[:use_cache] do
       with {:ok, response} <- request(http_client, Keyword.put(http_opts, :method, :head)) do
@@ -183,43 +170,33 @@ defmodule Tokenizers.Tokenizer do
     Base.encode32(etag, case: :lower, padding: false)
   end
 
-  @typedoc """
-  Options to set on the loaded tokenizer.
-
-    * `:additional_special_tokens - a list of special tokens to append to the tokenizer.
-      Defaults to `[]`.
-
-    * `:padding` - Override for padding configuration. Currently the only supported
-      value is `:none` to disable padding. By default the configuration is restored
-      from the file.
-
-    * `:truncation` - Override for truncation configuration. Currently the only supported
-      value is `:none` to disable truncation. By default the configuration is restored
-      from the file.
-
-  """
-  @type load_options ::
-          [
-            additional_special_tokens: [String.t() | Tokenizers.AddedToken.t()],
-            padding: :none,
-            truncation: :none
-          ]
-
   @doc """
   Instantiate a new tokenizer from the file at the given path.
   """
-  @spec from_file(path :: String.t(), load_options()) :: {:ok, t()} | {:error, term()}
-  defdelegate from_file(path, options \\ []),
-    to: Tokenizers.Native,
-    as: :tokenizer_from_file
+  @spec from_file(path :: String.t(), keyword()) :: {:ok, t()} | {:error, term()}
+  def from_file(path, options \\ []) do
+    if Keyword.has_key?(options, :additional_special_tokens) do
+      IO.warn(
+        "passing :additional_special_tokens as an option is deprecated. Use add_special_tokens/2 instead"
+      )
+    end
+
+    Tokenizers.Native.tokenizer_from_file(path, options)
+  end
 
   @doc """
   Instantiate a new tokenizer from the buffer.
   """
-  @spec from_buffer(data :: String.t(), load_options()) :: {:ok, t()} | {:error, term()}
-  defdelegate from_buffer(data, options \\ []),
-    to: Tokenizers.Native,
-    as: :tokenizer_from_buffer
+  @spec from_buffer(data :: String.t(), keyword()) :: {:ok, t()} | {:error, term()}
+  def from_buffer(data, options \\ []) do
+    if Keyword.has_key?(options, :additional_special_tokens) do
+      IO.warn(
+        "passing :additional_special_tokens as an option is deprecated. Use add_special_tokens/2 instead"
+      )
+    end
+
+    Tokenizers.Native.tokenizer_from_buffer(data, options)
+  end
 
   @doc """
   Save the tokenizer to the provided path. Options:

lib/tokenizers/trainer.ex

@@ -12,13 +12,6 @@ defmodule Tokenizers.Trainer do
   @spec info(t()) :: map()
   defdelegate info(trainer), to: Tokenizers.Native, as: :trainers_info
 
-  @doc """
-  The actual training method.
-  This will mutate a Model as well as return a list of special_tokens to be added directly to the tokenizer along with the model.
-  """
-  @spec train(t(), Tokenizers.Model.t()) :: {:ok, [String.t()]} | {:error, any()}
-  defdelegate train(trainer, model), to: Tokenizers.Native, as: :trainers_train
-
   @typedoc """
   Options for BPE trainer initialisation. All options can be ommited.
   """

native/ex_tokenizers/src/lib.rs

@@ -148,8 +148,6 @@ rustler::init!(
         // Trainers
         trainers_info,
         //
-        trainers_train,
-        //
         trainers_bpe_trainer,
         trainers_wordpiece_trainer,
         trainers_wordlevel_trainer,