Add steps for pipeline (#43)

Author: Źmićer Rubinštejn

lib/tokenizers/decoder.ex

@@ -0,0 +1,139 @@
+defmodule Tokenizers.Decoder do
+  @moduledoc """
+  The Decoder knows how to go from the IDs used by the Tokenizer, back to a readable piece of text.
+  Some Normalizer and PreTokenizer use special characters or identifiers that need to be reverted.
+  """
+
+  defstruct [:resource]
+  @type t() :: %__MODULE__{resource: reference()}
+
+  @doc """
+  Decodes tokens into string with provided decoder.
+  """
+  @spec decode(t(), [String.t()]) :: {:ok, String.t()} | {:error, any()}
+  defdelegate decode(decoder, tokens), to: Tokenizers.Native, as: :decoders_decode
+
+  @typedoc """
+  Options for BPE decoder initialization. All options can be ommited.
+
+  * `suffix` - The suffix to add to the end of each word, defaults to `</w>`
+  """
+  @type bpe_options :: [suffix: String.t()]
+
+  @doc """
+  Creates new BPE decoder
+  """
+  @spec bpe(bpe_options :: bpe_options()) :: t()
+  defdelegate bpe(options \\ []), to: Tokenizers.Native, as: :decoders_bpe
+
+  @doc """
+  Creates new ByteFallback decoder
+  """
+  @spec byte_fallback() :: t()
+  defdelegate byte_fallback(), to: Tokenizers.Native, as: :decoders_byte_fallback
+
+  @doc """
+  Creates new ByteLevel decoder
+  """
+  @spec byte_level() :: t()
+  defdelegate byte_level(), to: Tokenizers.Native, as: :decoders_byte_level
+
+  @typedoc """
+  Options for CTC decoder initialization. All options can be ommited.
+
+  * `pad_token` - The token used for padding, defaults to `<pad>`
+  * `word_delimiter_token` - The token used for word delimiter, defaults to `|`
+  * `cleanup` - Whether to cleanup tokenization artifacts, defaults to `true`
+  """
+  @type ctc_options :: [
+          pad_token: String.t(),
+          word_delimiter_token: String.t(),
+          cleanup: boolean()
+        ]
+
+  @doc """
+  Creates new CTC decoder
+  """
+  @spec ctc(ctc_options :: ctc_options()) :: t()
+  defdelegate ctc(options \\ []), to: Tokenizers.Native, as: :decoders_ctc
+
+  @doc """
+  Creates new Fuse decoder
+  """
+  @spec fuse :: t()
+  defdelegate fuse(), to: Tokenizers.Native, as: :decoders_fuse
+
+  @typedoc """
+  Options for Metaspace decoder initialization. All options can be ommited.
+
+  * `replacement` - The replacement character, defaults to `▁` (as char)
+  * `add_prefix_space` - Whether to add a space to the first word, defaults to `true`
+  """
+
+  @type metaspace_options :: [
+          replacement: char(),
+          add_prefix_space: boolean()
+        ]
+
+  @doc """
+  Creates new Metaspace decoder
+  """
+  @spec metaspace(metaspace_options :: metaspace_options()) :: t()
+  defdelegate metaspace(options \\ []),
+    to: Tokenizers.Native,
+    as: :decoders_metaspace
+
+  @doc """
+  Creates new Replace decoder
+  """
+  @spec replace(pattern :: String.t(), content :: String.t()) :: t()
+  defdelegate replace(pattern, content), to: Tokenizers.Native, as: :decoders_replace
+
+  @doc """
+  Creates new Sequence decoder
+  """
+  @spec sequence(decoders :: [Tokenizers.Decoder.t()]) :: t()
+  defdelegate sequence(decoders), to: Tokenizers.Native, as: :decoders_sequence
+
+  @doc """
+  Creates new Strip decoder.
+  
+  It expects a character and the number of times to strip the
+  character on `left` and `right` sides.
+  """
+  @spec strip(content :: char(), left :: non_neg_integer(), right :: non_neg_integer()) :: t()
+  defdelegate strip(content, left, right), to: Tokenizers.Native, as: :decoders_strip
+
+  @typedoc """
+  Options for WordPiece decoder initialization. All options can be ommited.
+
+  * `prefix` - The prefix to use for subwords, defaults to `##`
+  * `cleanup` - Whether to cleanup tokenization artifacts, defaults to `true`
+  """
+  @type word_piece_options :: [
+          prefix: String.t(),
+          cleanup: boolean()
+        ]
+
+  @doc """
+  Creates new WordPiece decoder
+  """
+  @spec word_piece(word_piece_options :: word_piece_options()) :: t()
+  defdelegate word_piece(options \\ []),
+    to: Tokenizers.Native,
+    as: :decoders_wordpiece
+end
+
+defimpl Inspect, for: Tokenizers.Decoder do
+  import Inspect.Algebra
+
+  @spec inspect(Tokenizers.Decoder.t(), Inspect.Opts.t()) :: Inspect.Algebra.t()
+  def inspect(decoder, opts) do
+    attrs =
+      decoder
+      |> Tokenizers.Native.decoders_info()
+      |> Keyword.new(fn {k, v} -> {String.to_atom(k), v} end)
+
+    concat(["#Tokenizers.Decoder<", to_doc(attrs, opts), ">"])
+  end
+end

lib/tokenizers/native.ex

@@ -15,6 +15,22 @@ defmodule Tokenizers.Native do
   #
   def added_token_info(_added_token), do: err()
 
+  # Decoders
+  def decoders_decode(_decoder, _tokens), do: err()
+  #
+  def decoders_info(_decoder), do: err()
+  #
+  def decoders_byte_level(), do: err()
+  def decoders_replace(_pattern, _content), do: err()
+  def decoders_wordpiece(_options), do: err()
+  def decoders_byte_fallback(), do: err()
+  def decoders_fuse(), do: err()
+  def decoders_strip(_content, _left, _right), do: err()
+  def decoders_metaspace(_options), do: err()
+  def decoders_bpe(_options), do: err()
+  def decoders_ctc(_options), do: err()
+  def decoders_sequence(_decoders), do: err()
+
   # Models
   def models_save(_model, _folder, _opts), do: err()
   #
@@ -35,6 +51,51 @@ defmodule Tokenizers.Native do
   def models_unigram_init(_vocab, _options), do: err()
   def models_unigram_empty(), do: err()
 
+  # Normalizers
+  def normalizers_normalize(_normalizer, _input), do: err()
+  #
+  def normalizers_info(_normalizer), do: err()
+  #
+  def normalizers_bert_normalizer(_opts), do: err()
+  def normalizers_nfd(), do: err()
+  def normalizers_nfkd(), do: err()
+  def normalizers_nfc(), do: err()
+  def normalizers_nfkc(), do: err()
+  def normalizers_strip(_opts), do: err()
+  def normalizers_prepend(_prepend), do: err()
+  def normalizers_strip_accents(), do: err()
+  def normalizers_sequence(_normalizers), do: err()
+  def normalizers_lowercase(), do: err()
+  def normalizers_replace(_pattern, _content), do: err()
+  def normalizers_nmt(), do: err()
+  def normalizers_precompiled(_data), do: err()
+
+  # PreTokenizers
+  def pre_tokenizers_pre_tokenize(_pre_tokenizer, _input), do: err()
+  #
+  def pre_tokenizers_info(_pre_tokenizer), do: err()
+  #
+  def pre_tokenizers_byte_level(_opts), do: err()
+  def pre_tokenizers_byte_level_alphabet(), do: err()
+  def pre_tokenizers_whitespace(), do: err()
+  def pre_tokenizers_whitespace_split(), do: err()
+  def pre_tokenizers_bert(), do: err()
+  def pre_tokenizers_metaspace(_opts), do: err()
+  def pre_tokenizers_char_delimiter_split(_delimiter), do: err()
+  def pre_tokenizers_split(_pattern, _behavior, _options), do: err()
+  def pre_tokenizers_punctuation(_behavior), do: err()
+  def pre_tokenizers_sequence(_pre_tokenizers), do: err()
+  def pre_tokenizers_digits(_options), do: err()
+
+  # PostProcessors
+  def post_processors_info(_post_processor), do: err()
+  #
+  def post_processors_bert(_sep, _cls), do: err()
+  def post_processors_roberta(_sep, _cls, _opts), do: err()
+  def post_processors_byte_level(_opts), do: err()
+  def post_processors_template(_opts), do: err()
+  def post_processors_sequence(_post_processors), do: err()
+
   # Trainers
   def trainers_info(_trainer), do: err()
   #

lib/tokenizers/normalizer.ex

@@ -0,0 +1,142 @@
+defmodule Tokenizers.Normalizer do
+  @moduledoc """
+  A Normalizer is in charge of pre-processing the input string
+  in order to normalize it as relevant for a given use case.
+
+  Some common examples of normalization are the Unicode normalization algorithms
+  (NFD, NFKD, NFC & NFKC), lowercasing etc...
+  The specificity of tokenizers is that we keep track of the alignment while normalizing.
+  This is essential to allow mapping from the generated tokens back to the input text.
+
+  The Normalizer is optional.
+  """
+
+  @type t() :: %__MODULE__{resource: reference()}
+  defstruct [:resource]
+
+  @doc """
+  Normalizes the input presented as string into new string
+  """
+  @spec normalize(normalizer :: t(), input :: String.t()) :: {:ok, String.t()}
+  defdelegate normalize(normalizer, input), to: Tokenizers.Native, as: :normalizers_normalize
+
+  @typedoc """
+  Options for BERT normalizer initialisation. All values are optional.
+
+  * `:clean_text` (default `true`) - Whether to clean the text, by removing any control characters and replacing all whitespaces by the classic one.
+  * `:handle_chinese_chars` (default `true`) - Whether to handle chinese chars by putting spaces around them.
+  * `:strip_accents` - Whether to strip all accents. If this option is not specified, then it will be determined by the value for lowercase (as in the original Bert).
+  * `:lowercase` (default `true`) - Whether to lowercase.
+  """
+  @type bert_opts() :: [
+          clean_text: boolean(),
+          handle_chinese_chars: boolean(),
+          strip_accents: boolean(),
+          lowercase: boolean()
+        ]
+  @doc """
+  Takes care of normalizing raw text before giving it to a Bert model. This includes cleaning the text, handling accents, chinese chars and lowercasing.
+  """
+  @spec bert_normalizer(opts :: bert_opts()) :: t()
+  defdelegate bert_normalizer(opts \\ []),
+    to: Tokenizers.Native,
+    as: :normalizers_bert_normalizer
+
+  @doc """
+  NFD Unicode Normalizer,
+  """
+  @spec nfd :: t()
+  defdelegate nfd(), to: Tokenizers.Native, as: :normalizers_nfd
+
+  @doc """
+  NFKD Unicode Normalizer
+  """
+  @spec nfkd :: t()
+  defdelegate nfkd(), to: Tokenizers.Native, as: :normalizers_nfkd
+
+  @doc """
+  NFC Unicode Normalizer
+  """
+  @spec nfc :: t()
+  defdelegate nfc(), to: Tokenizers.Native, as: :normalizers_nfc
+
+  @doc """
+  NFKC Unicode Normalizer
+  """
+  @spec nfkc :: t()
+  defdelegate nfkc(), to: Tokenizers.Native, as: :normalizers_nfkc
+
+  @typedoc """
+  Options for Strip normalizer initialisation. All values are optional.
+
+  * `:left` (default `true`) - Whether to strip left side.
+  * `:right` (default `true`) - Whether to strip right side.
+  """
+  @type strip_opts() :: [
+          left: boolean(),
+          right: boolean()
+        ]
+  @doc """
+  Strip normalizer. Removes all whitespace characters on the specified sides (left, right or both) of the input
+  """
+  @spec strip(opts :: strip_opts()) :: t()
+  defdelegate strip(opts \\ []), to: Tokenizers.Native, as: :normalizers_strip
+
+  @doc """
+  Prepend normalizer.
+  """
+  @spec prepend(prepend :: String.t()) :: t()
+  defdelegate prepend(prepend), to: Tokenizers.Native, as: :normalizers_prepend
+
+  @doc """
+  Strip Accent normalizer. Removes all accent symbols in unicode (to be used with NFD for consistency).
+  """
+  @spec strip_accents :: t()
+  defdelegate strip_accents(), to: Tokenizers.Native, as: :normalizers_strip_accents
+
+  @doc """
+  Composes multiple normalizers that will run in the provided order.
+  """
+  @spec sequence(normalizers :: [t()]) :: t()
+  defdelegate sequence(normalizers), to: Tokenizers.Native, as: :normalizers_sequence
+
+  @doc """
+  Replaces all uppercase to lowercase
+  """
+  @spec lowercase :: t()
+  defdelegate lowercase(), to: Tokenizers.Native, as: :normalizers_lowercase
+
+  @doc """
+  Replaces a custom string or regexp and changes it with given content
+  """
+  @spec replace(pattern :: String.t(), content :: String.t()) ::
+          t()
+  defdelegate replace(pattern, content),
+    to: Tokenizers.Native,
+    as: :normalizers_replace
+
+  @doc """
+  Nmt normalizer
+  """
+  @spec nmt :: t()
+  defdelegate nmt(), to: Tokenizers.Native, as: :normalizers_nmt
+
+  @doc """
+  Precompiled normalizer. Don’t use manually it is used for compatiblity for SentencePiece.
+  """
+  @spec precompiled(data :: binary()) :: {:ok, t()} | {:error, any()}
+  defdelegate precompiled(data), to: Tokenizers.Native, as: :normalizers_precompiled
+end
+
+defimpl Inspect, for: Tokenizers.Normalizer do
+  import Inspect.Algebra
+
+  def inspect(decoder, opts) do
+    attrs =
+      decoder
+      |> Tokenizers.Native.normalizers_info()
+      |> Keyword.new(fn {k, v} -> {String.to_atom(k), v} end)
+
+    concat(["#Tokenizers.Normalizer<", to_doc(attrs, opts), ">"])
+  end
+end