Update tokenizers to 0.15.0 (#55)

* Update tokenizers to 0.15.0

* Fix clippy complaints

* Update flake

* Expose byte fallback for unigram

Author: cigrainger

flake.lock

@@ -2,52 +2,63 @@
   description = "Tokenizers";
 
   inputs = {
+    fenix = {
+      url = "github:nix-community/fenix";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
     nixpkgs.url = "nixpkgs/nixos-unstable";
     flake-utils.url = "github:numtide/flake-utils";
   };
 
-  outputs = { self, nixpkgs, flake-utils }:
+  outputs = {
+    self,
+    nixpkgs,
+    flake-utils,
+    fenix,
+  }:
     flake-utils.lib.eachSystem [
       flake-utils.lib.system.x86_64-linux
       flake-utils.lib.system.x86_64-darwin
       flake-utils.lib.system.aarch64-darwin
       flake-utils.lib.system.aarch64-linux
     ]
-      (system:
-        let pkgs = import nixpkgs { inherit system; };
-        in
-        {
-          devShell = pkgs.mkShell {
-            buildInputs = with pkgs; [
-              act
-              binutils
-              cargo
-              cc
-              clang
-              clippy
-              elixir_1_14
-              erlang
-              gdb
-              gcc
-              libiconv
-              openssl
-              pkg-config
-              rustc
-            ] ++ lib.optionals stdenv.isDarwin [
-              darwin.apple_sdk.frameworks.Foundation
-              darwin.apple_sdk.frameworks.Carbon
-              darwin.apple_sdk.frameworks.AppKit
-            ];
-            shellHook = ''
-              mkdir -p .nix-mix
-              mkdir -p .nix-hex
-              export MIX_HOME=$PWD/.nix-mix
-              export HEX_HOME=$PWD/.nix-hex
-              export PATH=$MIX_HOME/bin:$PATH
-              export PATH=$HEX_HOME/bin:$PATH
-              export PATH=$MIX_HOME/escripts:$PATH
-              export ERL_AFLAGS="-kernel shell_history enabled"
-            '';
-          };
-        });
+    (system: let
+      pkgs = import nixpkgs {inherit system;};
+    in {
+      devShell = pkgs.mkShell {
+        buildInputs = with pkgs;
+          [
+            act
+            binutils
+            clang
+            elixir_1_15
+            (fenix.packages."${system}".complete.withComponents [
+              "cargo"
+              "clippy"
+              "rust-src"
+              "rustc"
+              "rustfmt"
+            ])
+            gcc
+            libiconv
+            openssl
+            pkg-config
+          ]
+          ++ lib.optionals stdenv.isDarwin [
+            darwin.apple_sdk.frameworks.Foundation
+            darwin.apple_sdk.frameworks.Carbon
+            darwin.apple_sdk.frameworks.AppKit
+          ];
+        shellHook = ''
+          mkdir -p .nix-mix
+          mkdir -p .nix-hex
+          export MIX_HOME=$PWD/.nix-mix
+          export HEX_HOME=$PWD/.nix-hex
+          export PATH=$MIX_HOME/bin:$PATH
+          export PATH=$HEX_HOME/bin:$PATH
+          export PATH=$MIX_HOME/escripts:$PATH
+          export ERL_AFLAGS="-kernel shell_history enabled"
+        '';
+      };
+    });
 }

flake.nix

@@ -2,6 +2,8 @@ defmodule Tokenizers.Model.BPE do
   @typedoc """
   Options for model initialisation.
 
+    * `:byte_fallback`- whether to use the byte fallback trick
+
     * `:cache_capacity` - the number of words that the BPE cache can
       contain. The cache allows to speed-up the process by keeping
       the result of the merge operations for a number of words.

lib/tokenizers/model/bpe.ex

@@ -2,10 +2,12 @@ defmodule Tokenizers.Model.Unigram do
   @typedoc """
   Options for model initialisation.
 
+    * `:byte_fallback`- whether to use the byte fallback trick
     * `:unk_id`- the unknown token id to be used by the model
 
   """
   @type options() :: [
+          byte_fallback: boolean(),
           unk_id: float()
         ]
 

lib/tokenizers/model/unigram.ex

@@ -2,7 +2,7 @@ defmodule Tokenizers.MixProject do
   use Mix.Project
 
   @source_url "https://github.com/elixir-nx/tokenizers"
-  @version "0.4.0"
+  @version "0.5.0-dev"
 
   def project do
     [