Add tests for info, temoving Char

Author: Virviil

lib/tokenizers/decode_stream.ex

@@ -28,30 +28,20 @@ defmodule Tokenizers.DecodeStream do
   Returns `{:error, reason}` if an error occurs during decoding.
   """
   def step(%__MODULE__{} = decode_stream, tokenizer, id) when is_integer(id) do
-    case Tokenizers.Native.decoder_stream_step(decode_stream, tokenizer, id) do
-      {:ok, result} -> {:ok, result}
-      {:error, reason} -> {:error, reason}
-    end
+    Tokenizers.Native.decoder_stream_step(decode_stream, tokenizer, id)
   end
 
   @doc """
   Returns information about the decode stream state.
   """
-  def info(%__MODULE__{} = decode_stream) do
-    Tokenizers.Native.decoder_stream_info(decode_stream)
-  end
+  defdelegate info(decode_stream), to: Tokenizers.Native, as: :decoder_stream_info
 
   defimpl Inspect do
-    def inspect(decode_stream, _opts) do
-      info = Tokenizers.DecodeStream.info(decode_stream)
-      "#Tokenizers.DecodeStream<#{inspect(info)}>"
-    end
-  end
+    import Inspect.Algebra
+    alias Tokenizers.DecodeStream
 
-  defimpl String.Chars do
-    def to_string(decode_stream) do
-      info = Tokenizers.DecodeStream.info(decode_stream)
-      "#Tokenizers.DecodeStream<#{inspect(info)}>"
+    def inspect(decode_stream, opts) do
+      "#Tokenizers.DecodeStream<#{to_doc(DecodeStream.info(decode_stream), opts)}>"
     end
   end
 end

lib/tokenizers/native.ex

@@ -10,7 +10,8 @@ defmodule Tokenizers.Native do
     crate: "ex_tokenizers",
     version: version,
     base_url: "#{github_url}/releases/download/v#{version}",
-    force_build: System.get_env("TOKENIZERS_BUILD") in ["1", "true"]
+    # force_build: System.get_env("TOKENIZERS_BUILD") in ["1", "true"]
+    force_build: true
 
   # Added tokens
   def added_token_new(_token, _opts), do: err()

test/tokenizers/decode_stream_test.exs

@@ -64,4 +64,24 @@ defmodule Tokenizers.DecodeStreamTest do
       {:ok, " This"} = Tokenizers.DecodeStream.step(ds, tk, 1)
     end
   end
+
+  describe "DecodeStream info" do
+    test "skip_special_tokens false" do
+      assert Tokenizers.DecodeStream.info(Tokenizers.DecodeStream.new(false)) == %{
+               "skip_special_tokens" => false
+             }
+    end
+
+    test "skip_special_tokens true" do
+      assert Tokenizers.DecodeStream.info(Tokenizers.DecodeStream.new(true)) == %{
+               "skip_special_tokens" => true
+             }
+    end
+
+    test "default DecodeStream" do
+      assert Tokenizers.DecodeStream.info(Tokenizers.DecodeStream.new()) == %{
+               "skip_special_tokens" => false
+             }
+    end
+  end
 end