Add riscv64gc-unknown-linux-gnu for RISC-V Nerves devices (#27)

Nerves supports 64-bit RISC-V devices. One of these is the MangoPi MQ
Pro which is being used as an alternative to the Raspberry Pi. Currently
it's much, much easier to use precompiled Rust binaries with Nerves. In
fact, manually crosscompiling `tokenizers` with Nerves doesn't work due
to a compiler error deep in the Rust build.

This adds the riscv64gc target to make it possible to use `tokenizers`
with Nerves on 64-bit RISC-V devices as easy as 64-bit ARM ones. This,
in turn, enables Bumblebee support.

Author: fhunleth

.github/workflows/release.yml

@@ -37,6 +37,7 @@ jobs:
           - { target: aarch64-unknown-linux-gnu, os: ubuntu-20.04, nif: "2.16", use-cross: true }
           - { target: aarch64-unknown-linux-musl, os: ubuntu-20.04, nif: "2.16", use-cross: true }
           - { target: aarch64-apple-darwin, os: macos-11, nif: "2.16" }
+          - { target: riscv64gc-unknown-linux-gnu, os: ubuntu-20.04, nif: "2.16", use-cross: true }
           - { target: x86_64-apple-darwin, os: macos-11, nif: "2.16" }
           - { target: x86_64-unknown-linux-gnu, os: ubuntu-20.04, nif: "2.16" }
           - { target: x86_64-unknown-linux-musl, os: ubuntu-20.04, nif: "2.16", use-cross: true }
@@ -52,6 +53,7 @@ jobs:
           - { target: aarch64-unknown-linux-gnu, os: ubuntu-20.04, nif: "2.15", use-cross: true }
           - { target: aarch64-unknown-linux-musl, os: ubuntu-20.04, nif: "2.15", use-cross: true }
           - { target: aarch64-apple-darwin, os: macos-11, nif: "2.15" }
+          - { target: riscv64gc-unknown-linux-gnu, os: ubuntu-20.04, nif: "2.15", use-cross: true }
           - { target: x86_64-apple-darwin, os: macos-11, nif: "2.15" }
           - { target: x86_64-unknown-linux-gnu, os: ubuntu-20.04, nif: "2.15" }
           - { target: x86_64-unknown-linux-musl, os: ubuntu-20.04, nif: "2.15", use-cross: true }
@@ -67,6 +69,7 @@ jobs:
           - { target: aarch64-unknown-linux-gnu, os: ubuntu-20.04, nif: "2.14", use-cross: true }
           - { target: aarch64-unknown-linux-musl, os: ubuntu-20.04, nif: "2.14", use-cross: true }
           - { target: aarch64-apple-darwin, os: macos-11, nif: "2.14" }
+          - { target: riscv64gc-unknown-linux-gnu, os: ubuntu-20.04, nif: "2.14", use-cross: true }
           - { target: x86_64-apple-darwin, os: macos-11, nif: "2.14" }
           - { target: x86_64-unknown-linux-gnu, os: ubuntu-20.04, nif: "2.14" }
           - { target: x86_64-unknown-linux-musl, os: ubuntu-20.04, nif: "2.14", use-cross: true }

lib/tokenizers/native.ex

@@ -9,7 +9,9 @@ defmodule Tokenizers.Native do
     version: version,
     base_url: "#{github_url}/releases/download/v#{version}",
     force_build: System.get_env("TOKENIZERS_BUILD") in ["1", "true"],
-    targets: RustlerPrecompiled.Config.default_targets() ++ ["aarch64-unknown-linux-musl"]
+    targets:
+      RustlerPrecompiled.Config.default_targets() ++
+        ["aarch64-unknown-linux-musl", "riscv64gc-unknown-linux-gnu"]
 
   def decode(_tokenizer, _ids, _skip_special_tokens), do: err()
   def decode_batch(_tokenizer, _ids, _skip_special_tokens), do: err()