Skip to content

Commit 9e21cf5

Browse files
committed
rename to fastokens
1 parent dbc6a55 commit 9e21cf5

File tree

2 files changed

+2
-2
lines changed

2 files changed

+2
-2
lines changed

lib/llm/src/tokenizers.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
22
// SPDX-License-Identifier: Apache-2.0
33

4-
pub mod fast;
4+
pub mod fastokens;
55
pub mod hf;
66
pub mod tiktoken;
77

@@ -16,7 +16,7 @@ use std::{ops::Deref, path::Path};
1616
use crate::protocols::TokenIdType;
1717
pub use anyhow::{Error, Result};
1818

19-
pub use fast::FastTokenizer;
19+
pub use fastokens::FastTokenizer;
2020
pub use hf::HuggingFaceTokenizer;
2121
pub use tiktoken::TikTokenTokenizer;
2222

0 commit comments

Comments
 (0)