duplodocus/Cargo.toml at main · allenai/duplodocus · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
[package]
name = "duplodocus"
version = "0.1.0"
edition = "2021"
repository = "https://github.com/allenai/duplodocus"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
anyhow = "1.0"
serde_json = "1.0.97"
zstd = "0.13.1"
glob = "0.3.1"
rand = "0.8"
clap = { version = "4.1.11", features = ["derive"] }
indicatif = "0.17.8"
flate2 = "1.0"
serde = { version = "1.0.198", features = ["derive"] }
tiktoken-rs = "0.5.8"
rustc-hash = "1.1.0"
regex = "1.10.4"
ndarray = "0.15.6"
rand_chacha = "0.3.1"
sha2 = "0.9"
bytemuck = {version = "1.16.0", features = ["extern_crate_alloc"]}
dashmap = { version = "5.5.3", features = ["rayon"] }
rayon = "1.10.0"
byteorder = "1.5.0"
serde_yaml = "0.9.34"
unicode-segmentation = "1.12.0"
ahash = "0.8.11"
mj_io = "0.1.7"
num-traits = "0.2.19"
xxhash-rust = { version = "0.8", features = ["xxh3"] }
gjson = "0.8.1"
smallvec = "1.15.1"
sysinfo = "0.37.2"
memmap2 = "0.9.9"
src = "0.0.6"


# The profile that 'dist' will build with
[profile.dist]
inherits = "release"
lto = "thin"