feat(*): string interning (#1313)

Author: Nickersoft

Cargo.lock

@@ -77,6 +77,7 @@ reqwest = { version = "0.12.24", optional = true, default-features = false, feat
   "http2",
   "stream",
 ] }
+strum = { version = "0.27.2", features = ["derive"] }
 reqwest-middleware = { version = "0.4.2", optional = true }
 reqwest-retry = { version = "0.7.0", optional = true }
 futures-util = { version = "0.3", optional = true }
@@ -87,7 +88,7 @@ tantivy = { version = "0.25.0", optional = true }
 dirs = { version = "6.0.0", optional = true }
 # feature=sql
 sea-query = { version = "0.32.7", optional = true }
-strum = { version = "0.27.2", features = ["derive"] }
+rkyv_intern = { git = "https://github.com/TheOpenDictionary/rkyv_intern.git", version = "0.1.0" }
 
 [dev-dependencies]
 criterion = { version = "0.7.0", features = ["async_tokio", "html_reports"] }

lib/Cargo.toml

@@ -21,6 +21,7 @@ mod core;
 mod error;
 mod ext;
 mod odict;
+mod se;
 
 pub mod format;
 pub mod fs;

lib/src/lib.rs

@@ -1,4 +1,4 @@
-use rkyv::with::{AsBox, MapNiche};
+use rkyv_intern::Intern;
 
 use crate::serializable;
 
@@ -8,11 +8,12 @@ serializable! {
   #[derive(Default)]
   pub struct Definition {
     #[serde(rename = "@id")]
-    #[rkyv(with = MapNiche<AsBox>)]
+    #[rkyv(with = rkyv::with::Map<rkyv_intern::Intern>)]
     #[serde(skip_serializing_if = "Option::is_none")]
     pub id: Option<String>,
 
     #[serde(rename = "@value")]
+    #[rkyv(with = Intern)]
     pub value: String,
 
     #[serde(default, rename="example")]

lib/src/schema/definition.rs

@@ -1,11 +1,9 @@
-use rkyv::{
-    deserialize, to_bytes,
-    with::{AsBox, MapNiche},
-};
+use rkyv::deserialize;
+
 use std::collections::HashSet;
 use std::str::FromStr;
 
-use crate::{error::Error, serializable};
+use crate::{error::Error, se::serialize_interned, serializable};
 
 use super::{entry::Entry, id::ID};
 
@@ -17,7 +15,7 @@ serializable! {
       pub id: ID,
 
       #[serde(rename = "@name")]
-      #[rkyv(with = MapNiche<AsBox>)]
+      #[rkyv(with = rkyv::with::Map<rkyv_intern::Intern>)]
       #[serde(skip_serializing_if = "Option::is_none")]
       pub name: Option<String>,
 
@@ -66,8 +64,7 @@ impl FromStr for Dictionary {
 
 impl Dictionary {
     pub(crate) fn serialize(&self) -> crate::Result<Vec<u8>> {
-        let bytes =
-            to_bytes::<rkyv::rancor::Error>(self).map_err(|e| Error::Serialize(e.to_string()))?;
+        let bytes = serialize_interned::<_, rkyv::rancor::Error>(self)?;
         Ok(bytes.to_vec())
     }
 }

lib/src/schema/dictionary.rs

@@ -1,13 +1,13 @@
 use rkyv::{
-    deserialize, to_bytes,
+    deserialize,
     with::{AsBox, MapNiche},
 };
 use std::{
     borrow::Borrow,
     hash::{Hash, Hasher},
 };
 
-use crate::{error::Error, schema::Etymology, serializable};
+use crate::{error::Error, schema::Etymology, se::serialize_interned, serializable};
 
 use super::{EntryRef, MediaURL};
 
@@ -68,9 +68,7 @@ impl Borrow<str> for ArchivedEntry {
 
 impl Entry {
     pub fn serialize(&self) -> crate::Result<Vec<u8>> {
-        let bytes =
-            to_bytes::<rkyv::rancor::Error>(self).map_err(|e| Error::Serialize(e.to_string()))?;
-
+        let bytes = serialize_interned::<_, rkyv::rancor::Error>(self)?;
         Ok(bytes.to_vec())
     }
 }

lib/src/schema/entry.rs

@@ -4,7 +4,10 @@ use crate::serializable;
 
 serializable! {
   #[derive(Default)]
-  pub struct EntryRef(pub String);
+  pub struct EntryRef(
+      #[rkyv(with = rkyv_intern::Intern)]
+      pub String
+  );
 }
 
 impl EntryRef {

lib/src/schema/entry_ref.rs

@@ -1,6 +1,8 @@
 use std::fmt::{Debug, Display};
 
-use super::{FormKind, PartOfSpeech, PronunciationKind};
+use crate::schema::PronunciationKind;
+
+use super::{FormKind, PartOfSpeech};
 
 pub trait EnumIdentifier {
     fn id(&self) -> String;

lib/src/schema/enums.rs

@@ -1,7 +1,5 @@
 use std::collections::HashSet;
 
-use rkyv::with::{AsBox, MapNiche};
-
 use crate::schema::pronunciation::Pronunciation;
 use crate::serializable;
 
@@ -13,7 +11,7 @@ serializable! {
   pub struct Etymology {
     #[serde(rename = "@id")]
     #[serde(skip_serializing_if = "Option::is_none")]
-    #[rkyv(with = MapNiche<AsBox>)]
+    #[rkyv(with = rkyv::with::Map<rkyv_intern::Intern>)]
     pub id: Option<String>,
 
     #[serde(default, rename = "pronunciation")]
@@ -24,7 +22,7 @@ serializable! {
     pub senses: HashSet<Sense>,
 
     #[serde(rename = "@description")]
-    #[rkyv(with = MapNiche<AsBox>)]
+    #[rkyv(with = rkyv::with::Map<rkyv_intern::Intern>)]
     #[serde(skip_serializing_if = "Option::is_none")]
     pub description: Option<String>,
   }

lib/src/schema/etymology.rs

@@ -6,6 +6,7 @@ serializable! {
   #[serde(rename = "example")]
   pub struct Example {
     #[serde(rename = "@value")]
+    #[rkyv(with = rkyv_intern::Intern)]
     pub value: String,
 
     #[serde(default, rename = "translation")]