feat: add rank attribute to Entry for storing word frequencies (#1210)

Nickersoft · web-flow · commit 2b2439a4dcb8 · 2025-05-10T00:50:34.000-04:00
diff --git a/examples/example1.xml b/examples/example1.xml
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <dictionary name="Example Dictionary 1"
   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../odict.xsd">
-  <entry term="run">
+  <entry rank="100" term="run">
     <ety description="Latin root">
       <sense pos="v">
         <group description="A number of verb senses">
diff --git a/lib/src/format/json/entry.rs b/lib/src/format/json/entry.rs
@@ -10,6 +10,9 @@ use super::{EntryRefJSON, EtymologyJSON};
 pub struct EntryJSON {
     pub term: String,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub rank: Option<u32>,
+
     #[serde(skip_serializing_if = "Option::is_none")]
     pub see_also: Option<EntryRefJSON>,
 
diff --git a/lib/src/models/entry.rs b/lib/src/models/entry.rs
@@ -10,6 +10,10 @@ serializable! {
     #[serde(rename = "@term")]
     pub term: String,
 
+    #[serde(rename = "@rank")]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub rank: Option<u32>,
+
     #[serde(rename = "@see")]
     #[serde(skip_serializing_if = "Option::is_none")]
     pub see_also: Option<EntryRef>,
diff --git a/lib/tests/merge.rs b/lib/tests/merge.rs
@@ -13,6 +13,7 @@ mod merge_tests {
             name: None,
             entries: hash_map! {
               String::from("dog") => Entry {
+                rank: None,
                 media: vec![],
                 term: "dog".to_string(),
                 see_also: None,
@@ -52,6 +53,7 @@ mod merge_tests {
             entries: hash_map! {
               "cat".to_string()=>
               Entry {
+                  rank: None,
                   media: vec![],
                   see_also: None,
                   term: "cat".to_string(),
@@ -89,6 +91,7 @@ mod merge_tests {
             name: None,
             entries: hash_map! {
               "dog".to_string() => Entry {
+                  rank: None,
                   see_also: None,
                   media: vec![],
                   term: "dog".to_string(),
@@ -120,6 +123,7 @@ mod merge_tests {
               },
               "cat".to_string() => Entry {
                   see_also: None,
+                  rank: None,
                   media: vec![],
                   term: "cat".to_string(),
                   etymologies: vec![Etymology {
diff --git a/lib/tests/resolve.rs b/lib/tests/resolve.rs
@@ -15,6 +15,7 @@ mod resolve_tests {
             name: None,
             entries: hash_map! {
               String::from("dog") => Entry {
+                rank: None,
                 media: vec![],
                 term: "dog".to_string(),
                 see_also: None,
diff --git a/lib/tests/snapshots/dump__dump_tests__json.snap b/lib/tests/snapshots/dump__dump_tests__json.snap
@@ -89,6 +89,7 @@ expression: dumped
     },
     "run": {
       "term": "run",
+      "rank": 100,
       "etymologies": [
         {
           "description": "Latin root",
diff --git a/lib/tests/tags.rs b/lib/tests/tags.rs
@@ -102,6 +102,7 @@ fn test_form_with_tags() {
     // Create an entry with the etymology
     let entry = Entry {
         media: vec![],
+        rank: None,
         term: "word".to_string(),
         see_also: None,
         etymologies: vec![etymology],
diff --git a/lib/tests/translation.rs b/lib/tests/translation.rs
@@ -68,6 +68,7 @@ fn test_translation_in_entry() {
     // Create an entry with the etymology
     let entry = Entry {
         media: vec![],
+        rank: None,
         term: "hello".to_string(),
         see_also: None,
         etymologies: vec![etymology],
diff --git a/node/index.d.ts b/node/index.d.ts
@@ -35,6 +35,7 @@ export interface DictionaryOptions {
 
 export interface Entry {
   term: string;
+  rank?: number;
   seeAlso?: string;
   etymologies: Array<Etymology>;
   media: Array<MediaURL>;
diff --git a/node/src/types/entry.rs b/node/src/types/entry.rs
@@ -11,6 +11,7 @@ use super::media_url::MediaURL;
 #[convert(from(odict::Entry))]
 pub struct Entry {
   pub term: String,
+  pub rank: Option<u32>,
   pub see_also: Option<String>,
   pub etymologies: Vec<Etymology>,
   pub media: Vec<MediaURL>,
diff --git a/node/tests/__snapshots__/dictionary.spec.ts.snap b/node/tests/__snapshots__/dictionary.spec.ts.snap
@@ -45,6 +45,7 @@ exports[`Dictionary > can index and search a dictionary 1`] = `
       },
     ],
     "media": [],
+    "rank": 100,
     "term": "run",
   },
 ]
diff --git a/python/src/types/entry.rs b/python/src/types/entry.rs
@@ -13,6 +13,8 @@ pub struct Entry {
     #[pyo3(get)]
     pub term: String,
     #[pyo3(get)]
+    pub rank: Option<u32>,
+    #[pyo3(get)]
     pub see_also: Option<String>,
     #[pyo3(get)]
     pub etymologies: Vec<Etymology>,
diff --git a/python/tests/__snapshots__/test_lookup.ambr b/python/tests/__snapshots__/test_lookup.ambr
@@ -1,26 +1,26 @@
 # serializer version: 1
 # name: test_index_and_search
   list([
-    Entry { term: "run", see_also: None, etymologies: [Etymology { id: None, pronunciations: [], description: Some("Latin root"), senses: [("n", Sense { pos: "n", lemma: None, definitions: [Left(Definition { id: None, value: "(vertebrates) To move swiftly", examples: [Example { value: "The dog runs after the cat.", translations: [], pronunciations: [] }, Example { value: "The horse ran away.", translations: [], pronunciations: [] }], notes: [] }), Left(Definition { id: None, value: "Act or instance of hurrying (to or from a place) (not necessarily by foot); dash or errand, trip.", examples: [], notes: [] }), Left(Definition { id: None, value: "A pleasure trip.", examples: [], notes: [] }), Left(Definition { id: None, value: "Flight, instance or period of fleeing.", examples: [], notes: [] }), Left(Definition { id: None, value: "Migration (of fish).", examples: [], notes: [] }), Left(Definition { id: None, value: "A group of fish that migrate, or ascend a river for the purpose of spawning.", examples: [], notes: [] })], tags: [], translations: [], forms: [] }), ("v", Sense { pos: "v", lemma: None, definitions: [Right(Group { id: None, description: "A number of verb senses", definitions: [Definition { id: None, value: "(vertebrates) To move swiftly (figuratively)", examples: [Example { value: "The dog runs after the cat.", translations: [], pronunciations: [] }, Example { value: "The horse ran away.", translations: [], pronunciations: [] }], notes: [] }, Definition { id: None, value: "(fluids) To flow.", examples: [], notes: [] }, Definition { id: None, value: "(nautical, of a vessel) To sail before the wind, in distinction from reaching or sailing close-hauled.", examples: [], notes: [] }, Definition { id: None, value: "(social) To carry out an activity.", examples: [], notes: [] }, Definition { id: None, value: "To extend or persist, statically or dynamically, through space or time.", examples: [], notes: [] }, Definition { id: None, value: "(transitive) To execute or carry out a plan, procedure or program.", examples: [], notes: [] }] })], tags: [], translations: [], forms: [] })] }], media: [] },
+    Entry { term: "run", rank: Some(100), see_also: None, etymologies: [Etymology { id: None, pronunciations: [], description: Some("Latin root"), senses: [("n", Sense { pos: "n", lemma: None, definitions: [Left(Definition { id: None, value: "(vertebrates) To move swiftly", examples: [Example { value: "The dog runs after the cat.", translations: [], pronunciations: [] }, Example { value: "The horse ran away.", translations: [], pronunciations: [] }], notes: [] }), Left(Definition { id: None, value: "Act or instance of hurrying (to or from a place) (not necessarily by foot); dash or errand, trip.", examples: [], notes: [] }), Left(Definition { id: None, value: "A pleasure trip.", examples: [], notes: [] }), Left(Definition { id: None, value: "Flight, instance or period of fleeing.", examples: [], notes: [] }), Left(Definition { id: None, value: "Migration (of fish).", examples: [], notes: [] }), Left(Definition { id: None, value: "A group of fish that migrate, or ascend a river for the purpose of spawning.", examples: [], notes: [] })], tags: [], translations: [], forms: [] }), ("v", Sense { pos: "v", lemma: None, definitions: [Right(Group { id: None, description: "A number of verb senses", definitions: [Definition { id: None, value: "(vertebrates) To move swiftly (figuratively)", examples: [Example { value: "The dog runs after the cat.", translations: [], pronunciations: [] }, Example { value: "The horse ran away.", translations: [], pronunciations: [] }], notes: [] }, Definition { id: None, value: "(fluids) To flow.", examples: [], notes: [] }, Definition { id: None, value: "(nautical, of a vessel) To sail before the wind, in distinction from reaching or sailing close-hauled.", examples: [], notes: [] }, Definition { id: None, value: "(social) To carry out an activity.", examples: [], notes: [] }, Definition { id: None, value: "To extend or persist, statically or dynamically, through space or time.", examples: [], notes: [] }, Definition { id: None, value: "(transitive) To execute or carry out a plan, procedure or program.", examples: [], notes: [] }] })], tags: [], translations: [], forms: [] })] }], media: [] },
   ])
 # ---
 # name: test_lookup
   list([
     LookupResult(
       directed_from=None,
-      entry=Entry { term: "cat", see_also: None, etymologies: [Etymology { id: None, pronunciations: [], description: Some("Latin root"), senses: [("n", Sense { pos: "n", lemma: None, definitions: [Left(Definition { id: None, value: "a cat", examples: [Example { value: "There goes a cat!", translations: [], pronunciations: [] }], notes: [Note { id: None, value: "Some definition note", examples: [Example { value: "Some example", translations: [], pronunciations: [] }] }] })], tags: [], translations: [], forms: [] })] }], media: [] },
+      entry=Entry { term: "cat", rank: None, see_also: None, etymologies: [Etymology { id: None, pronunciations: [], description: Some("Latin root"), senses: [("n", Sense { pos: "n", lemma: None, definitions: [Left(Definition { id: None, value: "a cat", examples: [Example { value: "There goes a cat!", translations: [], pronunciations: [] }], notes: [Note { id: None, value: "Some definition note", examples: [Example { value: "Some example", translations: [], pronunciations: [] }] }] })], tags: [], translations: [], forms: [] })] }], media: [] },
     ),
   ])
 # ---
 # name: test_lookup_with_split
   list([
     LookupResult(
       directed_from=None,
-      entry=Entry { term: "cat", see_also: None, etymologies: [Etymology { id: None, pronunciations: [], description: Some("Latin root"), senses: [("n", Sense { pos: "n", lemma: None, definitions: [Left(Definition { id: None, value: "a cat", examples: [Example { value: "There goes a cat!", translations: [], pronunciations: [] }], notes: [Note { id: None, value: "Some definition note", examples: [Example { value: "Some example", translations: [], pronunciations: [] }] }] })], tags: [], translations: [], forms: [] })] }], media: [] },
+      entry=Entry { term: "cat", rank: None, see_also: None, etymologies: [Etymology { id: None, pronunciations: [], description: Some("Latin root"), senses: [("n", Sense { pos: "n", lemma: None, definitions: [Left(Definition { id: None, value: "a cat", examples: [Example { value: "There goes a cat!", translations: [], pronunciations: [] }], notes: [Note { id: None, value: "Some definition note", examples: [Example { value: "Some example", translations: [], pronunciations: [] }] }] })], tags: [], translations: [], forms: [] })] }], media: [] },
     ),
     LookupResult(
       directed_from=None,
-      entry=Entry { term: "dog", see_also: None, etymologies: [Etymology { id: None, pronunciations: [], description: Some("Latin root"), senses: [("un", Sense { pos: "un", lemma: None, definitions: [Left(Definition { id: None, value: "a dog", examples: [], notes: [] })], tags: [], translations: [], forms: [] })] }], media: [] },
+      entry=Entry { term: "dog", rank: None, see_also: None, etymologies: [Etymology { id: None, pronunciations: [], description: Some("Latin root"), senses: [("un", Sense { pos: "un", lemma: None, definitions: [Left(Definition { id: None, value: "a dog", examples: [], notes: [] })], tags: [], translations: [], forms: [] })] }], media: [] },
     ),
   ])
 # ---
diff --git a/python/tests/__snapshots__/test_tokenize.ambr b/python/tests/__snapshots__/test_tokenize.ambr
@@ -1,9 +1,9 @@
 # serializer version: 1
 # name: test_tokenize
   list([
-    Token { lemma: "你好", language: Some("cmn"), entries: [LookupResult { entry: Entry { term: "你", see_also: None, etymologies: [Etymology { id: None, pronunciations: [], description: Some("Latin root"), senses: [("v", Sense { pos: "v", lemma: None, definitions: [Left(Definition { id: None, value: "Lorem ipsum", examples: [], notes: [] })], tags: [], translations: [], forms: [] })] }], media: [] }, directed_from: None }, LookupResult { entry: Entry { term: "好", see_also: None, etymologies: [Etymology { id: None, pronunciations: [], description: Some("Latin root"), senses: [("v", Sense { pos: "v", lemma: None, definitions: [Left(Definition { id: None, value: "Lorem ipsum", examples: [], notes: [] })], tags: [], translations: [], forms: [] })] }], media: [] }, directed_from: None }], kind: "Unknown", script: "Mandarin", start: 0, end: 2 },
+    Token { lemma: "你好", language: Some("cmn"), entries: [LookupResult { entry: Entry { term: "你", rank: None, see_also: None, etymologies: [Etymology { id: None, pronunciations: [], description: Some("Latin root"), senses: [("v", Sense { pos: "v", lemma: None, definitions: [Left(Definition { id: None, value: "Lorem ipsum", examples: [], notes: [] })], tags: [], translations: [], forms: [] })] }], media: [] }, directed_from: None }, LookupResult { entry: Entry { term: "好", rank: None, see_also: None, etymologies: [Etymology { id: None, pronunciations: [], description: Some("Latin root"), senses: [("v", Sense { pos: "v", lemma: None, definitions: [Left(Definition { id: None, value: "Lorem ipsum", examples: [], notes: [] })], tags: [], translations: [], forms: [] })] }], media: [] }, directed_from: None }], kind: "Unknown", script: "Mandarin", start: 0, end: 2 },
     Token { lemma: "！", language: None, entries: [], kind: "Unknown", script: "Hangul", start: 2, end: 3 },
-    Token { lemma: "你", language: Some("cmn"), entries: [LookupResult { entry: Entry { term: "你", see_also: None, etymologies: [Etymology { id: None, pronunciations: [], description: Some("Latin root"), senses: [("v", Sense { pos: "v", lemma: None, definitions: [Left(Definition { id: None, value: "Lorem ipsum", examples: [], notes: [] })], tags: [], translations: [], forms: [] })] }], media: [] }, directed_from: None }], kind: "Unknown", script: "Mandarin", start: 3, end: 4 },
+    Token { lemma: "你", language: Some("cmn"), entries: [LookupResult { entry: Entry { term: "你", rank: None, see_also: None, etymologies: [Etymology { id: None, pronunciations: [], description: Some("Latin root"), senses: [("v", Sense { pos: "v", lemma: None, definitions: [Left(Definition { id: None, value: "Lorem ipsum", examples: [], notes: [] })], tags: [], translations: [], forms: [] })] }], media: [] }, directed_from: None }], kind: "Unknown", script: "Mandarin", start: 3, end: 4 },
     Token { lemma: "是", language: Some("cmn"), entries: [], kind: "Unknown", script: "Mandarin", start: 4, end: 5 },
     Token { lemma: "谁", language: Some("cmn"), entries: [], kind: "Unknown", script: "Mandarin", start: 5, end: 6 },
     Token { lemma: "？", language: None, entries: [], kind: "Unknown", script: "Hangul", start: 6, end: 7 },

Original file line number	Diff line number	Diff line change
`@@ -89,6 +89,7 @@ expression: dumped`
`89`	`89`	`},`
`90`	`90`	`"run": {`
`91`	`91`	`"term": "run",`
	`92`	`+ "rank": 100,`
`92`	`93`	`"etymologies": [`
`93`	`94`	`{`
`94`	`95`	`"description": "Latin root",`