Update to finalfusion 0.9

Author: danieldk

Cargo.lock

@@ -21,7 +21,7 @@ features = ["extension-module"]
 [dependencies]
 itertools = "0.8"
 failure = "0.1"
-finalfusion = "0.8.2"
+finalfusion = "0.9"
 libc = "0.2"
 ndarray = "0.12"
 numpy = "0.6"

Cargo.toml

@@ -652,58 +652,58 @@ item that gets emitted.
 
 
 # end
-# finalfusion-0.8.0
+# finalfusion-0.9.0
 
-  crates.finalfusion."0.8.0" = deps: { features?(features_."finalfusion"."0.8.0" deps {}) }: buildRustCrate {
+  crates.finalfusion."0.9.0" = deps: { features?(features_."finalfusion"."0.9.0" deps {}) }: buildRustCrate {
     crateName = "finalfusion";
-    version = "0.8.0";
+    version = "0.9.0";
     description = "Reader and writer for common word embedding formats";
     homepage = "https://github.com/finalfusion/finalfusion-rust";
     authors = [ "Daniël de Kok <[email protected]>" ];
     edition = "2018";
-    sha256 = "1rbm5c95mxyiq07wzir2xzv0h7ryxfkwli3hnbabra8hpq05dj76";
+    sha256 = "05p31mv12h9168cbi62b6grk7spq6v9g8qr87pxgrhpllz93a2zr";
     dependencies = mapFeatures features ([
-      (crates."byteorder"."${deps."finalfusion"."0.8.0"."byteorder"}" deps)
-      (crates."fnv"."${deps."finalfusion"."0.8.0"."fnv"}" deps)
-      (crates."itertools"."${deps."finalfusion"."0.8.0"."itertools"}" deps)
-      (crates."memmap"."${deps."finalfusion"."0.8.0"."memmap"}" deps)
-      (crates."ndarray"."${deps."finalfusion"."0.8.0"."ndarray"}" deps)
-      (crates."ordered_float"."${deps."finalfusion"."0.8.0"."ordered_float"}" deps)
-      (crates."rand"."${deps."finalfusion"."0.8.0"."rand"}" deps)
-      (crates."rand_xorshift"."${deps."finalfusion"."0.8.0"."rand_xorshift"}" deps)
-      (crates."reductive"."${deps."finalfusion"."0.8.0"."reductive"}" deps)
-      (crates."serde"."${deps."finalfusion"."0.8.0"."serde"}" deps)
-      (crates."toml"."${deps."finalfusion"."0.8.0"."toml"}" deps)
+      (crates."byteorder"."${deps."finalfusion"."0.9.0"."byteorder"}" deps)
+      (crates."fnv"."${deps."finalfusion"."0.9.0"."fnv"}" deps)
+      (crates."itertools"."${deps."finalfusion"."0.9.0"."itertools"}" deps)
+      (crates."memmap"."${deps."finalfusion"."0.9.0"."memmap"}" deps)
+      (crates."ndarray"."${deps."finalfusion"."0.9.0"."ndarray"}" deps)
+      (crates."ordered_float"."${deps."finalfusion"."0.9.0"."ordered_float"}" deps)
+      (crates."rand"."${deps."finalfusion"."0.9.0"."rand"}" deps)
+      (crates."rand_xorshift"."${deps."finalfusion"."0.9.0"."rand_xorshift"}" deps)
+      (crates."reductive"."${deps."finalfusion"."0.9.0"."reductive"}" deps)
+      (crates."serde"."${deps."finalfusion"."0.9.0"."serde"}" deps)
+      (crates."toml"."${deps."finalfusion"."0.9.0"."toml"}" deps)
     ]);
   };
-  features_."finalfusion"."0.8.0" = deps: f: updateFeatures f (rec {
-    byteorder."${deps.finalfusion."0.8.0".byteorder}".default = true;
-    finalfusion."0.8.0".default = (f.finalfusion."0.8.0".default or true);
-    fnv."${deps.finalfusion."0.8.0".fnv}".default = true;
-    itertools."${deps.finalfusion."0.8.0".itertools}".default = true;
-    memmap."${deps.finalfusion."0.8.0".memmap}".default = true;
-    ndarray."${deps.finalfusion."0.8.0".ndarray}".default = true;
-    ordered_float."${deps.finalfusion."0.8.0".ordered_float}".default = true;
-    rand."${deps.finalfusion."0.8.0".rand}".default = true;
-    rand_xorshift."${deps.finalfusion."0.8.0".rand_xorshift}".default = true;
-    reductive."${deps.finalfusion."0.8.0".reductive}".default = true;
+  features_."finalfusion"."0.9.0" = deps: f: updateFeatures f (rec {
+    byteorder."${deps.finalfusion."0.9.0".byteorder}".default = true;
+    finalfusion."0.9.0".default = (f.finalfusion."0.9.0".default or true);
+    fnv."${deps.finalfusion."0.9.0".fnv}".default = true;
+    itertools."${deps.finalfusion."0.9.0".itertools}".default = true;
+    memmap."${deps.finalfusion."0.9.0".memmap}".default = true;
+    ndarray."${deps.finalfusion."0.9.0".ndarray}".default = true;
+    ordered_float."${deps.finalfusion."0.9.0".ordered_float}".default = true;
+    rand."${deps.finalfusion."0.9.0".rand}".default = true;
+    rand_xorshift."${deps.finalfusion."0.9.0".rand_xorshift}".default = true;
+    reductive."${deps.finalfusion."0.9.0".reductive}".default = true;
     serde = fold recursiveUpdate {} [
-      { "${deps.finalfusion."0.8.0".serde}"."derive" = true; }
-      { "${deps.finalfusion."0.8.0".serde}".default = true; }
+      { "${deps.finalfusion."0.9.0".serde}"."derive" = true; }
+      { "${deps.finalfusion."0.9.0".serde}".default = true; }
     ];
-    toml."${deps.finalfusion."0.8.0".toml}".default = true;
+    toml."${deps.finalfusion."0.9.0".toml}".default = true;
   }) [
-    (if deps."finalfusion"."0.8.0" ? "byteorder" then features_.byteorder."${deps."finalfusion"."0.8.0"."byteorder" or ""}" deps else {})
-    (if deps."finalfusion"."0.8.0" ? "fnv" then features_.fnv."${deps."finalfusion"."0.8.0"."fnv" or ""}" deps else {})
-    (if deps."finalfusion"."0.8.0" ? "itertools" then features_.itertools."${deps."finalfusion"."0.8.0"."itertools" or ""}" deps else {})
-    (if deps."finalfusion"."0.8.0" ? "memmap" then features_.memmap."${deps."finalfusion"."0.8.0"."memmap" or ""}" deps else {})
-    (if deps."finalfusion"."0.8.0" ? "ndarray" then features_.ndarray."${deps."finalfusion"."0.8.0"."ndarray" or ""}" deps else {})
-    (if deps."finalfusion"."0.8.0" ? "ordered_float" then features_.ordered_float."${deps."finalfusion"."0.8.0"."ordered_float" or ""}" deps else {})
-    (if deps."finalfusion"."0.8.0" ? "rand" then features_.rand."${deps."finalfusion"."0.8.0"."rand" or ""}" deps else {})
-    (if deps."finalfusion"."0.8.0" ? "rand_xorshift" then features_.rand_xorshift."${deps."finalfusion"."0.8.0"."rand_xorshift" or ""}" deps else {})
-    (if deps."finalfusion"."0.8.0" ? "reductive" then features_.reductive."${deps."finalfusion"."0.8.0"."reductive" or ""}" deps else {})
-    (if deps."finalfusion"."0.8.0" ? "serde" then features_.serde."${deps."finalfusion"."0.8.0"."serde" or ""}" deps else {})
-    (if deps."finalfusion"."0.8.0" ? "toml" then features_.toml."${deps."finalfusion"."0.8.0"."toml" or ""}" deps else {})
+    (if deps."finalfusion"."0.9.0" ? "byteorder" then features_.byteorder."${deps."finalfusion"."0.9.0"."byteorder" or ""}" deps else {})
+    (if deps."finalfusion"."0.9.0" ? "fnv" then features_.fnv."${deps."finalfusion"."0.9.0"."fnv" or ""}" deps else {})
+    (if deps."finalfusion"."0.9.0" ? "itertools" then features_.itertools."${deps."finalfusion"."0.9.0"."itertools" or ""}" deps else {})
+    (if deps."finalfusion"."0.9.0" ? "memmap" then features_.memmap."${deps."finalfusion"."0.9.0"."memmap" or ""}" deps else {})
+    (if deps."finalfusion"."0.9.0" ? "ndarray" then features_.ndarray."${deps."finalfusion"."0.9.0"."ndarray" or ""}" deps else {})
+    (if deps."finalfusion"."0.9.0" ? "ordered_float" then features_.ordered_float."${deps."finalfusion"."0.9.0"."ordered_float" or ""}" deps else {})
+    (if deps."finalfusion"."0.9.0" ? "rand" then features_.rand."${deps."finalfusion"."0.9.0"."rand" or ""}" deps else {})
+    (if deps."finalfusion"."0.9.0" ? "rand_xorshift" then features_.rand_xorshift."${deps."finalfusion"."0.9.0"."rand_xorshift" or ""}" deps else {})
+    (if deps."finalfusion"."0.9.0" ? "reductive" then features_.reductive."${deps."finalfusion"."0.9.0"."reductive" or ""}" deps else {})
+    (if deps."finalfusion"."0.9.0" ? "serde" then features_.serde."${deps."finalfusion"."0.9.0"."serde" or ""}" deps else {})
+    (if deps."finalfusion"."0.9.0" ? "toml" then features_.toml."${deps."finalfusion"."0.9.0"."toml" or ""}" deps else {})
   ];
 
 

nix/crates-io.nix

@@ -120,7 +120,7 @@ rec {
     syn = "0.15.42";
     synstructure = "0.10.2";
   };
-  deps.finalfusion."0.8.0" = {
+  deps.finalfusion."0.9.0" = {
     byteorder = "1.3.2";
     fnv = "1.0.6";
     itertools = "0.8.0";
@@ -135,7 +135,7 @@ rec {
   };
   deps.finalfusion_python."0.4.0" = {
     failure = "0.1.5";
-    finalfusion = "0.8.0";
+    finalfusion = "0.9.0";
     itertools = "0.8.0";
     libc = "0.2.60";
     ndarray = "0.12.1";

nix/finalfusion-python.nix

@@ -34,6 +34,24 @@ pub struct PyEmbeddings {
     embeddings: Rc<RefCell<EmbeddingsWrap>>,
 }
 
+impl PyEmbeddings {
+    /// Copy storage to an array.
+    ///
+    /// This should only be used for storage types that do not provide
+    /// an ndarray view that can be copied trivially, such as quantized
+    /// storage.
+    fn copy_storage_to_array(storage: &Storage) -> Array2<f32> {
+        let (rows, dims) = storage.shape();
+
+        let mut array = Array2::<f32>::zeros((rows, dims));
+        for idx in 0..rows {
+            array.row_mut(idx).assign(&storage.embedding(idx).as_view());
+        }
+
+        array
+    }
+}
+
 #[pymethods]
 impl PyEmbeddings {
     /// Load embeddings from the given `path`.
@@ -216,17 +234,16 @@ impl PyEmbeddings {
                 StorageWrap::MmapArray(mmap) => mmap.view(),
                 StorageWrap::NdArray(array) => array.0.view(),
                 StorageWrap::QuantizedArray(quantized) => {
-                    let (rows, dims) = quantized.shape();
-                    let mut array = Array2::<f32>::zeros((rows, dims));
-                    for idx in 0..rows {
-                        array
-                            .row_mut(idx)
-                            .assign(&quantized.embedding(idx).as_view());
-                    }
+                    let array = Self::copy_storage_to_array(quantized);
+                    return array.to_pyarray(gil.python()).to_owned();
+                }
+                StorageWrap::MmapQuantizedArray(quantized) => {
+                    let array = Self::copy_storage_to_array(quantized);
                     return array.to_pyarray(gil.python()).to_owned();
                 }
             },
         };
+
         matrix_view.to_pyarray(gil.python()).to_owned()
     }
 

src/embeddings.rs

@@ -32,7 +32,7 @@ impl PyIterProtocol for PyEmbeddingIterator {
         let embeddings = slf.embeddings.borrow();
         let vocab = embeddings.vocab();
 
-        if slf.idx < vocab.len() {
+        if slf.idx < vocab.words_len() {
             let word = vocab.words()[slf.idx].to_string();
             let embed = embeddings.storage().embedding(slf.idx);
             let norm = embeddings.norms().map(|n| n.0[slf.idx]).unwrap_or(1.);

src/iter.rs

@@ -62,7 +62,7 @@ impl PyVocab {
 impl PySequenceProtocol for PyVocab {
     fn __len__(&self) -> PyResult<usize> {
         let embeds = self.embeddings.borrow();
-        Ok(embeds.vocab().len())
+        Ok(embeds.vocab().words_len())
     }
 
     fn __getitem__(&self, idx: isize) -> PyResult<String> {