Updates (#12)

* wip

* Clippy

* Allow upgrading from in memory mmap to on disk mmap

Author: Keats

.github/workflows/ci.yml

@@ -10,13 +10,13 @@ jobs:
         build: [pinned, stable]
         include:
         - build: pinned
-          os: ubuntu-18.04
-          rust: 1.40.0
+          os: ubuntu-22.04
+          rust: 1.60.0
         - build: stable
-          os: ubuntu-18.04
+          os: ubuntu-22.04
           rust: stable
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Install Rust
       uses: hecrj/setup-rust-action@v1
       with:

Cargo.toml

@@ -1,17 +1,18 @@
 [package]
 name = "mmap-bitvec"
-version = "0.3.1"
+version = "0.4.0"
 authors = ["Roderick Bovee <[email protected]>"]
 autobenches = false
 edition = "2018"
 
 [dependencies]
-memmap = "0.7.0"
+memmap2 = "0.5"
 murmurhash3 = "0.0.5"
 once_cell = "1.3.1"
 
 [dev-dependencies]
 criterion = "0.3"
+tempfile = "3.3.0"
 
 [[bench]]
 name = "benchmark"

benches/benchmark.rs

@@ -1,12 +1,12 @@
 extern crate criterion;
-extern crate memmap;
+extern crate memmap2;
 extern crate mmap_bitvec;
 
 use std::fs::OpenOptions;
 use std::mem::transmute;
 use std::ops::Range;
 
-use memmap::{MmapMut, MmapOptions};
+use memmap2::{MmapMut, MmapOptions};
 use mmap_bitvec::{combinatorial::rank, BitVector, MmapBitVec};
 
 use criterion::{criterion_group, criterion_main, Criterion};

src/bitvec.rs

@@ -25,7 +25,7 @@ pub trait BitVector {
     }
 
     fn get_range(&self, r: Range<usize>) -> u128 {
-        if r.end - r.start > 128usize {
+        if r.end - r.start > 128 {
             panic!("Range too large (>128)")
         } else if r.end > self.size() {
             panic!("Range ends outside of BitVec")
@@ -90,7 +90,7 @@ impl_bitvector!(u128, 128);
 
 impl BitVector for &[u8] {
     fn get(&self, i: usize) -> bool {
-        if i / 8 >= self.len() {
+        if i / 8 >= self.size() {
             panic!("Invalid bit vector index");
         }
         self[i / 8] >> (8 - i % 8) & 1 == 1

src/combinatorial.rs

@@ -6,6 +6,7 @@ const MARKER_TABLE_SIZE: usize = 200_000;
 
 // TODO: replace with const fn when it is possible
 // (for and if are not allowed in const fn on current stable)
+// https://github.com/rust-lang/rust/issues/87575
 static MARKER_TABLES: Lazy<HashMap<u8, Vec<u128>>> = Lazy::new(|| {
     let mut m = HashMap::new();
     for k in 1..10u8 {
@@ -102,7 +103,7 @@ pub fn choose(n: u64, k: u8) -> u64 {
 #[inline]
 fn next_rank(marker: u128) -> u128 {
     if marker == 0 {
-        panic!("WOOPS");
+        unreachable!("WOOPS");
     }
     let t = marker | (marker - 1);
     (t + 1) | (((!t & (t + 1)) - 1) >> (marker.trailing_zeros() + 1))

src/mmap_bitvec.rs

@@ -5,7 +5,7 @@ use std::mem::transmute;
 use std::ops::Range;
 use std::path::Path;
 
-use memmap::{Mmap, MmapMut, MmapOptions};
+use memmap2::{Mmap, MmapMut, MmapOptions};
 
 use crate::bitvec::BitVector;
 
@@ -59,22 +59,20 @@ pub struct MmapBitVec {
     pub mmap: CommonMmap,
     pub size: usize,
     header: Box<[u8]>,
+    is_anon: bool,
 }
 
 impl MmapBitVec {
     /// Creates a new `MmapBitVec` file
     ///
     /// The overall size of bit vector (in bits) and a fixed-size header must
     /// also be provided (although the header can be 0-length).
-    pub fn create<P>(
+    pub fn create<P: AsRef<Path>>(
         filename: P,
         size: usize,
         magic: [u8; 2],
         header: &[u8],
-    ) -> Result<Self, io::Error>
-    where
-        P: AsRef<Path>,
-    {
+    ) -> Result<Self, io::Error> {
         assert!(
             header.len() < 65_536,
             "Headers longer than 65636 bytes not supported"
@@ -94,17 +92,18 @@ impl MmapBitVec {
         // file.seek(io::SeekFrom::Start(0))?;
 
         file.write_all(&magic)?;
-        let serialized_header_size: [u8; 2] = unsafe { transmute((header.len() as u16).to_be()) };
+        let serialized_header_size: [u8; 2] = (header.len() as u16).to_be_bytes();
         file.write_all(&serialized_header_size)?;
         file.write_all(header)?;
-        let serialized_size: [u8; 8] = unsafe { transmute((size as u64).to_be()) };
+        let serialized_size: [u8; 8] = (size as u64).to_be_bytes();
         file.write_all(&serialized_size)?;
 
         let mmap = unsafe { MmapOptions::new().offset(total_header_size).map_mut(&file) }?;
         Ok(MmapBitVec {
             mmap: CommonMmap::MmapMut(mmap),
             size,
             header: header.to_vec().into_boxed_slice(),
+            is_anon: false,
         })
     }
 
@@ -182,6 +181,7 @@ impl MmapBitVec {
             mmap,
             size: size as usize,
             header: header.into_boxed_slice(),
+            is_anon: false,
         })
     }
 
@@ -201,6 +201,7 @@ impl MmapBitVec {
             mmap: CommonMmap::Mmap(mmap),
             size: byte_size * 8,
             header: Box::new([]),
+            is_anon: false,
         })
     }
 
@@ -215,9 +216,33 @@ impl MmapBitVec {
             mmap: CommonMmap::MmapMut(mmap),
             size,
             header: vec![].into_boxed_slice(),
+            is_anon: true,
         })
     }
 
+    /// Converts an in-memory mmap bitvector to a file-backed one.
+    /// This is a no-op if the mmap is already file-backed.
+    /// Returns the new mmap after flushing.
+    pub fn into_mmap_file<P: AsRef<Path>>(
+        self,
+        filename: P,
+        magic: [u8; 2],
+        header: &[u8],
+    ) -> Result<Self, io::Error> {
+        if !self.is_anon {
+            return Ok(self);
+        }
+        let mut file_mmap = MmapBitVec::create(filename, self.size, magic, header)?;
+
+        // Not super efficient
+        for i in 0..self.size {
+            file_mmap.set(i, self.get(i));
+        }
+        file_mmap.mmap.flush()?;
+
+        Ok(file_mmap)
+    }
+
     // Returns the header
     pub fn header(&self) -> &[u8] {
         &self.header
@@ -533,10 +558,7 @@ impl BitVector for MmapBitVec {
         // u64 is stored in the "right" order in memory
         let main_chunk = (x << (128 - size_main)).to_be();
 
-        let bytes: [u8; 16];
-        unsafe {
-            bytes = transmute(main_chunk);
-        }
+        let bytes: [u8; 16] = main_chunk.to_le_bytes();
         for (byte_idx, byte) in ((byte_idx_st + 1)..byte_idx_en).zip(bytes.iter()) {
             unsafe {
                 *mmap.add(byte_idx) |= *byte;
@@ -783,4 +805,20 @@ mod test {
         assert_eq!(b.select(1, 0), Some(7));
         assert_eq!(b.select(3, 0), Some(127));
     }
+
+    #[test]
+    fn can_convert_memory_to_file() {
+        let mut b = MmapBitVec::from_memory(128).unwrap();
+        b.set(7, true);
+        b.set(56, true);
+        b.set(127, true);
+        let dir = tempfile::tempdir().unwrap();
+        let f = b
+            .into_mmap_file(dir.path().join("test"), *b"!!", &[])
+            .unwrap();
+        assert_eq!(f.get(7), true);
+        assert_eq!(f.get(56), true);
+        assert_eq!(f.get(127), true);
+        assert_eq!(f.get(10), false);
+    }
 }