Merge pull request #1337 from spacejam/tyler_debugging

various CI debugging efforts

Author: spacejam

.github/workflows/test.yml

@@ -32,14 +32,15 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - name: Cache target
-      uses: actions/cache@v1
+      uses: actions/cache@v2
       env:
-        cache-name: cache-target
+        cache-name: cache-default-target-and-lockfile
       with:
-        path: target
-        key: ${{ runner.os }}-${{ env.cache-name }}
-        restore-keys: |
-          ${{ runner.os }}-
+        path: |
+          target
+          Cargo.lock
+          ~/.rustup
+        key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('**/Cargo.toml') }}
     - name: linux coredump setup
       if: ${{ runner.os == 'linux' }}
       run: |
@@ -53,23 +54,25 @@ jobs:
     - uses: actions/upload-artifact@v2
       if: ${{ failure() && runner.os == 'linux' }}
       with:
-        name: core-dump-upload
+        name: linux-core-dumps
         path: |
           ./core-dumps/*
+          ./target/release/deps/test_*
   examples:
     name: Example Tests
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v1
     - name: Cache target
-      uses: actions/cache@v1
+      uses: actions/cache@v2
       env:
-        cache-name: cache-target
+        cache-name: cache-examples-target-and-lockfile
       with:
-        path: target
-        key: ${{ runner.os }}-${{ env.cache-name }}
-        restore-keys: |
-          ${{ runner.os }}-
+        path: |
+          target
+          Cargo.lock
+          ~/.rustup
+        key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('**/Cargo.toml') }}
     - name: example tests
       run: |
         rustup update --no-self-update
@@ -88,27 +91,27 @@ jobs:
   burn-in:
     name: Burn In
     env:
-      RUSTFLAGS: "-C target-cpu=native -C link-args=-Wl,-zstack-size=4194304"
       RUST_BACKTRACE: 1
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v1
     - name: Cache target
-      uses: actions/cache@v1
+      uses: actions/cache@v2
       env:
-        cache-name: cache-target
+        cache-name: cache-stress2-target-and-lockfile
       with:
-        path: target
-        key: ${{ runner.os }}-${{ env.cache-name }}
-        restore-keys: |
-          ${{ runner.os }}-
+        path: |
+          benchmarks/stress2/target
+          benchmarks/stress2/Cargo.lock
+        key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('**/Cargo.toml') }}
     - name: burn in
       run: |
         set -eo pipefail
         pushd benchmarks/stress2
         ulimit -c unlimited
         echo "$PWD/core-dumps/corefile-%e-%p-%t" | sudo tee /proc/sys/kernel/core_pattern
         mkdir core-dumps
+        rm -rf default.sled || true
         cargo run --release -- --duration=60
         rm -rf default.sled
     - name: print backtraces with gdb
@@ -127,7 +130,7 @@ jobs:
     - uses: actions/upload-artifact@v2
       if: ${{ failure() }}
       with:
-        name: core-dump-upload
+        name: linux-core-dumps
         path: |
           ./benchmarks/stress2/core-dumps/*
           ./benchmarks/stress2/target/release/stress2
@@ -139,14 +142,15 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - name: Cache rustup
-      uses: actions/cache@v1
+      uses: actions/cache@v2
       env:
-        cache-name: cache-target
+        cache-name: cache-sanitizers-target-and-lockfile
       with:
-        path: ~/.rustup
-        key: ${{ runner.os }}-${{ env.cache-name }}
-        restore-keys: |
-          ${{ runner.os }}-
+        path: |
+          ~/.rustup
+          benchmarks/stress2/target
+          benchmarks/stress2/Cargo.lock
+        key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('**/Cargo.toml') }}
     - name: sanitizers
       run: |
         set -eo pipefail

CHANGELOG.md

@@ -48,9 +48,10 @@
   platforms and 512mb on 32-bit platforms.
 * #1281 `Config`'s `cache_capacity` is now a usize, as u64
   doesn't make sense for things that must fit in memory anyway.
-* #1319 Bump MSRV to 1.46.
 * #1314 `Subscriber::next_timeout` now requires a mutable self
   reference.
+* #1337 Bump MSRV to 1.48.
+
 
 ## Bug Fixes
 

Cargo.toml

@@ -30,7 +30,8 @@ default = []
 # internal testing use only. It injects many delays and performs several
 # test-only configurations that cause performance to drop significantly.
 # It will cause your tests to take much more time, and possibly time out etc...
-testing = ["event_log", "lock_free_delays", "compression", "failpoints", "backtrace"]
+testing = ["event_log", "lock_free_delays", "light_testing"]
+light_testing = ["compression", "failpoints", "backtrace", "memshred"]
 compression = ["zstd"]
 lock_free_delays = []
 failpoints = []
@@ -44,10 +45,11 @@ io_uring = ["rio"]
 docs = []
 miri_optimizations = []
 mutex = []
+memshred = []
 
 [dependencies]
 libc = "0.2.81"
-zstd = { version = "0.7.0", optional = true }
+zstd = { version = "0.8", optional = true }
 crc32fast = "1.2.1"
 log = "0.4.11"
 parking_lot = "0.11.1"

README.md

@@ -161,7 +161,7 @@ extreme::run(async move {
 
 # minimum supported Rust version (MSRV)
 
-We support Rust 1.46.0 and up.
+We support Rust 1.48.0 and up.
 
 # architecture
 

benchmarks/stress2/Cargo.toml

@@ -24,6 +24,7 @@ measure_allocs = ["sled/measure_allocs"]
 jemalloc = ["jemallocator"]
 logging = ["env_logger", "log", "color-backtrace"]
 dh = ["dhat"]
+memshred = []
 
 [dependencies]
 rand = "0.7.3"

benchmarks/stress2/src/main.rs

@@ -6,18 +6,47 @@ use std::{
     thread,
 };
 
-#[cfg(feature = "jemalloc")]
-use jemallocator::Jemalloc;
-
 #[cfg(feature = "dh")]
 use dhat::{Dhat, DhatAlloc};
 
 use num_format::{Locale, ToFormattedString};
 use rand::{thread_rng, Rng};
 
-#[global_allocator]
 #[cfg(feature = "jemalloc")]
-static ALLOCATOR: Jemalloc = Jemalloc;
+mod alloc {
+    use std::alloc::Layout;
+    use jemallocator::Jemalloc;
+
+    #[global_allocator]
+    static ALLOCATOR: Jemalloc = Jemalloc;
+}
+
+#[cfg(feature = "memshred")]
+mod alloc {
+    use std::alloc::{System, Layout};
+
+    #[global_allocator]
+    static ALLOCATOR: Alloc = Alloc;
+
+    #[derive(Default, Debug, Clone, Copy)]
+    struct Alloc;
+
+    unsafe impl std::alloc::GlobalAlloc for Alloc {
+        unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
+            let ret = System.alloc(layout);
+            assert_ne!(ret, std::ptr::null_mut());
+            std::ptr::write_bytes(ret, 0xa1, layout.size());
+            ret
+        }
+
+        unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
+            std::ptr::write_bytes(ptr, 0xde, layout.size());
+            System.dealloc(ptr, layout)
+
+        }
+    }
+}
+
 
 #[global_allocator]
 #[cfg(feature = "dh")]

scripts/cross_compile.sh

@@ -12,10 +12,10 @@ rustup update --no-self-update
 
 RUSTFLAGS="--cfg miri" cargo check
 
-rustup toolchain install 1.46.0 --no-self-update
+rustup toolchain install 1.48.0 --no-self-update
 cargo clean
 rm Cargo.lock
-cargo +1.46.0 check
+cargo +1.48.0 check
 
 for target in $targets; do
   echo "setting up $target..."

scripts/sanitizers.sh

@@ -3,11 +3,22 @@ set -eo pipefail
 
 pushd benchmarks/stress2
 
-rustup toolchain install nightly --no-self-update
-rustup update --no-self-update
+rustup toolchain install nightly
+rustup toolchain install nightly --component rust-src
+rustup update
 
 export SLED_LOCK_FREE_DELAY_INTENSITY=2000
 
+echo "msan"
+cargo clean
+export RUSTFLAGS="-Zsanitizer=memory -Zsanitizer-memory-track-origins"
+cargo +nightly build -Zbuild-std --target x86_64-unknown-linux-gnu
+sudo rm -rf default.sled
+sudo target/x86_64-unknown-linux-gnu/debug/stress2 --duration=30 --set-prop=100000000 --val-len=1000 --entries=100 --threads=100
+sudo target/x86_64-unknown-linux-gnu/debug/stress2 --duration=30 --entries=100
+sudo target/x86_64-unknown-linux-gnu/debug/stress2 --duration=30
+unset MSAN_OPTIONS
+
 echo "asan"
 cargo clean
 export RUSTFLAGS="-Z sanitizer=address"

src/config.rs

@@ -538,33 +538,17 @@ impl Config {
         {
             use fs2::FileExt;
 
-            let try_lock = if cfg!(feature = "testing") {
-                // we block here because during testing
-                // there are many filesystem race condition
-                // that happen, causing locks to be held
-                // for long periods of time, so we should
-                // block to wait on reopening files.
-                let start = std::time::Instant::now();
-                loop {
-                    let ret = file.try_lock_exclusive();
-
-                    if ret.is_ok() {
-                        break ret;
-                    }
-
-                    if start.elapsed() > std::time::Duration::from_secs(10) {
-                        log::error!(
-                            "failed to open file in exclusive mode: {:?}",
-                            ret
-                        );
-                        break ret;
-                    }
-
-                    std::thread::sleep(std::time::Duration::from_millis(50));
-                }
-            } else {
-                file.try_lock_exclusive()
-            };
+            let try_lock =
+                if cfg!(any(feature = "testing", feature = "light_testing")) {
+                    // we block here because during testing
+                    // there are many filesystem race condition
+                    // that happen, causing locks to be held
+                    // for long periods of time, so we should
+                    // block to wait on reopening files.
+                    file.lock_exclusive()
+                } else {
+                    file.try_lock_exclusive()
+                };
 
             if let Err(e) = try_lock {
                 return Err(Error::Io(io::Error::new(
@@ -774,6 +758,16 @@ impl Deref for RunningConfig {
     }
 }
 
+#[cfg(all(not(miri), any(windows, target_os = "linux", target_os = "macos")))]
+impl Drop for RunningConfig {
+    fn drop(&mut self) {
+        use fs2::FileExt;
+        if Arc::strong_count(&self.file) == 1 {
+            let _ = self.file.unlock();
+        }
+    }
+}
+
 impl Drop for Inner {
     fn drop(&mut self) {
         if self.temporary {

src/db.rs

@@ -12,6 +12,7 @@ const DEFAULT_TREE_ID: &[u8] = b"__sled__default";
 /// to disk, using the same method used by
 /// `Tree::flush`.
 #[derive(Clone)]
+#[doc(alias = "database")]
 pub struct Db {
     #[doc(hidden)]
     pub context: Context,