Merge pull request #322 from IntersectMBO/jdral/conf-index

Add configuration option for the fence pointer index

Author: jorisdral

bench/micro/Bench/Database/LSMTree/Monoidal.hs

@@ -57,6 +57,7 @@ benchConfig = Normal.TableConfig {
     , confSizeRatio = Normal.Four
     , confWriteBufferAlloc = Normal.AllocNumEntries (Normal.NumEntries 20000)
     , confBloomFilterAlloc = Normal.AllocFixed 10
+    , confFencePointerIndex = Normal.CompactIndex
     , confDiskCachePolicy = Normal.DiskCacheAll
     }
 

src/Database/LSMTree/Internal.hs

@@ -39,7 +39,7 @@ module Database.LSMTree.Internal (
   , listSnapshots
     -- * Mutiple writable table handles
   , duplicate
-    -- * configuration
+    -- * Configuration
   , TableConfig (..)
   , defaultTableConfig
   , MergePolicy (..)
@@ -48,6 +48,7 @@ module Database.LSMTree.Internal (
   , NumEntries (..)
   , BloomFilterAlloc (..)
   , defaultBloomFilterAlloc
+  , FencePointerIndex (..)
   , DiskCachePolicy (..)
     -- * Exported for cabal-docspec
   , MergePolicyForLevel (..)
@@ -1277,22 +1278,24 @@ duplicate th = withOpenTable th $ \thEnv -> do
 --
 -- * Size ratio: 4
 data TableConfig = TableConfig {
-    confMergePolicy      :: !MergePolicy
+    confMergePolicy       :: !MergePolicy
     -- Size ratio between the capacities of adjacent levels.
-  , confSizeRatio        :: !SizeRatio
+  , confSizeRatio         :: !SizeRatio
     -- | Total number of bytes that the write buffer can use.
     --
     -- The maximum is 4GiB, which should be more than enough for realistic
     -- applications.
-  , confWriteBufferAlloc :: !WriteBufferAlloc
-  , confBloomFilterAlloc :: !BloomFilterAlloc
+  , confWriteBufferAlloc  :: !WriteBufferAlloc
+  , confBloomFilterAlloc  :: !BloomFilterAlloc
+  , confFencePointerIndex :: !FencePointerIndex
     -- | The policy for caching key\/value data from disk in memory.
-  , confDiskCachePolicy  :: !DiskCachePolicy
+  , confDiskCachePolicy   :: !DiskCachePolicy
   }
-  deriving stock Show
+  deriving stock (Show, Eq)
 
 instance NFData TableConfig where
-  rnf (TableConfig a b c d e) = rnf a `seq` rnf b `seq` rnf c `seq` rnf d `seq` rnf e
+  rnf (TableConfig a b c d e f) =
+      rnf a `seq` rnf b `seq` rnf c `seq` rnf d `seq` rnf e `seq` rnf f
 
 -- | TODO: this should be removed once we have proper snapshotting with proper
 -- persistence of the config to disk.
@@ -1310,6 +1313,7 @@ defaultTableConfig =
       , confSizeRatio        = Four
       , confWriteBufferAlloc = AllocNumEntries (NumEntries 20_000)
       , confBloomFilterAlloc = defaultBloomFilterAlloc
+      , confFencePointerIndex = CompactIndex
       , confDiskCachePolicy  = DiskCacheAll
       }
 
@@ -1451,6 +1455,39 @@ bloomFilterAllocForLevel conf (LevelNo l) =
                                       0 -> Just x
                                       _ -> r (k-1)) (const Nothing) xs n
 
+-- | Configure the type of fence pointer index.
+--
+-- TODO: this configuration option currently has no effect: 'CompactIndex' is
+-- always used.
+data FencePointerIndex =
+    -- | Use a compact fence pointer index.
+    --
+    -- The compact index type is designed to work with keys that are large
+    -- cryptographic hashes, e.g. 32 bytes.
+    --
+    -- When using the 'IndexCompact', additional constraints apply to the
+    -- 'Database.LSMTree.Internal.Serialise.Class.serialiseKey' function. The
+    -- __Minimal size__ law should be satisfied:
+    --
+    -- [Minimal size] @'Database.LSMTree.Internal.RawBytes.size'
+    --   ('Database.LSMTree.Internal.Serialise.Class.serialiseKey' x) >= 8@
+    --
+    -- Use 'Database.LSMTree.Internal.Serialise.Class.serialiseKeyMinimalSize'
+    -- to test this law.
+    CompactIndex
+    -- | Use an ordinary fence pointer index, without any constraints on
+    -- serialised keys.
+  | OrdinaryIndex
+  deriving stock (Show, Eq)
+
+instance NFData FencePointerIndex where
+  rnf CompactIndex  = ()
+  rnf OrdinaryIndex = ()
+
+-- | TODO: this should be removed once we have proper snapshotting with proper
+-- persistence of the config to disk.
+deriving stock instance Read FencePointerIndex
+
 -- | The policy for caching data from disk in memory (using the OS page cache).
 --
 -- Caching data in memory can improve performance if the access pattern has

src/Database/LSMTree/Internal/Serialise/Class.hs

@@ -43,18 +43,10 @@ import           Numeric (showInt)
 -- [Ordering-preserving] @x \`'compare'\` y == 'serialiseKey' x \`'compare'\` 'serialiseKey' y@
 --
 -- Raw bytes are lexicographically ordered, so in particular this means that
--- values should be serialised into big-endian formats.
--- This constraint mainly exists for range queries, where the range is specified
--- in terms of unserialised values, but the internal implementation works on the
--- serialised representation.
---
--- === IndexCompact constraints
---
--- When using the 'IndexCompact', additional constraints apply to the
--- serialisation function, so in that case instances should also satisfy the
--- following:
---
--- [Minimal size] @'sizeofRawBytes' >= 8@
+-- values should be serialised into big-endian formats. This constraint mainly
+-- exists for range queries, where the range is specified in terms of
+-- unserialised values, but the internal implementation works on the serialised
+-- representation.
 class SerialiseKey k where
   serialiseKey :: k -> RawBytes
   -- TODO: 'deserialiseKey' is only strictly necessary for range queries.

src/Database/LSMTree/Monoidal.hs

@@ -43,6 +43,7 @@ module Database.LSMTree.Monoidal (
   , Internal.NumEntries (..)
   , Internal.BloomFilterAlloc (..)
   , Internal.defaultBloomFilterAlloc
+  , Internal.FencePointerIndex (..)
   , Internal.DiskCachePolicy (..)
   , withTable
   , new

src/Database/LSMTree/Normal.hs

@@ -42,6 +42,7 @@ module Database.LSMTree.Normal (
   , Internal.NumEntries (..)
   , Internal.BloomFilterAlloc (..)
   , Internal.defaultBloomFilterAlloc
+  , Internal.FencePointerIndex (..)
   , Internal.DiskCachePolicy (..)
   , withTable
   , new

test/Test/Database/LSMTree/Class/Monoidal.hs

@@ -46,6 +46,7 @@ tests = testGroup "Test.Database.LSMTree.Class.Monoidal"
             , R.confSizeRatio = R.Four
             , R.confWriteBufferAlloc = R.AllocNumEntries (R.NumEntries 3)
             , R.confBloomFilterAlloc = R.AllocFixed 10
+            , R.confFencePointerIndex = R.CompactIndex
             , R.confDiskCachePolicy = R.DiskCacheNone
             }
         , testWithSessionArgs = \action ->

test/Test/Database/LSMTree/Class/Normal.hs

@@ -52,6 +52,7 @@ tests = testGroup "Test.Database.LSMTree.Class.Normal"
             , R.confSizeRatio = R.Four
             , R.confWriteBufferAlloc = R.AllocNumEntries (R.NumEntries 3)
             , R.confBloomFilterAlloc = R.AllocFixed 10
+            , R.confFencePointerIndex = R.CompactIndex
             , R.confDiskCachePolicy = R.DiskCacheNone
             }
         , testWithSessionArgs = \action ->

test/Test/Database/LSMTree/Internal.hs

@@ -127,6 +127,7 @@ prop_interimRestoreSessionUniqueRunNames (Positive (Small n)) (NonNegative m) =
         -- flushes and merges.
       , confWriteBufferAlloc = AllocNumEntries (NumEntries n)
       , confBloomFilterAlloc = AllocFixed 10
+      , confFencePointerIndex = CompactIndex
       , confDiskCachePolicy = DiskCacheNone
       }
 
@@ -172,6 +173,7 @@ prop_interimOpenTable dat = ioProperty $
         -- flushes and merges.
       , confWriteBufferAlloc = AllocNumEntries (NumEntries 3)
       , confBloomFilterAlloc = AllocFixed 10
+      , confFencePointerIndex = CompactIndex
       , confDiskCachePolicy = DiskCacheNone
       }
 

test/Test/Database/LSMTree/Normal/StateMachine.hs

@@ -227,24 +227,14 @@ instance Arbitrary M.TableConfig where
 instance Arbitrary R.TableConfig where
   arbitrary :: Gen R.TableConfig
   arbitrary = pure $ R.TableConfig {
-        R.confMergePolicy      = R.MergePolicyLazyLevelling
-      , R.confSizeRatio        = R.Four
-      , R.confWriteBufferAlloc = R.AllocNumEntries (R.NumEntries 30)
-      , R.confBloomFilterAlloc = R.AllocFixed 10
-      , R.confDiskCachePolicy  = R.DiskCacheNone
+        R.confMergePolicy       = R.MergePolicyLazyLevelling
+      , R.confSizeRatio         = R.Four
+      , R.confWriteBufferAlloc  = R.AllocNumEntries (R.NumEntries 30)
+      , R.confBloomFilterAlloc  = R.AllocFixed 10
+      , R.confFencePointerIndex = R.CompactIndex
+      , R.confDiskCachePolicy   = R.DiskCacheNone
       }
 
-instance Eq R.TableConfig where
-  R.TableConfig pol1 size1 wbAlloc1 bfAlloc1 cache1
-    == R.TableConfig pol2 size2 wbAlloc2 bfAlloc2 cache2
-    = and [
-        pol1 == pol2
-      , size1 == size2
-      , wbAlloc1 == wbAlloc2
-      , bfAlloc1 == bfAlloc2
-      , cache1 == cache2
-      ]
-
 {-------------------------------------------------------------------------------
   Key and value types
 -------------------------------------------------------------------------------}