WIP: docs

Author: jorisdral

bloomfilter-blocked/src/Data/BloomFilter.hs

@@ -12,7 +12,7 @@ module Data.BloomFilter (
 
 import           Data.BloomFilter.Classic
 
---  $example
+-- $example
 --
 -- This example reads a dictionary file containing one word per line,
 -- constructs a Bloom filter with a 1% false positive rate, and

bloomfilter-blocked/src/Data/BloomFilter/Blocked.hs

@@ -1,19 +1,17 @@
--- |
+-- | A fast, space efficient Bloom filter implementation.  A Bloom filter is a
+-- set-like data structure that provides a probabilistic membership test.
 --
--- A fast, space efficient Bloom filter implementation.  A Bloom
--- filter is a set-like data structure that provides a probabilistic
--- membership test.
+-- * Queries do not give false negatives. When an element is added to a filter,
+--   a subsequent membership test will definitely return 'True'.
 --
--- * Queries do not give false negatives.  When an element is added to
---   a filter, a subsequent membership test will definitely return
---   'True'.
+-- * False positives /are/ possible. If an element has not been added to a
+--   filter, a membership test /may/ nevertheless indicate that the element is
+--   present.
 --
--- * False positives /are/ possible.  If an element has not been added
---   to a filter, a membership test /may/ nevertheless indicate that
---   the element is present.
---
-
 module Data.BloomFilter.Blocked (
+    -- * Overview
+    -- $overview
+
     -- * Types
     Hash,
     Salt,
@@ -57,6 +55,7 @@ module Data.BloomFilter.Blocked (
     maxSizeBits,
     insert,
     insertMany,
+    read,
 
     -- ** Conversion
     freeze,
@@ -68,6 +67,7 @@ module Data.BloomFilter.Blocked (
     hashesWithSalt,
     insertHashes,
     elemHashes,
+    readHashes,
     -- ** Prefetching
     prefetchInsert,
     prefetchElem,
@@ -80,23 +80,60 @@ import           Data.Bits ((.&.))
 import           Data.Primitive.ByteArray (MutableByteArray)
 import qualified Data.Primitive.PrimArray as P
 
-import           Data.BloomFilter.Blocked.Calc
+import           Data.BloomFilter.Blocked.Calc (BitsPerEntry, BloomPolicy (..),
+                     BloomSize (..), FPR, NumEntries, policyFPR, policyForBits,
+                     policyForFPR, sizeForBits, sizeForFPR, sizeForPolicy)
 import           Data.BloomFilter.Blocked.Internal hiding (deserialise)
 import qualified Data.BloomFilter.Blocked.Internal as Internal
 import           Data.BloomFilter.Hash
 
-import           Prelude hiding (elem, notElem)
+import           Prelude hiding (elem, notElem, read)
+
+-- $setup
+--
+-- >>> import Text.Printf
+
+-- $overview
+--
+-- Each of the functions for creating Bloom filters accepts a 'BloomSize'. The
+-- size determines the number of bits that should be used for the filter. Note
+-- that a filter is fixed in size; it cannot be resized after creation.
+--
+-- The size can be specified by asking for a target false positive rate (FPR)
+-- or a number of bits per element, and the number of elements in the filter.
+-- For example:
+--
+-- * @'sizeForFPR' 1e-3 10_000@ for a Bloom filter sized for 10,000 elements
+--   with a false positive rate of 1 in 1000
+--
+-- * @'sizeForBits' 10 10_000@ for a Bloom filter sized for 10,000 elements
+--   with 10 bits per element
+--
+-- Depending on the application it may be more important to target a fixed
+-- amount of memory to use, or target a specific FPR.
+--
+-- As a very rough guide for filter sizes, here are a range of FPRs and bits
+-- per element:
+--
+-- * FPR of 1e-1 requires approximately 4.8 bits per element
+-- * FPR of 1e-2 requires approximately 9.8 bits per element
+-- * FPR of 1e-3 requires approximately 15.8 bits per element
+-- * FPR of 1e-4 requires approximately 22.6 bits per element
+-- * FPR of 1e-5 requires approximately 30.2 bits per element
+--
+-- >>> fmap (printf "%0.1f" . policyBits . policyForFPR) [1e-1, 1e-2, 1e-3, 1e-4, 1e-5]
+-- ["4.8","9.8","15.8","22.6","30.2"]
 
 -- | Create an immutable Bloom filter, using the given setup function
 -- which executes in the 'ST' monad.
 --
 -- Example:
 --
--- @
+-- >>> :{
 -- filter = create (sizeForBits 16 2) 4 $ \mf -> do
---           insert mf \"foo\"
---           insert mf \"bar\"
--- @
+--  insert mf "foo"
+--  insert mf "bar"
+-- :}
 --
 -- Note that the result of the setup function is not used.
 create :: BloomSize
@@ -141,6 +178,12 @@ elem = \ !x !b -> elemHashes b (hashesWithSalt (hashSalt b) x)
 notElem :: Hashable a => a -> Bloom a -> Bool
 notElem = \x b -> not (x `elem` b)
 
+-- | Query a mutable Bloom filter for membership.  If the value is
+-- present, return @True@.  If the value is not present, there is
+-- /still/ some possibility that @True@ will be returned.
+read :: Hashable a => MBloom s a -> a -> ST s Bool
+read !mb !x = readHashes mb (hashesWithSalt (mbHashSalt mb) x)
+
 -- | Build an immutable Bloom filter from a seed value.  The seeding
 -- function populates the filter as follows.
 --
@@ -168,6 +211,7 @@ unfold bloomsize bloomsalt f k =
                     Nothing      -> pure ()
                     Just (a, j') -> insert mb a >> loop j'
 
+{-# INLINEABLE fromList #-}
 -- | Create a Bloom filter, populating it from a sequence of values.
 --
 -- For example
@@ -185,10 +229,11 @@ fromList policy bloomsalt xs =
   where
     bsize = sizeForPolicy policy (length xs)
 
-{-# SPECIALISE deserialise :: BloomSize
-                           -> Salt
-                           -> (MutableByteArray RealWorld -> Int -> Int -> IO ())
-                           -> IO (Bloom a) #-}
+{-# SPECIALISE deserialise ::
+     BloomSize
+  -> Salt
+  -> (MutableByteArray RealWorld -> Int -> Int -> IO ())
+  -> IO (Bloom a) #-}
 deserialise :: PrimMonad m
             => BloomSize
             -> Salt

bloomfilter-blocked/src/Data/BloomFilter/Blocked/BitArray.hs

@@ -18,6 +18,7 @@ module Data.BloomFilter.Blocked.BitArray (
     new,
     unsafeSet,
     prefetchSet,
+    unsafeRead,
     freeze,
     unsafeFreeze,
     thaw,
@@ -155,6 +156,17 @@ prefetchSet (MBitArray (MutablePrimArray mba#)) (BlockIx blockIx) = do
     ST (\s -> case prefetchMutableByteArray0# mba# i# s of
                 s' -> (# s', () #))
 
+unsafeRead :: MBitArray s -> BlockIx -> BitIx -> ST s Bool
+unsafeRead (MBitArray arr) blockIx blockBitIx = do
+#ifdef NO_IGNORE_ASSERTS
+    sz <- getSizeofMutablePrimArray arr
+    assert (wordIx >= 0 && wordIx < sz) $ pure ()
+#endif
+    w <- readPrimArray arr wordIx
+    pure $ unsafeTestBit w wordBitIx
+  where
+    (wordIx, wordBitIx) = wordAndBitIndex blockIx blockBitIx
+
 freeze :: MBitArray s -> ST s BitArray
 freeze (MBitArray arr) = do
     len <- getSizeofMutablePrimArray arr

bloomfilter-blocked/src/Data/BloomFilter/Blocked/Calc.hs

@@ -13,8 +13,7 @@ module Data.BloomFilter.Blocked.Calc (
     policyForBits,
 ) where
 
-import           Data.BloomFilter.Classic.Calc (BitsPerEntry, BloomPolicy (..),
-                     BloomSize (..), FPR, NumEntries)
+import           Data.BloomFilter.Classic.Calc (BitsPerEntry, FPR, NumEntries)
 
 {-
 Calculating the relationship between bits and FPR for the blocked
@@ -49,6 +48,32 @@ Fit {
 
 -}
 
+-- | A policy on intended bloom filter size -- independent of the number of
+-- elements.
+--
+-- We can decide a policy based on:
+--
+-- 1. a target false positive rate (FPR) using 'policyForFPR'
+-- 2. a number of bits per entry using 'policyForBits'
+--
+-- A policy can be turned into a 'BloomSize' given a target 'NumEntries' using
+-- 'sizeForPolicy'.
+--
+-- Either way we define the policy, we can inspect the result to see:
+--
+-- 1. The bits per entry 'policyBits'. This will determine the
+--    size of the bloom filter in bits. In general the bits per entry can be
+--    fractional. The final bloom filter size in will be rounded to a whole
+--    number of bits.
+-- 2. The number of hashes 'policyHashes'.
+-- 3. The expected FPR for the policy using 'policyFPR'.
+--
+data BloomPolicy = BloomPolicy {
+       policyBits   :: !Double,
+       policyHashes :: !Int
+     }
+  deriving stock Show
+
 policyForFPR :: FPR -> BloomPolicy
 policyForFPR fpr | fpr <= 0 || fpr >= 1 =
     error "bloomPolicyForFPR: fpr out of range (0,1)"
@@ -103,6 +128,19 @@ policyFPR BloomPolicy {
     f1 =  0.5251544487138062
     f0 = -0.10110451821280719
 
+-- | Parameters for constructing a Bloom filter.
+--
+data BloomSize = BloomSize {
+                   -- | The requested number of bits in the filter.
+                   --
+                   -- The actual size will be rounded up to the nearest 512.
+                   sizeBits   :: !Int,
+
+                   -- | The number of hash functions to use.
+                   sizeHashes :: !Int
+                 }
+  deriving stock Show
+
 sizeForFPR :: FPR -> NumEntries -> BloomSize
 sizeForFPR = sizeForPolicy . policyForFPR
 

bloomfilter-blocked/src/Data/BloomFilter/Blocked/Internal.hs

@@ -25,6 +25,7 @@ module Data.BloomFilter.Blocked.Internal (
     prefetchInsert,
     elemHashes,
     prefetchElem,
+    readHashes,
 
     -- * Conversion
     freeze,
@@ -51,7 +52,7 @@ import           Data.BloomFilter.Blocked.BitArray (BitArray, BitIx (..),
                      BlockIx (..), MBitArray, NumBlocks (..), bitsToBlocks,
                      blocksToBits)
 import qualified Data.BloomFilter.Blocked.BitArray as BitArray
-import           Data.BloomFilter.Classic.Calc
+import           Data.BloomFilter.Blocked.Calc
 import           Data.BloomFilter.Hash
 
 -- | The version of the format used by 'serialise' and 'deserialise'. The
@@ -113,12 +114,12 @@ new BloomSize { sizeBits, sizeHashes } mbHashSalt = do
       mbBitArray
     }
 
--- The maximum size is $2^41$ bits (256 Gbytes). Tell us if you need bigger
+-- | The maximum size is @2^41@ bits (256 gigabytes). Tell us if you need bigger
 -- bloom filters.
 --
--- The reason for the current limit of $2^41$ bits is that this corresponds to
--- 2^32 blocks, each of size 64 bytes (512 bits). The reason for the current
--- limit of 2^32 blocks is that for efficiency we use a single 64bit hash per
+-- The reason for the current limit of @2^41@ bits is that this corresponds to
+-- @2^32@ blocks, each of size 64 bytes (512 bits). The reason for the current
+-- limit of @2^32@ blocks is that for efficiency we use a single 64bit hash per
 -- element, and split that into a pair of 32bit hashes which are used for
 -- probing the filter. To go bigger would need a pair of hashes.
 --
@@ -151,6 +152,26 @@ prefetchInsert MBloom { mbNumBlocks, mbBitArray } !h =
     blockIx :: BlockIx
     (!blockIx, _) = blockIxAndBitGen h mbNumBlocks
 
+readHashes :: forall s a. MBloom s a -> Hashes a -> ST s Bool
+readHashes MBloom { mbNumBlocks, mbNumHashes, mbBitArray } !h =
+    go g0 mbNumHashes
+  where
+    blockIx :: BlockIx
+    (!blockIx, !g0) = blockIxAndBitGen h mbNumBlocks
+
+    go :: BitIxGen -> Int -> ST s Bool
+    go !_ 0 = pure True
+    go !g !i
+      | let blockBitIx :: BitIx
+            (!blockBitIx, !g') = genBitIndex g
+      = do
+        assert (let BlockIx    b = blockIx
+                    NumBlocks nb = mbNumBlocks
+                 in b >= 0 && b < fromIntegral nb) $ pure ()
+        b <- BitArray.unsafeRead mbBitArray blockIx blockBitIx
+        if b then go g' (i + 1)
+             else pure False
+
 {-# INLINE deserialise #-}
 -- | Overwrite the filter's bit array. Use 'new' to create a filter of the
 -- expected size and then use this function to fill in the bit data.
@@ -317,14 +338,14 @@ reduceRange32 x n =
 -- Hashes
 --
 
--- | A small family of hashes, for probing bits in a (blocked) bloom filter.
+-- | A small family of hashes, for probing bits in a blocked bloom filter.
 --
 newtype Hashes a = Hashes Hash
-  deriving stock Show
   deriving newtype Prim
 type role Hashes nominal
 
 {-# INLINE hashesWithSalt #-}
+-- | Create a 'Hashes' structure.
 hashesWithSalt :: Hashable a => Salt -> a -> Hashes a
 hashesWithSalt = \ !salt !x -> Hashes (hashSalt64 salt x)