Improved README.md for a 0.1.0

Author: blenessy

README.md

@@ -1,10 +1,99 @@
 # PersistentCollections.jl
 
-Julia AbstractDict and AbstractSet data structures persisted (ACID) to disk.
+Julia `Dict` and `Set` data structures safely persisted to disk.
+
+All collections are backed by [LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database) - a super fast B-Tree based embedded KV database with ACID guaranties.
+As with other B-Tree based databases reads are generally faster than writes. LMDB is not an exception, although write performance is relatively good to (expect 1k-10k TPS).
+
+Care was taken to make the datastructures thread-safe. LMDB handles most of the locking well, we just have to serialise the writes to an LMDB Environment in julia so that
+multiple threads do not attempt to write at once (deadlock will occur).
 
 ## Quick Start
 
-TODO
+1. Install this package:
+   ```julia
+   import Pkg
+   Pkg.add("https://github.com/blenessy/PersistentCollections.jl.git")
+   ```
+1. Create an `LMDB.Environment` in a directory called `data` (in your current working directory):
+   ```julia
+   using PersistentCollections
+   env = LMDB.Environment("data")
+   ```
+1. Create an `AbstractDict` in your LMDB environment:
+   ```julia
+   dict = PersistentDict{String,String}(env)
+   ```
+1. Use it as any other dict:
+   ```julia
+   dict["foo"] = "bar"
+   @assert dict["foo"] == "bar"
+   @assert collect(keys(dict)) == ["foo"]
+   @assert collect(values(dict)) == ["bar"]
+   ```
+1. (Optional) note the asymetric performance characteristic of LMDB (B-Tree) based database:
+   ```julia
+   @time dict["bar"] = "baz";  # Writes to LMDB (B-Tree) are relatively slow
+   @time dict["bar"];          # Reads are very fast though :)
+   ```
+
+## User Guide
+
+### Dynamic types
+
+It is possible to create persistent collection of `Any` type although some methods will not be able to convert the value to the correct type because no metadata is stored for this in DB.
+Most notably the `getindex` method (e.g. `dict["foo"]`) will not return a converted value. To mitigate this limitation, use the `get` method, which includes a default value.
+The type of the default value (if other than `nothing`) will be used to convert the value to the desired type.
+
+```julia
+env = LMDB.Environment("data")
+dict = PersistentDict{Any,Any}(env)
+dict["foo"] == "bar"
+dict["foo"]                  # PersistentCollections.LMDB.MDBValue{Nothing}(0x0000000000000003, Ptr{Nothing} @0x000000012c806ffd, nothing)
+get(dict, "foo", "")         # "bar"
+convert(String, dict["foo"]) # "bar"
+```
+
+### Multiple persistent collections in the same LMDB Environment
+
+It is possible if you need transactional consistency between multiple persistent collections:
+
+1. Create your `LMDB.Environment` with "named database" support by specifying the number of persistent collections yoy want with the `maxdbs` keyword argument:
+   ```julia
+   env = LMDB.Environment("data", maxdbs=2)
+   ```
+2. Instantiate your persistent collections with a unique (within LMDB env.) id:
+   ```julia
+   dict1 = PersistentDict{String,String}(env, id="mydict1")
+   dict2 = PersistentDict{String,Int}(env, id="mydict2")
+   ```
+
+### Danger Zone: Manual sync writes to disc
+
+Yes, you can expect significant increase with write throughput if you are willing to risk loosing your last written transactions.
+Please note that database integrity (risk of curruption) is not in danger here.
+
+```julia
+unsafe_env = LMDB.Environment("data", flags=LMDB.MDB_NOSYNC)
+unsafe_dict = PersistentDict{String,String}(unsafe_env)
+flush(unsafe_env) do 
+    unsafe_dict["foo"] = "bar"
+    unsafe_dict["foo"] = "baz"
+end # <== data is flushed to disk here
+```
+
+This is equvalent to: 
+
+```julia
+unsafe_env = LMDB.Environment("data", flags=LMDB.MDB_NOSYNC)
+unsafe_dict = PersistentDict{String,String}(unsafe_env)
+try
+    unsafe_dict["foo"] = "bar"
+    unsafe_dict["foo"] = "baz"
+finally
+    flush(unsafe_env)
+end
+```
 
 ## Running Tests
 

src/PersistentCollections.jl

@@ -1,22 +1,24 @@
 module PersistentCollections
+    export LMDB, PersistentDict
+
     module LMDB
         include(joinpath(@__DIR__, "lmdb.jl"))
     end
 
-    abstract type PersistentAbstractDict{K,V} <: AbstractDict{K,V} end
-    struct PersistentDict{K,V} <: PersistentAbstractDict{K,V}
+    struct PersistentDict{K,V} <: AbstractDict{K,V}
         env::LMDB.Environment
         id::String
         PersistentDict{K,V}(env; id="") where {K,V} = new{K,V}(env, id)
     end
 
+    Base.show(io::IO, d::PersistentDict) = print(io, typeof(d), "(", isempty(d.id) ? "" : repr(e.id), ")")
+
     function Base.get(d::PersistentDict{K,V}, key::K, default::D) where {K,V,D}
         isopen(d.env) || error("Environment is closed")
         txn = d.env.rotxn[Threads.threadid()]
         LMDB.mdb_txn_renew(txn)
         try
             dbi = LMDB.mdb_dbi_open(txn, d.id, zero(Cuint))
-            @assert txn != C_NULL && !iszero(dbi) "txn and/or dbi handles are not initialized"
             mdbkey, mdbval = convert(LMDB.MDBValue, key), LMDB.MDBValue()
             found = GC.@preserve mdbkey LMDB.mdb_get!(txn, dbi, pointer(mdbkey), pointer(mdbval))
             found || return default
@@ -154,4 +156,16 @@ module PersistentCollections
     Base.keys(d::PersistentDict{K,V}) where {K,V} = MDBKeyCursor{K,V}(create_atomic_cursor(d))
     Base.values(d::PersistentDict{K,V}) where {K,V} = MDBValCursor{K,V}(create_atomic_cursor(d))
 
+    function Base.length(d::PersistentDict)
+        isopen(d.env) || error("Environment is closed")
+        txn = d.env.rotxn[Threads.threadid()]
+        LMDB.mdb_txn_renew(txn)
+        try
+            dbi = LMDB.mdb_dbi_open(txn, d.id, zero(Cuint))
+            return convert(Int, LMDB.mdb_stat(txn, dbi).ms_entries)
+        finally
+            LMDB.mdb_txn_reset(txn)
+        end
+    end
+
 end # module

src/lmdb.jl

@@ -225,6 +225,12 @@ function mdb_env_stat(env::Ptr{Cvoid})
     return statref[]
 end
 
+function mdb_stat(txn::Ptr{Cvoid}, dbi::Cuint)
+    statref = Ref(MDBStat())
+    @chkres ccall((:mdb_stat, liblmdb), Cint, (Ptr{Cvoid}, Cuint, Ptr{MDBStat}), txn, dbi, statref)
+    return statref[]
+end
+
 mutable struct Environment
     handle::Ptr{Cvoid}
     path::String
@@ -237,6 +243,8 @@ const OPENED_ENVS = Dict{String,Environment}()
 function Environment(path::String; flags::Cuint=zero(Cuint), mode::Cmode_t=0o755, maxdbs=0, mapsize=10485760, maxreaders=126, rotxnflags::Cuint=DEFAULT_ROTXN_FLAGS)
     env = get(OPENED_ENVS, path, nothing)
     if isnothing(env)
+        # create all directories needed to host the data
+        mkpath(iszero(flags & MDB_NOSUBDIR) ? path : dirname(path), mode=mode)
         rotxn = [C_NULL for i in 1:Threads.nthreads()]
         env = Environment(mdb_env_create(), path, rotxn, ReentrantLock())
         mdb_env_set_maxdbs(env.handle, convert(Cuint, maxdbs))
@@ -260,6 +268,8 @@ function Environment(path::String; flags::Cuint=zero(Cuint), mode::Cmode_t=0o755
     return env
 end
 
+Base.show(io::IO, e::Environment) = print(io, typeof(e), "(", repr(e.path), ")")
+
 Base.isopen(env::Environment) = env.handle != C_NULL
 
 function Base.close(env::Environment)
@@ -276,6 +286,7 @@ function Base.close(env::Environment)
     return false
 end
 
+Base.flush(env::Environment) = mdb_env_sync(env.handle, one(Cint))
 function Base.flush(func::Function, env::Environment)
     try
         func()

test/runtests.jl

@@ -1,7 +1,7 @@
 using Test
 using BenchmarkTools
 
-using PersistentCollections: LMDB, PersistentDict
+using PersistentCollections
 
 const ENV_DIR = "env.lmdb"
 const UNSAFE_ENV_DIR = "unsafe_env.lmdb"