Pass Env parameters via kwargs for LMDBDict

.github/workflows/CI.yml

@@ -14,7 +14,7 @@ jobs:
       matrix:
         version:
           - '1'
-          - 'nightly'
+          - 'lts'
         os:
           - ubuntu-latest
           - macOS-latest
@@ -27,7 +27,7 @@ jobs:
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
-      - uses: actions/cache@v2
+      - uses: actions/cache@v4
         env:
           cache-name: cache-artifacts
         with:
@@ -44,10 +44,3 @@ jobs:
         env:
           PYTHON:
       - uses: julia-actions/julia-processcoverage@v1
-      - uses: codecov/codecov-action@v1
-        with:
-          file: lcov.info
-      - uses: coverallsapp/github-action@master
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          path-to-lcov: ./lcov.info

Project.toml

@@ -7,6 +7,7 @@ version = "1.0.0"
 CEnum = "fa961155-64e5-5f13-b03f-caf6b980ea82"
 LMDB_jll = "6206cf0b-f360-5984-af49-5437264c140e"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
+Mmap = "a63ad114-7e13-5084-954f-fe012c677804"
 
 [compat]
 CEnum = "0.4, 0.5"

README.md

@@ -1,6 +1,5 @@
 # LMDB
 [![CI](https://github.com/plantingspace/LMDB.jl/actions/workflows/CI.yml/badge.svg)](https://github.com/plantingspace/LMDB.jl/actions/workflows/CI.yml)
-[![Coverage Status](https://img.shields.io/coveralls/wildart/LMDB.jl.svg)](https://coveralls.io/r/wildart/LMDB.jl)
 
 Lightning Memory-Mapped Database (LMDB) is an ultra-fast, ultra-compact key-value embedded data store developed by Symas for the OpenLDAP Project. It uses memory-mapped files, so it has the read performance of a pure in-memory database while still offering the persistence of standard disk-based databases, and is only limited to the size of the virtual address space. This module provides a Julia interface to [LMDB (v0.9.27)](https://github.com/LMDB/lmdb).
 

src/LMDB.jl

@@ -5,6 +5,7 @@ module LMDB
     import Base: open, close, getindex, setindex!, put!, reset,
                  isopen, count, delete!, keys, get, show, show
     import Base.Iterators: drop
+    import Mmap: PAGESIZE
 
     export Environment, create, open, close, sync, set!, unset!, getindex, setindex!, path, info, show,
            Transaction, start, abort, commit, reset, renew, environment,

src/cur.jl

@@ -158,25 +158,32 @@ This function retrieves key/data pairs from the database.
 """
 function get(cur::Cursor, key, ::Type{T}, op::MDB_cursor_op=MDB_SET_KEY) where T
     # Setup parameters
-    mdb_key_ref = Ref(MDBValue(toref(key)))
-    mdb_val_ref = Ref(MDBValue())
+    key_ref = toref(key)
+    GC.@preserve key_ref begin
+        mdb_key_ref = Ref(MDBValue(key_ref))
+        mdb_val_ref = Ref(MDBValue())
 
-    # Get value
-    mdb_cursor_get(cur.handle, mdb_key_ref, mdb_val_ref, op)
+        # Get value
+        mdb_cursor_get(cur.handle, mdb_key_ref, mdb_val_ref, op)
 
-    # Convert to proper type
-    return mbd_unpack(T, mdb_val_ref)
+        # Convert to proper type
+        return mbd_unpack(T, mdb_val_ref)
+    end
 end
 
 """Store by cursor.
 
 This function stores key/data pairs into the database. The cursor is positioned at the new item, or on failure usually near it.
 """
 function put!(cur::Cursor, key, val; flags::Cuint = zero(Cuint))
-    mdb_key_ref = Ref(MDBValue(toref(key)))
-    mdb_val_ref = Ref(MDBValue(toref(val)))
+    key_ref = toref(key)
+    val_ref = toref(val)
+    GC.@preserve key_ref val_ref begin
+        mdb_key_ref = Ref(MDBValue(key_ref))
+        mdb_val_ref = Ref(MDBValue(val_ref))
 
-    mdb_cursor_put(cur.handle, mdb_key_ref, mdb_val_ref, flags)
+        mdb_cursor_put(cur.handle, mdb_key_ref, mdb_val_ref, flags)
+    end
 end
 
 "Delete current key/data pair to which the cursor refers"

src/dbi.jl

@@ -60,28 +60,46 @@ toref(v::Ptr{Nothing}) = v
 
 "Store items into a database"
 function put!(txn::Transaction, dbi::DBI, key, val; flags::Cuint = zero(Cuint))
-    mdb_key_ref = Ref(MDBValue(toref(key)))
-    mdb_val_ref = Ref(MDBValue(toref(val)))
-    r = mdb_put(txn.handle, dbi.handle, mdb_key_ref, mdb_val_ref, flags)
-    r
+    key_ref = toref(key)
+    val_ref = toref(val)
+    GC.@preserve key_ref val_ref begin
+        mdb_key_ref = Ref(MDBValue(key_ref))
+        mdb_val_ref = Ref(MDBValue(val_ref))
+        r = mdb_put(txn.handle, dbi.handle, mdb_key_ref, mdb_val_ref, flags)
+        r
+    end
 end
 
 "Delete items from a database"
 function delete!(txn::Transaction, dbi::DBI, key, val=C_NULL)
-    mdb_key_ref = Ref(MDBValue(toref(key)))
-    mdb_val_ref = val === C_NULL ? Ref(MDBValue()) : Ref(MDBValue(toref(val)))
-
-    mdb_del(txn.handle, dbi.handle, mdb_key_ref, mdb_val_ref)
+    key_ref = toref(key)
+    if val === C_NULL
+        GC.@preserve key_ref begin
+            mdb_key_ref = Ref(MDBValue(key_ref))
+            mdb_val_ref = Ref(MDBValue())
+            mdb_del(txn.handle, dbi.handle, mdb_key_ref, mdb_val_ref)
+        end
+    else
+        val_ref = toref(val)
+        GC.@preserve key_ref val_ref begin
+            mdb_key_ref = Ref(MDBValue(key_ref))
+            mdb_val_ref = Ref(MDBValue(val_ref))
+            mdb_del(txn.handle, dbi.handle, mdb_key_ref, mdb_val_ref)
+        end
+    end
 end
 
 "Get items from a database"
 function get(txn::Transaction, dbi::DBI, key, ::Type{T}) where T
-    mdb_key_ref = Ref(MDBValue(toref(key)))
-    mdb_val_ref = Ref(MDBValue())
+    key_ref = toref(key)
+    GC.@preserve key_ref begin
+        mdb_key_ref = Ref(MDBValue(key_ref))
+        mdb_val_ref = Ref(MDBValue())
 
-    # Get value
-    mdb_get(txn.handle, dbi.handle, mdb_key_ref, mdb_val_ref)
+        # Get value
+        mdb_get(txn.handle, dbi.handle, mdb_key_ref, mdb_val_ref)
 
-    # Convert to proper type
-    return mbd_unpack(T, mdb_val_ref)
+        # Convert to proper type
+        return mbd_unpack(T, mdb_val_ref)
+    end
 end

src/dicts.jl

@@ -10,12 +10,22 @@ mutable struct LMDBDict{K,V}
         x
     end
 end
-function LMDBDict{K,V}(path::String; readonly = false, rdahead=false) where {K,V}
+function LMDBDict{K,V}(path::String; readonly::Bool = false, rdahead::Bool = false, mapsize::Union{Nothing, Int} = nothing, readers::Union{Nothing, Int} = nothing, dbs::Union{Nothing, Int} = nothing) where {K,V}
     flags = readonly ? MDB_RDONLY : zero(Cuint)
     if !rdahead
         flags = flags | MDB_NORDAHEAD
     end
     env = LMDB.create()
+    if !isnothing(mapsize)
+        # The size given needs to be rounded to the next multiple of the system's PAGESIZE
+        env[:MapSize] = cld(mapsize, PAGESIZE) * PAGESIZE
+    end
+    if !isnothing(readers)
+        env[:Readers] = readers
+    end
+    if !isnothing(dbs)
+        env[:DBs] = dbs
+    end
     open(env, path)
     #A transaction just for getting a DBI handle
     dbi = LMDB.start(env,flags=flags) do txn
@@ -87,84 +97,99 @@ end
 
 function Base.haskey(d::LMDBDict{K}, key) where K
     txn_dbi_do(d, readonly = true) do txn, dbi
-        mdb_key_ref = Ref(MDBValue(toref(convert(K,key))))
-        mdb_val_ref = Ref(MDBValue())
-        # Get value
-        ret = _mdb_get(txn.handle, dbi.handle, mdb_key_ref, mdb_val_ref)
-        if ret == MDB_NOTFOUND
-            return false
-        elseif ret == Cint(0)
-            return true
-        else
-            throw(LMDB.LMDBError(ret))
+        key_ref = toref(convert(K,key))
+        GC.@preserve key_ref begin
+            mdb_key_ref = Ref(MDBValue(key_ref))
+            mdb_val_ref = Ref(MDBValue())
+            # Get value
+            ret = _mdb_get(txn.handle, dbi.handle, mdb_key_ref, mdb_val_ref)
+            if ret == MDB_NOTFOUND
+                return false
+            elseif ret == Cint(0)
+                return true
+            else
+                throw(LMDB.LMDBError(ret))
+            end
         end
     end
 end
 
 function Base.get(d::LMDBDict{K,V}, key, default) where {K,V}
     txn_dbi_do(d, readonly = true) do txn, dbi
-        mdb_key_ref = Ref(MDBValue(toref(convert(K,key))))
-        mdb_val_ref = Ref(MDBValue())
-        # Get value
-        ret = _mdb_get(txn.handle, dbi.handle, mdb_key_ref, mdb_val_ref)
-        if ret == MDB_NOTFOUND
-            return default
-        elseif ret == Cint(0)
-            return mbd_unpack(V, mdb_val_ref)
-        else
-            throw(LMDB.LMDBError(ret))
+        key_ref = toref(convert(K,key))
+        GC.@preserve key_ref begin
+            mdb_key_ref = Ref(MDBValue(key_ref))
+            mdb_val_ref = Ref(MDBValue())
+            # Get value
+            ret = _mdb_get(txn.handle, dbi.handle, mdb_key_ref, mdb_val_ref)
+            if ret == MDB_NOTFOUND
+                return default
+            elseif ret == Cint(0)
+                return mbd_unpack(V, mdb_val_ref)
+            else
+                throw(LMDB.LMDBError(ret))
+            end
         end
     end
 end
 
 function Base.get!(d::LMDBDict{K,V}, key, default) where {K,V}
     txn_dbi_do(d, readonly = true) do txn, dbi
-        mdb_key_ref = Ref(MDBValue(toref(convert(K,key))))
-        mdb_val_ref = Ref(MDBValue())
-        # Get value
-        ret = _mdb_get(txn.handle, dbi.handle, mdb_key_ref, mdb_val_ref)
-        if ret == MDB_NOTFOUND
-            d[key] = default
-            return default
-        elseif ret == Cint(0)
-            return mbd_unpack(V, mdb_val_ref)
-        else
-            throw(LMDB.LMDBError(ret))
+        key_ref = toref(convert(K,key))
+        GC.@preserve key_ref begin
+            mdb_key_ref = Ref(MDBValue(key_ref))
+            mdb_val_ref = Ref(MDBValue())
+            # Get value
+            ret = _mdb_get(txn.handle, dbi.handle, mdb_key_ref, mdb_val_ref)
+            if ret == MDB_NOTFOUND
+                d[key] = default
+                return default
+            elseif ret == Cint(0)
+                return mbd_unpack(V, mdb_val_ref)
+            else
+                throw(LMDB.LMDBError(ret))
+            end
         end
     end
 end
 
 function Base.get(f::F, d::LMDBDict{K,V}, key) where {K,V,F<:Union{Function, Type}}
     txn_dbi_do(d, readonly = true) do txn, dbi
-        mdb_key_ref = Ref(MDBValue(toref(convert(K,key))))
-        mdb_val_ref = Ref(MDBValue())
-        # Get value
-        ret = _mdb_get(txn.handle, dbi.handle, mdb_key_ref, mdb_val_ref)
-        if ret == MDB_NOTFOUND
-            default = f()
-            return default
-        elseif ret == Cint(0)
-            return mbd_unpack(V, mdb_val_ref)
-        else
-            throw(LMDB.LMDBError(ret))
+        key_ref = toref(convert(K,key))
+        GC.@preserve key_ref begin
+            mdb_key_ref = Ref(MDBValue(key_ref))
+            mdb_val_ref = Ref(MDBValue())
+            # Get value
+            ret = _mdb_get(txn.handle, dbi.handle, mdb_key_ref, mdb_val_ref)
+            if ret == MDB_NOTFOUND
+                default = f()
+                return default
+            elseif ret == Cint(0)
+                return mbd_unpack(V, mdb_val_ref)
+            else
+                throw(LMDB.LMDBError(ret))
+            end
         end
     end
 end
 
 function Base.get!(f::F, d::LMDBDict{K,V}, key) where {K,V,F<:Union{Function, Type}}
     txn_dbi_do(d, readonly = true) do txn, dbi
-        mdb_key_ref = Ref(MDBValue(toref(convert(K,key))))
-        mdb_val_ref = Ref(MDBValue())
-        # Get value
-        ret = _mdb_get(txn.handle, dbi.handle, mdb_key_ref, mdb_val_ref)
-        if ret == MDB_NOTFOUND
-            default = f()
-            d[key] = default
-            return default
-        elseif ret == Cint(0)
-            return mbd_unpack(V, mdb_val_ref)
-        else
-            throw(LMDB.LMDBError(ret))
+        key_ref = toref(convert(K,key))
+        GC.@preserve key_ref begin
+            mdb_key_ref = Ref(MDBValue(key_ref))
+            mdb_val_ref = Ref(MDBValue())
+            # Get value
+            ret = _mdb_get(txn.handle, dbi.handle, mdb_key_ref, mdb_val_ref)
+            if ret == MDB_NOTFOUND
+                default = f()
+                d[key] = default
+                return default
+            elseif ret == Cint(0)
+                return mbd_unpack(V, mdb_val_ref)
+            else
+                throw(LMDB.LMDBError(ret))
+            end
         end
     end
 end

test/dbi.jl

@@ -2,12 +2,11 @@ module LMDB_DBI
     using LMDB
     using Test
 
-    const dbname = "testdb"
     key = 10
     val = "key value is "
 
     # Create dir
-    mkdir(dbname)
+    mktempdir() do dbname
     try
 
         # Procedural style
@@ -23,8 +22,8 @@ module LMDB_DBI
             @test isopen(txn)
             commit(txn)
             @test !isopen(txn)
-            close(env, dbi)
-            @test !isopen(dbi)
+            # Don't close dbi after modifying it - this can cause corruption
+            # The environment will automatically close it when env is closed
         finally
             close(env)
         end
@@ -35,7 +34,7 @@ module LMDB_DBI
             set!(env, LMDB.MDB_NOSYNC)
             open(env, dbname)
             start(env) do txn
-                open(txn, flags = Cuint(LMDB.MDB_REVERSEKEY)) do dbi
+                open(txn) do dbi
                     k = key
                     value = get(txn, dbi, k, String)
                     println("Got value for key $(k): $(value)")
@@ -59,6 +58,7 @@ module LMDB_DBI
             end
         end
     finally
-        rm(dbname, recursive=true)
+        # mktempdir will clean up automatically
+    end
     end
 end