Skip to content

Commit af1163a

Browse files
marisnmarisn
authored
LMDB: fix LMDB environment management in case of multiple caches (#360)
Old code resulted in storage of all data in a single database New code stores all environments in a hash table with base path as a key thus linking cache with its LMBD environment Fixes: #358
1 parent 47bdb17 commit af1163a

File tree

1 file changed

+111
-46
lines changed

1 file changed

+111
-46
lines changed

‎lib/cache_lmdb.c‎

Lines changed: 111 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343

4444
#include "lmdb.h"
4545

46+
/* Cache specific configuration */
4647
typedef struct mapcache_cache_lmdb mapcache_cache_lmdb;
4748
struct mapcache_cache_lmdb {
4849
mapcache_cache cache;
@@ -51,17 +52,20 @@ struct mapcache_cache_lmdb {
5152
size_t max_size;
5253
unsigned int max_readers;
5354
MDB_env *env;
55+
MDB_dbi dbi;
5456
};
5557

56-
/* LMDB env should be opened only once per process */
58+
/* A LMDB DB environment for a single directory */
5759
typedef struct lmdb_env_s lmdb_env_s;
5860
struct lmdb_env_s {
5961
MDB_env *env;
6062
MDB_dbi dbi;
6163
int is_open;
6264
};
6365

64-
static lmdb_env_s *lmdb_env;
66+
/* A hash table of all open environments with directories as keys */
67+
static apr_hash_t* lmdb_env_ht = NULL;
68+
static apr_thread_mutex_t *lmdb_env_mutex = NULL;
6569

6670
static int _mapcache_cache_lmdb_has_tile(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile)
6771
{
@@ -71,7 +75,7 @@ static int _mapcache_cache_lmdb_has_tile(mapcache_context *ctx, mapcache_cache *
7175
mapcache_cache_lmdb *cache = (mapcache_cache_lmdb*)pcache;
7276
char *skey;
7377

74-
if (lmdb_env->is_open == 0) {
78+
if (!cache->env) {
7579
ctx->set_error(ctx,500,"lmdb is not open %s",cache->basedir);
7680
return MAPCACHE_FALSE;
7781
}
@@ -80,13 +84,13 @@ static int _mapcache_cache_lmdb_has_tile(mapcache_context *ctx, mapcache_cache *
8084
key.mv_size = strlen(skey)+1;
8185
key.mv_data = skey;
8286

83-
rc = mdb_txn_begin(lmdb_env->env, NULL, MDB_RDONLY, &txn);
87+
rc = mdb_txn_begin(cache->env, NULL, MDB_RDONLY, &txn);
8488
if (rc) {
8589
ctx->set_error(ctx,500,"lmdb failed to begin transaction for has_tile in %s:%s",cache->basedir,mdb_strerror(rc));
8690
return MAPCACHE_FALSE;
8791
}
8892

89-
rc = mdb_get(txn, lmdb_env->dbi, &key, &data);
93+
rc = mdb_get(txn, cache->dbi, &key, &data);
9094
if(rc == 0) {
9195
ret = MAPCACHE_TRUE;
9296
} else if(rc == MDB_NOTFOUND) {
@@ -113,7 +117,7 @@ static void _mapcache_cache_lmdb_delete(mapcache_context *ctx, mapcache_cache *p
113117
mapcache_cache_lmdb *cache = (mapcache_cache_lmdb*)pcache;
114118
char *skey;
115119

116-
if (lmdb_env->is_open == 0) {
120+
if (!cache->env) {
117121
ctx->set_error(ctx,500,"lmdb is not open %s",cache->basedir);
118122
return;
119123
}
@@ -122,13 +126,13 @@ static void _mapcache_cache_lmdb_delete(mapcache_context *ctx, mapcache_cache *p
122126
key.mv_size = strlen(skey)+1;
123127
key.mv_data = skey;
124128

125-
rc = mdb_txn_begin(lmdb_env->env, NULL, 0, &txn);
129+
rc = mdb_txn_begin(cache->env, NULL, 0, &txn);
126130
if (rc) {
127131
ctx->set_error(ctx,500,"lmdb failed to begin transaction for delete in %s:%s",cache->basedir,mdb_strerror(rc));
128132
return;
129133
}
130134

131-
rc = mdb_del(txn, lmdb_env->dbi, &key, NULL);
135+
rc = mdb_del(txn, cache->dbi, &key, NULL);
132136
if (rc) {
133137
if (rc == MDB_NOTFOUND) {
134138
ctx->log(ctx,MAPCACHE_DEBUG,"attempt to delete tile %s absent in the db %s",skey,cache->basedir);
@@ -152,7 +156,7 @@ static int _mapcache_cache_lmdb_get(mapcache_context *ctx, mapcache_cache *pcach
152156
char *skey;
153157
mapcache_cache_lmdb *cache = (mapcache_cache_lmdb*)pcache;
154158

155-
if (lmdb_env->is_open == 0) {
159+
if (!cache->env) {
156160
ctx->set_error(ctx,500,"lmdb is not open %s",cache->basedir);
157161
return MAPCACHE_FALSE;
158162
}
@@ -161,13 +165,13 @@ static int _mapcache_cache_lmdb_get(mapcache_context *ctx, mapcache_cache *pcach
161165
key.mv_size = strlen(skey)+1;
162166
key.mv_data = skey;
163167

164-
rc = mdb_txn_begin(lmdb_env->env, NULL, MDB_RDONLY, &txn);
168+
rc = mdb_txn_begin(cache->env, NULL, MDB_RDONLY, &txn);
165169
if (rc) {
166170
ctx->set_error(ctx,500,"lmdb failed to begin transaction for get in %s:%s",cache->basedir,mdb_strerror(rc));
167171
return MAPCACHE_FALSE;
168172
}
169173

170-
rc = mdb_get(txn, lmdb_env->dbi, &key, &data);
174+
rc = mdb_get(txn, cache->dbi, &key, &data);
171175
if(rc == 0) {
172176
if(((char*)(data.mv_data))[0] == '#') {
173177
tile->encoded_data = mapcache_empty_png_decode(ctx,tile->grid_link->grid->tile_sx, tile->grid_link->grid->tile_sy, (unsigned char*)data.mv_data,&tile->nodata);
@@ -178,6 +182,7 @@ static int _mapcache_cache_lmdb_get(mapcache_context *ctx, mapcache_cache *pcach
178182
tile->encoded_data->avail = data.mv_size;
179183
}
180184
tile->mtime = *((apr_time_t*)(((char*)data.mv_data)+data.mv_size-sizeof(apr_time_t)));
185+
181186
ret = MAPCACHE_SUCCESS;
182187
} else if(rc == MDB_NOTFOUND) {
183188
ret = MAPCACHE_CACHE_MISS;
@@ -232,18 +237,18 @@ static void _mapcache_cache_lmdb_set(mapcache_context *ctx, mapcache_cache *pcac
232237
tile->encoded_data->size -= sizeof(apr_time_t);
233238
}
234239

235-
if (lmdb_env->is_open == 0) {
240+
if (!cache->env) {
236241
ctx->set_error(ctx,500,"lmdb is not open %s",cache->basedir);
237242
return;
238243
}
239244

240-
rc = mdb_txn_begin(lmdb_env->env, NULL, 0, &txn);
245+
rc = mdb_txn_begin(cache->env, NULL, 0, &txn);
241246
if (rc) {
242247
ctx->set_error(ctx,500,"lmdb failed to begin transaction for set in %s:%s",cache->basedir,mdb_strerror(rc));
243248
return;
244249
}
245250

246-
rc = mdb_put(txn, lmdb_env->dbi, &key, &data, 0);
251+
rc = mdb_put(txn, cache->dbi, &key, &data, 0);
247252
if(rc) {
248253
ctx->set_error(ctx,500,"lmbd failed to put for tile_set in %s:%s",cache->basedir,mdb_strerror(rc));
249254
}
@@ -265,12 +270,12 @@ static void _mapcache_cache_lmdb_multiset(mapcache_context *ctx, mapcache_cache
265270

266271
now = apr_time_now();
267272

268-
if (lmdb_env->is_open == 0) {
273+
if (!cache->env) {
269274
ctx->set_error(ctx,500,"lmdb is not open %s",cache->basedir);
270275
return;
271276
}
272277

273-
rc = mdb_txn_begin(lmdb_env->env, NULL, 0, &txn);
278+
rc = mdb_txn_begin(cache->env, NULL, 0, &txn);
274279
if (rc) {
275280
ctx->set_error(ctx,500,"lmdb failed to begin transaction for multiset in %s:%s",cache->basedir,mdb_strerror(rc));
276281
return;
@@ -308,7 +313,7 @@ static void _mapcache_cache_lmdb_multiset(mapcache_context *ctx, mapcache_cache
308313
key.mv_data = skey;
309314
key.mv_size = strlen(skey)+1;
310315

311-
rc = mdb_put(txn, lmdb_env->dbi, &key, &data, 0);
316+
rc = mdb_put(txn, cache->dbi, &key, &data, 0);
312317
if(rc) {
313318
ctx->set_error(ctx,500,"lmbd failed to put for multiset in %s:%s",cache->basedir,mdb_strerror(rc));
314319
goto abort_txn;
@@ -386,76 +391,136 @@ static void _mapcache_cache_lmdb_configuration_post_config(mapcache_context *ctx
386391
}
387392
}
388393

394+
/**
395+
* Clean-up LMDB at shutdown
396+
*
397+
* \private \memberof mapcache_cache_lmdb
398+
*/
399+
static apr_status_t _lmdb_cleanup(void *data) {
400+
apr_hash_index_t *hi;
401+
if(lmdb_env_ht) {
402+
for (hi = apr_hash_first(NULL, lmdb_env_ht); hi; hi = apr_hash_next(hi)) {
403+
lmdb_env_s *env_s;
404+
apr_hash_this(hi, NULL, NULL, (void**)&env_s);
405+
if(env_s->is_open) {
406+
mdb_dbi_close(env_s->env, env_s->dbi);
407+
mdb_env_close(env_s->env);
408+
env_s->is_open = 0;
409+
}
410+
}
411+
lmdb_env_ht = NULL;
412+
}
413+
if(lmdb_env_mutex) {
414+
apr_thread_mutex_destroy(lmdb_env_mutex);
415+
lmdb_env_mutex = NULL;
416+
}
417+
return APR_SUCCESS;
418+
}
419+
420+
421+
/**
422+
* Open LMDB database at a process start
423+
*
424+
* LMDB requires single DB environment to be opened only once per process
425+
* thus here new environments are created when a new process starts
426+
*/
389427
static void _mapcache_cache_lmdb_child_init(mapcache_context *ctx, mapcache_cache *cache, apr_pool_t *pchild)
390428
{
391429
mapcache_cache_lmdb *dcache = (mapcache_cache_lmdb*)cache;
392-
393430
int rc, dead=0;
394431
MDB_txn *txn;
432+
lmdb_env_s *env_s = NULL;
433+
434+
if(!lmdb_env_mutex) {
435+
apr_thread_mutex_create(&lmdb_env_mutex, APR_THREAD_MUTEX_DEFAULT, pchild);
436+
}
437+
438+
apr_thread_mutex_lock(lmdb_env_mutex);
439+
440+
if(!lmdb_env_ht) {
441+
lmdb_env_ht = apr_hash_make(pchild);
442+
apr_pool_cleanup_register(pchild, NULL, _lmdb_cleanup, apr_pool_cleanup_null);
443+
}
444+
445+
env_s = apr_hash_get(lmdb_env_ht, dcache->basedir, APR_HASH_KEY_STRING);
446+
447+
/* Environment for particular base dir is alreay open */
448+
if(env_s) {
449+
dcache->env = env_s->env;
450+
dcache->dbi = env_s->dbi;
451+
apr_thread_mutex_unlock(lmdb_env_mutex);
452+
return;
453+
}
454+
455+
env_s = apr_pcalloc(pchild,sizeof(lmdb_env_s));
456+
rc = mdb_env_create(&(env_s->env));
395457

396-
lmdb_env_s *var = apr_pcalloc(ctx->pool,sizeof(lmdb_env_s));
397-
lmdb_env = var;
398-
lmdb_env->is_open = 0;
399-
rc = mdb_env_create(&(lmdb_env->env));
400458
if (rc) {
401459
ctx->set_error(ctx,500,"lmdb failed to create environment of database %s:%s",dcache->basedir,mdb_strerror(rc));
402-
return;
460+
goto cleanup;
403461
}
404-
rc = mdb_env_set_mapsize(lmdb_env->env, dcache->max_size);
462+
rc = mdb_env_set_mapsize(env_s->env, dcache->max_size);
405463
if (rc) {
406464
ctx->set_error(ctx,500,"lmdb failed to set maximum size of database %s:%s",dcache->basedir,mdb_strerror(rc));
407-
mdb_env_close(lmdb_env->env);
408-
return;
465+
mdb_env_close(env_s->env);
466+
goto cleanup;
409467
}
410468
if (dcache->max_readers) {
411-
rc = mdb_env_set_maxreaders(lmdb_env->env, dcache->max_readers);
469+
rc = mdb_env_set_maxreaders(env_s->env, dcache->max_readers);
412470
if (rc) {
413471
ctx->set_error(ctx,500,"lmdb failed to set maximum readers of database %s:%s",dcache->basedir,mdb_strerror(rc));
414-
mdb_env_close(lmdb_env->env);
415-
return;
472+
mdb_env_close(env_s->env);
473+
goto cleanup;
416474
}
417475
}
418476
/* Clean out any stale reader entries from lock table */
419-
rc = mdb_reader_check(lmdb_env->env, &dead);
477+
rc = mdb_reader_check(env_s->env, &dead);
420478
if (rc) {
421479
ctx->set_error(ctx,500,"lmdb failed to clear stale readers of database %s:%s",dcache->basedir,mdb_strerror(rc));
422-
mdb_env_close(lmdb_env->env);
423-
return;
480+
mdb_env_close(env_s->env);
481+
goto cleanup;
424482
}
425483
if (dead) {
426484
ctx->log(ctx,MAPCACHE_NOTICE,"lmdb cleared %d stale readers of database %s",dead,dcache->basedir);
427485
}
428-
rc = mdb_env_open(lmdb_env->env, dcache->basedir, 0, 0664);
486+
rc = mdb_env_open(env_s->env, dcache->basedir, 0, 0664);
429487
if (rc) {
430488
ctx->set_error(ctx,500,"lmdb failed to open environment of database %s:%s",dcache->basedir,mdb_strerror(rc));
431-
mdb_env_close(lmdb_env->env);
432-
return;
489+
mdb_env_close(env_s->env);
490+
goto cleanup;
433491
}
434-
rc = mdb_txn_begin(lmdb_env->env, NULL, MDB_CREATE, &txn);
492+
rc = mdb_txn_begin(env_s->env, NULL, MDB_CREATE, &txn);
435493
if (rc) {
436494
ctx->set_error(ctx,500,"lmdb failed to begin transaction of database %s:%s",dcache->basedir,mdb_strerror(rc));
437-
mdb_env_close(lmdb_env->env);
438-
return;
495+
mdb_env_close(env_s->env);
496+
goto cleanup;
439497
}
440-
rc = mdb_dbi_open(txn, NULL, 0, &(lmdb_env->dbi));
498+
rc = mdb_dbi_open(txn, NULL, 0, &(env_s->dbi));
441499
if (rc) {
442500
ctx->set_error(ctx,500,"lmdb failed to open dbi of database %s:%s",dcache->basedir,mdb_strerror(rc));
443501
mdb_txn_abort(txn);
444-
mdb_env_close(lmdb_env->env);
445-
return;
502+
mdb_env_close(env_s->env);
503+
goto cleanup;
446504
}
447505
rc = mdb_txn_commit(txn);
448506
if (rc) {
449507
ctx->set_error(ctx,500,"lmdb failed to commit transaction of database %s:%s",dcache->basedir,mdb_strerror(rc));
450-
mdb_dbi_close(lmdb_env->env, lmdb_env->dbi);
451-
mdb_env_close(lmdb_env->env);
452-
return;
508+
mdb_dbi_close(env_s->env, env_s->dbi);
509+
mdb_env_close(env_s->env);
510+
goto cleanup;
453511
}
454-
lmdb_env->is_open = 1;
512+
env_s->is_open = 1;
513+
dcache->env = env_s->env;
514+
dcache->dbi = env_s->dbi;
515+
apr_hash_set(lmdb_env_ht, dcache->basedir, APR_HASH_KEY_STRING, env_s);
516+
517+
cleanup:
518+
apr_thread_mutex_unlock(lmdb_env_mutex);
455519
}
456520

521+
457522
/**
458-
* \brief creates and initializes a mapcache_dbd_cache
523+
* \brief creates and initializes a mapcache_lmdb_cache
459524
*/
460525
mapcache_cache* mapcache_cache_lmdb_create(mapcache_context *ctx)
461526
{

0 commit comments

Comments
 (0)