Skip to content

Commit 440fded

Browse files
Stefan EilemannStefan Eilemann
authored andcommitted
memcached::PersistentMap
1 parent 7062df5 commit 440fded

File tree

10 files changed

+247
-41
lines changed

10 files changed

+247
-41
lines changed

CMakeLists.txt

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,9 @@ set(LUNCHBOX_LICENSE LGPL)
2222
set(LUNCHBOX_DEB_DEPENDS libboost-regex-dev libboost-serialization-dev
2323
libboost-filesystem-dev libboost-system-dev libboost-test-dev
2424
libboost-thread-dev libhwloc-dev avahi-daemon libavahi-client-dev
25-
libleveldb-dev libopenmpi-dev openmpi-bin)
26-
set(LUNCHBOX_PORT_DEPENDS boost)
25+
libleveldb-dev libopenmpi-dev openmpi-bin libmemcached-dev libmemcached-tools
26+
memcached)
27+
set(LUNCHBOX_PORT_DEPENDS boost libmemcached memcached)
2728
set(COMMON_PROJECT_DOMAIN ch.eyescale)
2829

2930
include(Common)
@@ -34,15 +35,16 @@ else()
3435
list(APPEND COMMON_FIND_PACKAGE_DEFINES LUNCHBOX_USE_V1_API)
3536
endif()
3637

37-
common_find_package(Boost REQUIRED COMPONENTS regex serialization filesystem system
38-
thread unit_test_framework)
38+
common_find_package(Boost REQUIRED COMPONENTS
39+
filesystem regex serialization system thread unit_test_framework)
3940
common_find_package(hwloc)
4041
common_find_package(leveldb)
4142
if(LUNCHBOX_USE_MPI)
4243
common_find_package(MPI)
4344
endif()
4445
common_find_package(OpenMP)
4546
common_find_package(Servus REQUIRED)
47+
common_find_package(libmemcached)
4648
common_find_package(skv)
4749
common_find_package_post()
4850

doc/Changelog.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22

33
# git master
44

5+
* [263](https://github.com/Eyescale/Lunchbox/pull/263):
6+
Add memcached PersistentMap backend, add
7+
PersistentMap::createCache
58
* [252](https://github.com/Eyescale/Lunchbox/pull/252):
69
Monitor::set() returns old value
710

lunchbox/CMakeLists.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,9 @@ endif()
138138
if(LEVELDB_FOUND)
139139
list(APPEND LUNCHBOX_LINK_LIBRARIES ${LEVELDB_LIBRARIES})
140140
endif()
141+
if(LIBMEMCACHED_FOUND)
142+
list(APPEND LUNCHBOX_LINK_LIBRARIES ${libmemcached_LIBRARIES})
143+
endif()
141144
if(SKV_FOUND)
142145
list(APPEND LUNCHBOX_LINK_LIBRARIES skv_client fxlogger)
143146
endif()

lunchbox/leveldb/persistentMap.h

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11

2-
/* Copyright (c) 2014, Stefan.Eilemann@epfl.ch
2+
/* Copyright (c) 2014-2016, Stefan.Eilemann@epfl.ch
33
*
44
* This library is free software; you can redistribute it and/or modify it under
55
* the terms of the GNU Lesser General Public License version 2.1 as published
@@ -68,12 +68,6 @@ class PersistentMap : public detail::PersistentMap
6868
return std::string();
6969
}
7070

71-
bool contains( const std::string& key ) const final
72-
{
73-
std::string value;
74-
return _db->Get( db::ReadOptions(), key, &value ).ok();
75-
}
76-
7771
bool flush() final { /*NOP?*/ return true; }
7872

7973
private:

lunchbox/memcached/persistentMap.h

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
2+
/* Copyright (c) 2016, Stefan.Eilemann@epfl.ch
3+
*
4+
* This library is free software; you can redistribute it and/or modify it under
5+
* the terms of the GNU Lesser General Public License version 2.1 as published
6+
* by the Free Software Foundation.
7+
*
8+
* This library is distributed in the hope that it will be useful, but WITHOUT
9+
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
10+
* FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
11+
* details.
12+
*
13+
* You should have received a copy of the GNU Lesser General Public License
14+
* along with this library; if not, write to the Free Software Foundation, Inc.,
15+
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
16+
*/
17+
18+
#ifdef LUNCHBOX_USE_LIBMEMCACHED
19+
#include <libmemcached/memcached.h>
20+
21+
namespace lunchbox
22+
{
23+
namespace memcached
24+
{
25+
namespace
26+
{
27+
memcached_st* _getInstance( const servus::URI& uri )
28+
{
29+
const std::string& host = uri.getHost();
30+
const int16_t port = uri.getPort() ? uri.getPort() : 11211;
31+
memcached_st* instance = memcached_create( 0 );
32+
33+
if( uri.getHost().empty( ))
34+
{
35+
const char* servers = ::getenv( "MEMCACHED_SERVERS" );
36+
if( servers )
37+
{
38+
instance->server_failure_limit = 10;
39+
std::string data = servers;
40+
while( !data.empty( ))
41+
{
42+
const size_t comma = data.find( ',' );
43+
const std::string& server = data.substr( 0, comma );
44+
45+
const size_t colon = server.find( ':' );
46+
if( colon == std::string::npos )
47+
memcached_server_add( instance, server.c_str(), port );
48+
else
49+
memcached_server_add( instance,
50+
server.substr( 0, colon ).c_str(),
51+
std::stoi( server.substr( colon+1 )));
52+
53+
if( comma == std::string::npos )
54+
data.clear();
55+
else
56+
data = data.substr( comma + 1 );
57+
}
58+
59+
}
60+
else
61+
memcached_server_add( instance, "127.0.0.1", port );
62+
63+
}
64+
else
65+
memcached_server_add( instance, host.c_str(), port );
66+
67+
return instance;
68+
}
69+
}
70+
71+
// memcached has relative strict requirements on keys (no whitespace or control
72+
// characters, max length). We therefore hash incoming keys and use their string
73+
// representation.
74+
75+
class PersistentMap : public detail::PersistentMap
76+
{
77+
public:
78+
explicit PersistentMap( const servus::URI& uri )
79+
: _instance( _getInstance( uri ))
80+
, _lastError( MEMCACHED_SUCCESS )
81+
{
82+
if( !_instance )
83+
throw std::runtime_error( std::string( "Open of " ) +
84+
std::to_string( uri ) + " failed" );
85+
}
86+
87+
virtual ~PersistentMap() { memcached_free( _instance ); }
88+
89+
static bool handles( const servus::URI& uri )
90+
{ return uri.getScheme() == "memcached"; }
91+
92+
bool insert( const std::string& key, const void* data, const size_t size )
93+
final
94+
{
95+
const std::string& hash = servus::make_uint128( key ).getString();
96+
const memcached_return_t ret =
97+
memcached_set( _instance, hash.c_str(), hash.length(),
98+
(const char*)data, size, (time_t)0, (uint32_t)0 );
99+
100+
if( ret != MEMCACHED_SUCCESS && _lastError != ret )
101+
{
102+
LBWARN << "memcached_set failed: "
103+
<< memcached_strerror( _instance, ret ) << std::endl;
104+
_lastError = ret;
105+
}
106+
107+
return ret == MEMCACHED_SUCCESS;
108+
}
109+
110+
std::string operator [] ( const std::string& key ) const final
111+
{
112+
const std::string& hash = servus::make_uint128( key ).getString();
113+
size_t size = 0;
114+
uint32_t flags = 0;
115+
memcached_return_t ret = MEMCACHED_SUCCESS;
116+
char* data = memcached_get( _instance, hash.c_str(), hash.length(),
117+
&size, &flags, &ret );
118+
std::string value;
119+
if( ret == MEMCACHED_SUCCESS )
120+
{
121+
value.assign( data, data + size );
122+
free( data );
123+
}
124+
return value;
125+
}
126+
127+
bool flush() final { /*NOP?*/ return true; }
128+
129+
private:
130+
memcached_st* const _instance;
131+
memcached_return_t _lastError;
132+
};
133+
}
134+
}
135+
136+
#endif

lunchbox/persistentMap.cpp

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11

2-
/* Copyright (c) 2014-2015, Stefan.Eilemann@epfl.ch
2+
/* Copyright (c) 2014-2016, Stefan.Eilemann@epfl.ch
33
*
44
* This library is free software; you can redistribute it and/or modify it under
55
* the terms of the GNU Lesser General Public License version 2.1 as published
@@ -35,7 +35,6 @@ class PersistentMap
3535
virtual std::string operator [] ( const std::string& key ) const = 0;
3636
virtual bool fetch( const std::string&, const size_t ) const
3737
{ return true; }
38-
virtual bool contains( const std::string& key ) const = 0;
3938
virtual bool flush() = 0;
4039

4140
bool swap;
@@ -49,6 +48,7 @@ class PersistentMap
4948

5049
// Impls - need detail::PersistentMap interface above
5150
#include "leveldb/persistentMap.h"
51+
#include "memcached/persistentMap.h"
5252
#include "skv/persistentMap.h"
5353

5454
namespace
@@ -60,6 +60,10 @@ lunchbox::detail::PersistentMap* _newImpl( const servus::URI& uri )
6060
if( lunchbox::leveldb::PersistentMap::handles( uri ))
6161
return new lunchbox::leveldb::PersistentMap( uri );
6262
#endif
63+
#ifdef LUNCHBOX_USE_LIBMEMCACHED
64+
if( lunchbox::memcached::PersistentMap::handles( uri ))
65+
return new lunchbox::memcached::PersistentMap( uri );
66+
#endif
6367
#ifdef LUNCHBOX_USE_SKV
6468
if( lunchbox::skv::PersistentMap::handles( uri ))
6569
return new lunchbox::skv::PersistentMap( uri );
@@ -102,12 +106,32 @@ PersistentMap::~PersistentMap()
102106
delete _impl;
103107
}
104108

109+
PersistentMapPtr PersistentMap::createCache()
110+
{
111+
#ifdef LUNCHBOX_USE_LIBMEMCACHED
112+
if( ::getenv( "MEMCACHED_SERVERS" ))
113+
return PersistentMapPtr( new PersistentMap( "memcached://" ));
114+
#endif
115+
#ifdef LUNCHBOX_USE_LEVELDB
116+
const char* leveldb = ::getenv( "LEVELDB_CACHE" );
117+
if( leveldb )
118+
return PersistentMapPtr( new PersistentMap(
119+
std::string( "leveldb://" ) + leveldb ));
120+
#endif
121+
122+
return PersistentMapPtr();
123+
}
124+
105125
bool PersistentMap::handles( const servus::URI& uri )
106126
{
107127
#ifdef LUNCHBOX_USE_LEVELDB
108128
if( lunchbox::leveldb::PersistentMap::handles( uri ))
109129
return true;
110130
#endif
131+
#ifdef LUNCHBOX_USE_LIBMEMCACHED
132+
if( lunchbox::memcached::PersistentMap::handles( uri ))
133+
return true;
134+
#endif
111135
#ifdef LUNCHBOX_USE_SKV
112136
if( lunchbox::skv::PersistentMap::handles( uri ))
113137
return true;
@@ -127,8 +151,8 @@ size_t PersistentMap::setQueueDepth( const size_t depth )
127151
return _impl->setQueueDepth( depth );
128152
}
129153

130-
bool PersistentMap::_insert( const std::string& key, const void* data,
131-
const size_t size )
154+
bool PersistentMap::insert( const std::string& key, const void* data,
155+
const size_t size )
132156
{
133157
#ifdef HISTOGRAM
134158
++_impl->keys[ key.size() ];
@@ -147,11 +171,6 @@ bool PersistentMap::fetch( const std::string& key, const size_t sizeHint ) const
147171
return _impl->fetch( key, sizeHint );
148172
}
149173

150-
bool PersistentMap::contains( const std::string& key ) const
151-
{
152-
return _impl->contains( key );
153-
}
154-
155174
bool PersistentMap::flush()
156175
{
157176
return _impl->flush();

0 commit comments

Comments
 (0)