Merge pull request ceph#52933 from dang/wip-dang-posix-driver

RGW - Add POSIX Driver

Reviewed-by: Casey Bodley <cbodley@redhat.com>
Reviewed-by: Matt Benjamin <mbenjamin@redhat.com>

Author: dang

CMakeLists.txt

@@ -454,6 +454,7 @@ option(WITH_RADOSGW_DBSTORE "DBStore backend for RADOS Gateway" ON)
 option(WITH_RADOSGW_MOTR "CORTX-Motr backend for RADOS Gateway" OFF)
 option(WITH_RADOSGW_DAOS "DAOS backend for RADOS Gateway" OFF)
 option(WITH_RADOSGW_D4N "D4N wrapper for RADOS Gateway" ON)
+option(WITH_RADOSGW_POSIX "POSIX backend for Rados Gateway" ON)
 option(WITH_RADOSGW_SELECT_PARQUET "Support for s3 select on parquet objects" ON)
 option(WITH_RADOSGW_ARROW_FLIGHT "Build arrow flight when not using system-provided arrow" OFF)
 option(WITH_RADOSGW_BACKTRACE_LOGGING "Enable backtraces in rgw logs" OFF)

ceph.spec.in

@@ -270,6 +270,7 @@ BuildRequires:	xfsprogs-devel
 BuildRequires:	xmlstarlet
 BuildRequires:	nasm
 BuildRequires:	lua-devel
+BuildRequires:  lmdb-devel
 %if 0%{with seastar} || 0%{with jaeger}
 BuildRequires:  yaml-cpp-devel >= 0.6
 %endif

cmake/modules/FindLMDB.cmake

@@ -0,0 +1,61 @@
+# Copyright (c) 2014, The Monero Project
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+# 
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+#    conditions and the following disclaimer.
+# 
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+#    of conditions and the following disclaimer in the documentation and/or other
+#    materials provided with the distribution.
+# 
+# 3. Neither the name of the copyright holder nor the names of its contributors may be
+#    used to endorse or promote products derived from this software without specific
+#    prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+MESSAGE(STATUS "Looking for liblmdb")
+
+FIND_PATH(LMDB_INCLUDE_DIR
+  NAMES lmdb.h
+  PATH_SUFFIXES include/ include/lmdb/
+  PATHS "${PROJECT_SOURCE_DIR}"
+  ${LMDB_ROOT}
+  $ENV{LMDB_ROOT}
+  /usr/local/
+  /usr/
+)
+
+if(STATIC)
+  if(MINGW)
+    find_library(LMDB_LIBRARIES liblmdb.dll.a)
+  else()
+    find_library(LMDB_LIBRARIES liblmdb.a)
+  endif()
+else()
+  find_library(LMDB_LIBRARIES lmdb)
+endif()
+
+IF(LMDB_INCLUDE_DIR)
+  MESSAGE(STATUS "Found liblmdb include (lmdb.h) in ${LMDB_INCLUDE_DIR}")
+  IF(LMDB_LIBRARIES)
+    MESSAGE(STATUS "Found liblmdb library")
+    set(LMDB_INCLUDE ${LMDB_INCLUDE_DIR})
+    set(LMDB_LIBRARY ${LMDB_LIBRARIES})
+  ELSE()
+    MESSAGE(FATAL_ERROR "${BoldRed}Could not find liblmdb library, please make sure you have installed liblmdb or liblmdb-dev or the equivalent${ColourReset}")
+  ENDIF()
+ELSE()
+  MESSAGE(FATAL_ERROR "${BoldRed}Could not find liblmdb library, please make sure you have installed liblmdb or liblmdb-dev or the equivalent${ColourReset}")
+ENDIF()

debian/control

@@ -65,6 +65,7 @@ Build-Depends: automake,
                libsqlite3-dev,
                libssl-dev,
                libtool,
+               liblmdb-dev,
                libudev-dev,
                libnl-genl-3-dev,
                libxml2-dev,

src/common/cohort_lru.h

@@ -16,6 +16,12 @@
 #include <boost/intrusive/list.hpp>
 #include <boost/intrusive/slist.hpp>
 
+#ifdef __CEPH__
+# include "include/ceph_assert.h"
+#else
+# include <assert.h>
+#endif
+
 #include "common/likely.h"
 
 #ifndef CACHE_LINE_SIZE

src/common/options/rgw.yaml.in

@@ -3618,6 +3618,7 @@ options:
   - none
   - base
   - d4n
+  - posix
 - name: dbstore_db_dir
   type: str
   level: advanced
@@ -3705,6 +3706,51 @@ options:
   default: false
   services:
   - rgw
+- name: rgw_posix_base_path
+  type: str
+  level: advanced
+  desc: experimental Option to set base path for POSIX Driver
+  long_desc: Base path for the POSIX driver.  All operations are relative to this path.
+    Defaults to /tmp/rgw_posix_driver
+  default: /tmp/rgw_posix_driver
+  services:
+  - rgw
+- name: rgw_posix_database_root
+  type: str
+  level: advanced
+  desc: experimental Path to parent of POSIX Driver LMDB bucket listing cache
+  long_desc: Parent directory of LMDB bucket listing cache databases.
+  default: /var/lib/ceph/radosgw
+  services:
+  - rgw
+- name: rgw_posix_cache_max_buckets
+  type: int
+  level: advanced
+  desc: experimental Number of buckets to maintain in the ordered listing cache
+  default: 100
+  services:
+  - rgw
+- name: rgw_posix_cache_lanes
+  type: int
+  level: advanced
+  desc: experimental Number of lanes in cache LRU
+  default: 3
+  services:
+  - rgw
+- name: rgw_posix_cache_partitions
+  type: int
+  level: advanced
+  desc: experimental Number of partitions in cache AVL
+  default: 3
+  services:
+  - rgw
+- name: rgw_posix_cache_lmdb_count
+  type: int
+  level: advanced
+  desc: experimental Number of lmdb partitions in the ordered listing cache
+  default: 3
+  services:
+  - rgw
 - name: rgw_luarocks_location
   type: str
   level: advanced

src/include/config-h.in.cmake

@@ -363,6 +363,9 @@
 /* Backend CORTX-DAOS for Rados Gateway */
 #cmakedefine WITH_RADOSGW_DAOS
 
+/* Backend POSIX for Rados Gateway */
+#cmakedefine WITH_RADOSGW_POSIX
+
 /* Defined if std::map::merge() is supported */
 #cmakedefine HAVE_STDLIB_MAP_SPLICING
 

src/rgw/CMakeLists.txt

@@ -227,6 +227,15 @@ if(WITH_RADOSGW_D4N)
   list(APPEND librgw_common_srcs driver/d4n/d4n_datacache.cc)
   list(APPEND librgw_common_srcs driver/d4n/rgw_sal_d4n.cc)
 endif()
+if(WITH_RADOSGW_POSIX)
+  #add_subdirectory(driver/posix)
+  find_package(LMDB REQUIRED)
+  add_compile_definitions(LMDB_SAFE_NO_CPP_UTILITIES)
+  list(APPEND librgw_common_srcs
+	      driver/posix/rgw_sal_posix.cc
+	      driver/posix/lmdb-safe.cc
+	      driver/posix/notify.cpp)
+endif()
 if(WITH_JAEGER)
   list(APPEND librgw_common_srcs rgw_tracer.cc)
 endif()
@@ -236,7 +245,6 @@ if(WITH_RADOSGW_ARROW_FLIGHT)
   list(APPEND librgw_common_srcs rgw_flight.cc rgw_flight_frontend.cc)
 endif(WITH_RADOSGW_ARROW_FLIGHT)
 
-
 add_library(rgw_common STATIC ${librgw_common_srcs})
 
 include(CheckCXXCompilerFlag)
@@ -270,6 +278,7 @@ target_link_libraries(rgw_common
     ${EXPAT_LIBRARIES}
     ${ARROW_LIBRARIES}
     ${ARROW_FLIGHT_LIBRARIES}
+    ${LMDB_LIBRARIES}
     ${ALLOC_LIBS}
   PUBLIC
     ${LUA_LIBRARIES}

src/rgw/driver/posix/README.md

@@ -0,0 +1,37 @@
+# POSIX Driver
+Standalone Rados Gateway (RGW) on a local POSIX filesystem (Experimental)
+
+
+## CMake Option
+Add below cmake option (enabled by default)
+
+    -DWITH_RADOSGW_POSIX=ON 
+
+
+## Build
+
+    cd build
+    ninja [vstart]
+
+
+## Running Test cluster
+Currently, POSIXDriver depends on DBStore for user storage.  This will change, eventually, but for now, it's run as a filter on top of DBStore.  Not that only users are stored in DBStore, the rest is in the POSIX filesystem.
+Edit ceph.conf to add below option
+
+    [client]
+        rgw backend store = dbstore
+        rgw config store = dbstore
+        rgw filter = posix
+
+Start vstart cluster
+
+    MON=0 OSD=0 MDS=0 MGR=0 RGW=1 ../src/vstart.sh -o rgw_backend_store=dbstore -o rgw_config_store=dbstore -o rgw_filter=posix -n -d
+
+The above vstart command brings up RGW server on POSIXDriver. It creates default zonegroup, zone and few default users (eg., testid) to be used for s3 operations.
+
+`radosgw-admin` can be used to create and remove other users, zonegroups and zones.
+
+By default, the directory exported is *'/tmp/rgw_posix_driver'*.   This can be changed with the `rgw_posix_base_path` option, either in ceph.conf or on the vstart command line above.
+
+The POSIXDriver keeps a LMDB based cache of directories, so that it can provide ordered listings.  This directory lives in `rgw_posix_database_root`, which by default is in *'/var/lib/ceph/radosgw'*
+

src/rgw/driver/posix/bucket_cache.cpp

@@ -0,0 +1,4 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+#include "bucket_cache.h"
+