client: support multi-threading

.clang-format

@@ -8,9 +8,11 @@ UseTab: Always
 AccessModifierOffset: -8
 AlignAfterOpenBracket: Align
 AlwaysBreakAfterReturnType: TopLevel
+AlwaysBreakTemplateDeclarations: Yes
 ColumnLimit: 120
 ContinuationIndentWidth: 8
 Cpp11BracedListStyle: true
 SpaceInEmptyBlock: false
+PackConstructorInitializers: CurrentLine
 PointerAlignment: Right
 ReferenceAlignment: Right

.github/actions/build-tntcxx/action.yml

@@ -6,7 +6,7 @@ inputs:
     required: true
   enable-sanitizers:
     description: 'Corresponds to TNTCXX_ENABLE_SANITIZERS option of CMake'
-    default: 'false'
+    default: ''
   cxx-standard:
     description: 'Corresponds to CMAKE_CXX_STANDARD option of CMake'
   cxx-compiler:

.github/actions/setup-tarantool/action.yml

@@ -10,7 +10,7 @@ runs:
   steps:
     - name: Setup Tarantool 2.11
       if: startsWith(inputs.runs-on, 'ubuntu')
-      uses: tarantool/setup-tarantool@v1
+      uses: tarantool/setup-tarantool@v3
       with:
         tarantool-version: '2.11'
 

.github/workflows/reusable-testing.yml

@@ -22,8 +22,8 @@ on:
         default: false
         type: boolean
       enable-sanitizers:
-        default: false
-        type: boolean
+        default: ''
+        type: string
       build-only:
         default: false
         type: boolean

.github/workflows/testing.yml

@@ -100,10 +100,13 @@ jobs:
         exclude:
           - runs-on: macos-14
             compiler: {c: gcc, cxx: g++}
-    name: sanitizers (${{ matrix.runs-on }}, ${{ matrix.build-type }}, ${{ matrix.compiler.c }})
+        sanitizer:
+          - thread
+          - address
+    name: sanitizers (${{ matrix.sanitizer }}, ${{ matrix.runs-on }}, ${{ matrix.build-type }}, ${{ matrix.compiler.c }})
     with:
       runs-on: ${{ matrix.runs-on }}
       build-type: ${{ matrix.build-type }}
       c-compiler: ${{ matrix.compiler.c }}
       cxx-compiler: ${{ matrix.compiler.cxx }}
-      enable-sanitizers: true
+      enable-sanitizers: ${{ matrix.sanitizer }}

CMakeLists.txt

@@ -86,13 +86,17 @@ IF (TNTCXX_BUILD_TESTING)
     # Retrieve the source directory to later get the header path.
     FETCHCONTENT_GETPROPERTIES(msgpuck)
     FETCHCONTENT_MAKEAVAILABLE(msgpuck)
+
+    find_package(Threads REQUIRED)
 ENDIF()
 
 OPTION(TNTCXX_ENABLE_SANITIZERS
-    "If ON, tntcxx will be instrumented with sanitizers."
+    "If passed, tntcxx will be instrumented with sanitizers. Possible values: \"address\", \"thread\""
     OFF)
 
-IF (TNTCXX_ENABLE_SANITIZERS)
+IF (NOT TNTCXX_ENABLE_SANITIZERS)
+    # Sanitizers are disabled - do nothing.
+ELSEIF (TNTCXX_ENABLE_SANITIZERS STREQUAL "address")
     SET(SANITIZER_FLAGS "-fsanitize=address")
     # FIXME(gh-62)
     IF (NOT CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
@@ -112,6 +116,12 @@ IF (TNTCXX_ENABLE_SANITIZERS)
     # for details).
     ADD_COMPILE_OPTIONS(${SANITIZER_FLAGS})
     ADD_LINK_OPTIONS(${SANITIZER_FLAGS})
+ELSEIF (TNTCXX_ENABLE_SANITIZERS STREQUAL "thread")
+    SET(SANITIZER_FLAGS "-fsanitize=thread")
+    ADD_COMPILE_OPTIONS(${SANITIZER_FLAGS})
+    ADD_LINK_OPTIONS(${SANITIZER_FLAGS})
+ELSE ()
+    MESSAGE(FATAL_ERROR "Unknown TNTCXX_ENABLE_SANITIZERS value")
 ENDIF()
 
 # Common function for building tests.
@@ -209,6 +219,11 @@ TNTCXX_TEST(NAME Client.test TYPE ctest
             LIBRARIES ${COMMON_LIB}
 )
 
+TNTCXX_TEST(NAME ClientMultithread.test TYPE ctest
+            SOURCES src/Client/Connector.hpp test/ClientMultithreadTest.cpp
+            LIBRARIES ${COMMON_LIB} Threads::Threads
+)
+
 IF (TNTCXX_ENABLE_SSL)
     TNTCXX_TEST(NAME ClientSSL.test TYPE ctest
                SOURCES src/Client/Connector.hpp test/ClientTest.cpp

README.md

@@ -1,7 +1,7 @@
 # tntcxx — Tarantool C++ Connector
 
-This repository contains the tntcxx Tarantool C++ connector code. tntcxx is an 
-open-source Tarantool C++ connector (compliant to C++17) designed with high 
+This repository contains the tntcxx Tarantool C++ connector code. tntcxx is an
+open-source Tarantool C++ connector (compliant to C++17) designed with high
 efficiency in mind.
 
 ## Building tntcxx
@@ -25,21 +25,21 @@ subdirectory of your project or as an embedded dependency.
 
 1. Make tntcxx's source code available to the main build. This can be done a few
 different ways:
-    * Download the tntcxx source code manually and place it at a known location. 
+    * Download the tntcxx source code manually and place it at a known location.
     This is the least flexible approach and can make it more difficult to use
     with continuous integration systems, etc.
     * Embed the tntcxx source code as a direct copy in the main project's source
-    tree. This is often the simplest approach, but is also the hardest to keep 
+    tree. This is often the simplest approach, but is also the hardest to keep
     up to date. Some organizations may not permit this method.
     * Add tntcxx as a [git submodule](https://git-scm.com/docs/git-submodule) or
     equivalent. This may not always be possible or appropriate. Git submodules,
     for example, have their own set of advantages and drawbacks.
     * Use the CMake [`FetchContent`](https://cmake.org/cmake/help/latest/module/FetchContent.html)
-    commands to download tntcxx as part of the build's configure step. This 
+    commands to download tntcxx as part of the build's configure step. This
     approach doesn't have the limitations of the other methods.
 
-The last of the above methods is implemented with a small piece of CMake code 
-that downloads and pulls the tntcxx code into the main build. Just add the 
+The last of the above methods is implemented with a small piece of CMake code
+that downloads and pulls the tntcxx code into the main build. Just add the
 following snippet to your CMakeLists.txt:
 ```cmake
 include(FetchContent)
@@ -50,7 +50,7 @@ FetchContent_Declare(
 FetchContent_MakeAvailable(tntcxx)
 ```
 
-After obtaining tntcxx sources using the rest of the methods, you can use the 
+After obtaining tntcxx sources using the rest of the methods, you can use the
 following CMake command to incorporate tntcxx into your CMake project:
 ```cmake
 add_subdirectory(${TNTCXX_SOURCE_DIR})
@@ -64,8 +64,8 @@ target_link_libraries(example tntcxx::tntcxx)
 
 ##### Running tntcxx Tests with CMake
 
-Use the `-DTNTCXX_BUILD_TESTING=ON` option to run the tntcxx tests. This option 
-is enabled by default if the tntcxx project is determined to be the top level 
+Use the `-DTNTCXX_BUILD_TESTING=ON` option to run the tntcxx tests. This option
+is enabled by default if the tntcxx project is determined to be the top level
 project. Note that `BUILD_TESTING` must also be on (the default).
 
 For example, to run the tntcxx tests, you could use this script:
@@ -80,8 +80,8 @@ ctest
 
 ### CMake Option Synopsis
 
-- `-DTNTCXX_BUILD_TESTING=ON` must be set to enable testing. This option is 
-enabled by default if the tntcxx project is determined to be the top level 
+- `-DTNTCXX_BUILD_TESTING=ON` must be set to enable testing. This option is
+enabled by default if the tntcxx project is determined to be the top level
 project.
 
 ## Internals
@@ -107,7 +107,7 @@ as template parameter of buffer.
 Connector can be embedded in any C++ application with including main header:
 `#include "<path-to-cloned-repo>/src/Client/Connector.hpp"`
 
-### Objects instantiation
+### Objects Instantiation
 
 To create client one should specify buffer's and network provider's implementations
 as template parameters. Connector's main class has the following signature:
@@ -139,12 +139,12 @@ Connection<Buf_t, Net_t> conn(client);
 
 Now assume Tarantool instance is listening `3301` port on localhost. To connect
 to the server we should invoke `Connector::connect()` method of client object and
-pass three arguments: connection instance, address and port.  
+pass three arguments: connection instance, address and port.
 ```c++
 int rc = client.connect(conn, address, port);
 ```
 
-### Error handling
+### Error Handling
 
 Implementation of connector is exception
 free, so we rely on return codes: in case of fail, `connect()` will return `rc < 0`.
@@ -159,18 +159,18 @@ if (rc != 0) {
 To reset connection after errors (clean up error message and connection status),
 one can use `Connection::reset()`.
 
-### Preparing requests
+### Preparing Requests
 
 To execute simplest request (i.e. ping), one can invoke corresponding method of
 connection object:
 ```c++
 rid_t ping = conn.ping();
-```  
+```
 Each request method returns request id, which is sort of future. It can be used
 to get the result of request execution once it is ready (i.e. response). Requests
 are queued in the input buffer of connection until `Connector::wait()` is called.
 
-### Sending requests
+### Sending Requests
 
 That said, to send requests to the server side, we should invoke `client.wait()`:
 ```c++
@@ -182,7 +182,7 @@ request is ready, `wait()` terminates. It also provides negative return code in
 case of system related fails (e.g. broken or time outed connection). If `wait()`
 returns 0, then response is received and expected to be parsed.
 
-### Receiving responses
+### Receiving Responses
 
 To get the response when it is ready, we can use `Connection::getResponse()`.
 It takes request id and returns optional object containing response (`nullptr`
@@ -200,11 +200,11 @@ either runtime error(s) (accessible by `response.body.error_stack`) or data
 tuples are not decoded and come in form of pointers to the start and end of
 msgpacks. See section below to understand how to decode tuples.
 
-### Data manipulation
+### Data Manipulation
 
 Now let's consider a bit more sophisticated requests.
 Assume we have space with `id = 512` and following format on the server:
-`CREATE TABLE t(id INT PRIMARY KEY, a TEXT, b DOUBLE);`  
+`CREATE TABLE t(id INT PRIMARY KEY, a TEXT, b DOUBLE);`
 Preparing analogue of `t:replace(1, "111", 1.01);` request can be done this way:
 
 ```c++
@@ -218,7 +218,7 @@ auto i = conn.space[512].index[1];
 rid_t select = i.select(std::make_tuple(1), 1, 0 /*offset*/, IteratorType::EQ);
 ```
 
-### Data readers
+### Data Readers
 
 Responses from server contain raw data (i.e. encoded into MsgPack tuples).
 Let's define structure describing data stored in space `t`:
@@ -263,3 +263,9 @@ an array of tuples as value in response to `select`. So, in order to
 successfully decode them, we should pass an array of tuples to decoder - that's
 why `std::vector<UserTuple>` is needed. If decoding was successful, `results`
 will contain all decoded `UserTuples`.
+
+## Multi-Threading Model
+
+A `Connector` object and all its instances of `Connection` objects must be used in a single thread. For multi-threaded usage, create one or several `Connector` instances for each thread. Each `Connection` object must be used only with the `Connector` object that it was created from.
+
+If custom `Buffer` or `NetProvider` implementations are used for `Connector` objects, the custom implementations must not share any state (e.g., `static` fields).

src/Buffer/Buffer.hpp

@@ -37,6 +37,7 @@
 #include <cstring>
 #include <string>
 #include <type_traits>
+#include <utility>
 
 #include "../Utils/Mempool.hpp"
 #include "../Utils/List.hpp"
@@ -58,9 +59,8 @@ namespace tnt {
  * REAL_SIZE - constant determines real size of allocated chunk (excluding
  * overhead taken by allocator).
  */
-template <size_t N = 16 * 1024, class allocator = MempoolHolder<N>>
-class Buffer
-{
+template <size_t N = 16 * 1024, class allocator = MempoolInstance<N, 16>>
+class Buffer {
 private:
 	/** =============== Block definition =============== */
 	/** Blocks are organized into linked list. */
@@ -115,6 +115,12 @@ class Buffer
 	 * to the next byte after the last valid byte in block.
 	 * */
 	bool isEndOfBlock(const char *ptr);
+	/** Delete blocks and release occupied memory. */
+	void releaseBlocks(void)
+	{
+		while (!m_blocks.isEmpty())
+			delBlock(&m_blocks.first());
+	}
 
 public:
 	/** =============== Convenient wrappers =============== */
@@ -265,11 +271,35 @@ class Buffer
 
 	/** =============== Buffer definition =============== */
 	/** Copy of any kind is disabled. Move is allowed. */
-	Buffer(const allocator& all = allocator());
+	Buffer(allocator &&all = allocator());
 	Buffer(const Buffer& buf) = delete;
 	Buffer& operator = (const Buffer& buf) = delete;
-	Buffer(Buffer &&buf) noexcept = default;
-	Buffer &operator=(Buffer &&buf) noexcept = default;
+	Buffer(Buffer &&other) noexcept
+	{
+		/* Call move assignment operator. */
+		*this = std::forward<Buffer>(other);
+	}
+	Buffer &operator=(Buffer &&other) noexcept
+	{
+		if (this == &other)
+			return *this;
+
+		m_blocks = std::move(other.m_blocks);
+		/*
+		 * Release blocks of `other` right on the move because
+		 * we are going to move its allocator as well and we
+		 * must not use it after it is moved.
+		 */
+		other.releaseBlocks();
+		m_all = std::move(other.m_all);
+
+		m_iterators = std::move(other.m_iterators);
+		m_begin = other.m_begin;
+		other.m_begin = nullptr;
+		m_end = other.m_end;
+		other.m_end = nullptr;
+		return *this;
+	}
 	~Buffer() noexcept;
 
 	/**
@@ -648,7 +678,7 @@ Buffer<N, allocator>::iterator_common<LIGHT>::moveBackward(size_t step)
 }
 
 template <size_t N, class allocator>
-Buffer<N, allocator>::Buffer(const allocator &all) : m_all(all)
+Buffer<N, allocator>::Buffer(allocator &&all) : m_all(std::forward<allocator>(all))
 {
 	static_assert((N & (N - 1)) == 0, "N must be power of 2");
 	static_assert(allocator::REAL_SIZE % alignof(Block) == 0,
@@ -666,9 +696,7 @@ Buffer<N, allocator>::Buffer(const allocator &all) : m_all(all)
 template <size_t N, class allocator>
 Buffer<N, allocator>::~Buffer() noexcept
 {
-	/* Delete blocks and release occupied memory. */
-	while (!m_blocks.isEmpty())
-		delBlock(&m_blocks.first());
+	releaseBlocks();
 }
 
 template <size_t N, class allocator>