Update to version v4.20.0

Author: MadSchemas

.github/workflows/install_grpc.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-git clone --branch v1.44.0 https://github.com/grpc/grpc
+git clone --branch v1.54.3 https://github.com/grpc/grpc
 cd grpc
 git submodule update --init
 mkdir -p cmake/build

.github/workflows/test.yml

@@ -47,15 +47,10 @@ jobs:
   build:
     strategy:
       matrix:
-        os: [ubuntu-22.04, macos-13]
+        os: [ubuntu-22.04, ubuntu-24.04, macos-13]
         include:
-          - os: ubuntu-24.04
-            cc: gcc-12
-            cxx: g++-12
           - os: ubuntu-latest
             sanitizer: ASAN
-            cc: gcc-10
-            cxx: g++-10
           - os: ubuntu-latest
             sanitizer: TSAN
             cc: gcc-12
@@ -66,6 +61,7 @@ jobs:
       OS: ${{matrix.os}}
       CC: ${{matrix.cc}}
       CXX: ${{matrix.cxx}}
+      SANITIZER: ${{matrix.sanitizer}}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -85,7 +81,11 @@ jobs:
           export CPLUS_INCLUDE_PATH=$GITHUB_WORKSPACE/grpc/third_party/abseil-cpp
           mkdir build && cd build
           if [[ $OS == ubuntu-latest ]]; then
-            cmake -DENABLE_GRPC=ON -DWITH_${{matrix.sanitizer}}=On ..
+            if [[ $SANITIZER == 'ASAN' ]]; then
+              cmake -DENABLE_GRPC=ON -DWITH_${{matrix.sanitizer}}=On -DGTEST_SKIP_SHARDING=On ..
+            else
+              cmake -DENABLE_GRPC=ON -DWITH_${{matrix.sanitizer}}=On ..
+            fi
           elif [[ $OS == macos* ]]; then
             cmake -DGH_CI_OSX=ON ..
           else
@@ -119,8 +119,6 @@ jobs:
         include:
           - os: ubuntu-latest
             sanitizer: ASAN
-            cc: gcc-10
-            cxx: g++-10
             test: 'C++'
           - os: ubuntu-latest
             sanitizer: ASAN

bindings/builtin/builtin.go

@@ -255,7 +255,7 @@ func (binding *Builtin) Init(u []url.URL, eh bindings.EventsHandler, options ...
 		caps: C.int64_t(caps.Value),
 	}
 
-	return err2go(C.reindexer_connect_v4(binding.rx, str2c(u[0].Path), opts, str2c(bindings.ReindexerVersion), ccaps))
+	return err2go(C.reindexer_connect_v4(binding.rx, str2c(u[0].Host+u[0].Path), opts, str2c(bindings.ReindexerVersion), ccaps))
 }
 
 func (binding *Builtin) StartWatchOnCtx(ctx context.Context) (CCtxWrapper, error) {

bindings/consts.go

@@ -2,7 +2,7 @@ package bindings
 
 const CInt32Max = int(^uint32(0) >> 1)
 
-const ReindexerVersion = "v4.19.0"
+const ReindexerVersion = "v4.20.0"
 
 // public go consts from type_consts.h and reindexer_ctypes.h
 const (

changelog.md

@@ -1,7 +1,32 @@
+# Version 4.20.0 (04.02.2025)
+## Core
+- [fea] Optimized indexed strings memory layout (each unique indexed string now requires 20-36 bytes less memery, depending on platform)
+- [fea] Optimized non-built btree-index selection
+- [fea] Optimized selections with large documents count in results
+- [fea] Reduced allocations count in inserts/upserts and select-queries
+- [fea] Changed index compatibility check in `AddIndex`: now `dense` and `regular` indexes are treated compatible
+- [fix] Fixed parsing for [EQUAL_POSITION](readme.md#search-in-array-fields-with-matching-array-indexes) with brackets in SQL
+
+## Replication
+- [fea] Improved full sync performance for namespaces with large amount of empty documents
+
+## Reindexer server
+- [fix] Added more TCP-requests format checks
+- [fix] Fixed [server.yml](cpp_src/server/contrib/server.yml) format errors
+
+## Go connector
+- [fea] Added support for absolute path on Windows platforms in `builtin`-binding
+
+## Build
+- [fix] Fixed build for ARM docker image
+
+## Ported
+- [fea/fix] Ported all the fixes and features from [v3.31.0](https://github.com/Restream/reindexer/releases/tag/v3.31.0)
+
 # Version 4.19.0 (17.12.2024)
 
 ## Core
-- [fea] Added automatic masking for reindexer user's credentials in log files and cluster/sharding JSON's (except for `async_replication` configs in `#config`-namespace)
+- [fea] Added automatic masking for reindexer user's credentials in log files and cluster/sharding JSON's (except for `async_replication`-config in `#config`-namespace)
 - [fix] Fixed assertion on attempt to use 'null'-values with `=`, `IN()`, `<`, `>`, `<=`, `>=` and `RANGE()` operators
 
 ## Replication

cjson/decoder.go

@@ -36,8 +36,7 @@ func fieldByTag(t reflect.Type, tag string) (result reflect.StructField, ok bool
 	}
 	for i := 0; i < t.NumField(); i++ {
 		result = t.Field(i)
-		if ftag := result.Tag.Get("json"); len(ftag) > 0 {
-			ftag, _ = splitStr(ftag, ',')
+		if ftag, _ := splitStr(result.Tag.Get("json"), ','); len(ftag) > 0 {
 			if tag == ftag {
 				return result, true
 			}

clang-tidy/run-clang-tidy-18.py clang-tidy/run-clang-tidy-19.pyclang-tidy/run-clang-tidy-18.py renamed to clang-tidy/run-clang-tidy-19.py

@@ -251,10 +251,10 @@ def main():
                       action='store_true', help='allow alpha checkers from '
                                                 'clang-analyzer.')
   parser.add_argument('-clang-tidy-binary', metavar='PATH',
-                      default='clang-tidy-18',
+                      default='clang-tidy-19',
                       help='path to clang-tidy binary')
   parser.add_argument('-clang-apply-replacements-binary', metavar='PATH',
-                      default='clang-apply-replacements-18',
+                      default='clang-apply-replacements-19',
                       help='path to clang-apply-replacements binary')
   parser.add_argument('-checks', default=None,
                       help='checks filter, when not specified, use clang-tidy '

cpp_src/CMakeLists.txt

@@ -50,7 +50,7 @@ else()
   option(LINK_RESOURCES "Link web resources as binary data" ON)
 endif()
 
-set (REINDEXER_VERSION_DEFAULT "4.19.0")
+set (REINDEXER_VERSION_DEFAULT "4.20.0")
 
 if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE "RelWithDebInfo")
@@ -72,6 +72,13 @@ if(MSVC)
   set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O2 -Zi")
   set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG -Zi")
   set(CMAKE_C_FLAGS_RELEASE "-O2 -DNDEBUG -Zi")
+elseif(WITH_ASAN)
+  # Using O2 instead of O3 to build a bit faster.
+  # Also this allows to avoid SEGFAULT in libasan.so during coroutines interaction on CentOS7 (gcc-12).
+  set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g1")
+  set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O2 -g1")
+  set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
+  set(CMAKE_C_FLAGS_RELEASE "-O2 -DNDEBUG")
 else()
   set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g1")
   set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O3 -g1")

cpp_src/client/itemimplbase.h

@@ -1,6 +1,5 @@
 #pragma once
 
-#include <deque>
 #include <vector>
 #include "core/keyvalue/variant.h"
 #include "core/payload/payloadiface.h"
@@ -76,7 +75,7 @@ class ItemImplBase {
 
 	std::vector<std::string> precepts_;
 	bool unsafe_ = false;
-	std::deque<std::string> holder_;
+	h_vector<key_string, 16> holder_;
 	std::vector<std::unique_ptr<char[]>> largeJSONStrings_;
 };
 

cpp_src/client/queryresults.h

@@ -37,9 +37,8 @@ class QueryResults {
 
 	class Iterator : public CoroQueryResults::Iterator {
 	public:
-		Iterator(const QueryResults* r, const CoroQueryResults* qr, int idx, int pos, int nextPos,
-				 ResultSerializer::ItemParams itemParams) noexcept
-			: CoroQueryResults::Iterator{qr, idx, pos, nextPos, itemParams, {}}, r_(r) {}
+		Iterator(const QueryResults& r, int idx, int pos, int nextPos, ResultSerializer::ItemParams itemParams) noexcept
+			: CoroQueryResults::Iterator{&r.results_, idx, pos, nextPos, itemParams, {}}, r_(&r) {}
 		Iterator& operator*() { return *this; }
 		Iterator& operator++() noexcept {
 			try {
@@ -57,11 +56,12 @@ class QueryResults {
 			}
 			return *this;
 		}
+
 		const QueryResults* r_;
 	};
 
-	Iterator begin() const noexcept { return Iterator{this, &results_, 0, 0, 0, {}}; }
-	Iterator end() const noexcept { return Iterator{this, &results_, int(results_.Count()), 0, 0, {}}; }
+	Iterator begin() const noexcept { return Iterator{*this, 0, 0, 0, {}}; }
+	Iterator end() const noexcept { return Iterator{*this, int(results_.Count()), 0, 0, {}}; }
 
 	size_t Count() const noexcept { return results_.Count(); }
 	int TotalCount() const noexcept { return results_.TotalCount(); }