Merge branch 'ydb-platform:main' into add-cpm

Author: Repin-Daniil

.devcontainer/Dockerfile

@@ -7,7 +7,7 @@ RUN apt-get -y update && apt-get -y install software-properties-common && add-ap
 RUN apt-get -y update && apt-get -y install \
     git gdb ninja-build libidn11-dev ragel yasm libc-ares-dev libre2-dev \
     rapidjson-dev zlib1g-dev libxxhash-dev libzstd-dev libsnappy-dev libgtest-dev libgmock-dev \
-    libbz2-dev libdouble-conversion-dev libstdc++-13-dev gcc-13 g++-13 liblz4-dev libssl-dev \
+    libbz2-dev liblz4-dev libdouble-conversion-dev libssl-dev libstdc++-13-dev gcc-13 g++-13
     && apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # Install CMake
@@ -35,32 +35,39 @@ RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 10000 && \
 
 # Install abseil-cpp
 ENV ABSEIL_CPP_VERSION=20230802.0
+ENV ABSEIL_CPP_INSTALL_DIR=~/ydb_deps/abseil-cpp
 RUN wget -O abseil-cpp-${ABSEIL_CPP_VERSION}.tar.gz https://github.com/abseil/abseil-cpp/archive/refs/tags/${ABSEIL_CPP_VERSION}.tar.gz && \
     tar -xvzf abseil-cpp-${ABSEIL_CPP_VERSION}.tar.gz && cd abseil-cpp-${ABSEIL_CPP_VERSION} && \
     mkdir build && cd build && \
     cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DABSL_PROPAGATE_CXX_STD=ON .. && \
-    cmake --build . --config Release --target install
+    cmake --build . --config Release && \
+    cmake --install . --config Release --prefix ${ABSEIL_CPP_INSTALL_DIR}
 
 # Install protobuf
 ENV PROTOBUF_VERSION=3.21.12
+ENV PROTOBUF_INSTALL_DIR=~/ydb_deps/protobuf
 RUN wget -O protobuf-${PROTOBUF_VERSION}.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v${PROTOBUF_VERSION}.tar.gz && \
     tar -xvzf protobuf-${PROTOBUF_VERSION}.tar.gz && cd protobuf-${PROTOBUF_VERSION} && \
     mkdir build && cd build && \
     cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. && \
-    cmake --build . --config Release --target install
+    cmake --build . --config Release && \
+    cmake --install . --config Release --prefix ${PROTOBUF_INSTALL_DIR}
 
 # Install grpc
-ENV GRPC_VERSION=1.41.0
+ENV GRPC_VERSION=1.54.3
+ENV GRPC_INSTALL_DIR=~/ydb_deps/grpc
 RUN wget -O grpc-${GRPC_VERSION}.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v${GRPC_VERSION}.tar.gz && \
     tar -xvzf grpc-${GRPC_VERSION}.tar.gz && cd grpc-${GRPC_VERSION} && \
     mkdir build && cd build && \
-    cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \
+    cmake -G Ninja -DCMAKE_PREFIX_PATH="${ABSEIL_CPP_INSTALL_DIR};${PROTOBUF_INSTALL_DIR}" \
+    -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \
     -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_BUILD_CSHARP_EXT=OFF \
     -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_RE2_PROVIDER=package \
     -DgRPC_SSL_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ABSL_PROVIDER=package \
     -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \
     -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF .. && \
-    cmake --build . --config Release --target install
+    cmake --build . --config Release && \
+    cmake --install . --config Release --prefix ${GRPC_INSTALL_DIR}
 
 # Install base64
 ENV BASE64_VERSION=0.5.2

.github/actions/prepare_vm/action.yaml

@@ -11,9 +11,19 @@ runs:
       run: |
         sudo add-apt-repository ppa:ubuntu-toolchain-r/test
         sudo apt-get -y update
-        sudo apt-get -y install git ninja-build libidn11-dev ragel yasm libc-ares-dev libre2-dev \
+        sudo apt-get -y install git gdb ninja-build libidn11-dev ragel yasm libc-ares-dev libre2-dev \
           rapidjson-dev zlib1g-dev libxxhash-dev libzstd-dev libsnappy-dev libgtest-dev libgmock-dev \
-          libbz2-dev liblz4-dev libdouble-conversion-dev libstdc++-13-dev gcc-13 g++-13
+          libbz2-dev liblz4-dev libdouble-conversion-dev libssl-dev libstdc++-13-dev gcc-13 g++-13
+
+        # Install ccache
+        (V=4.8.1; curl -L https://github.com/ccache/ccache/releases/download/v${V}/ccache-${V}-linux-x86_64.tar.xz | \
+        sudo tar -xJ -C /usr/local/bin/ --strip-components=1 --no-same-owner ccache-${V}-linux-x86_64/ccache)
+
+        ccache -o base_dir=`realpath ..`
+        ccache -o cache_dir=~/.ccache
+        ccache -z
+        export ENABLE_CCACHE="-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache"
+
         wget https://apt.llvm.org/llvm.sh
         chmod u+x llvm.sh
         sudo ./llvm.sh 16
@@ -28,59 +38,64 @@ runs:
         tar -xvzf abseil-cpp-20230802.0.tar.gz
         cd abseil-cpp-20230802.0
         mkdir build && cd build
-        cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DABSL_PROPAGATE_CXX_STD=ON ..
-        sudo cmake --build . --config Release --target install
+        cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_BUILD_TYPE=Release -DABSL_PROPAGATE_CXX_STD=ON ..
+        cmake --build . --config Release
+        cmake --install . --config Release --prefix ~/ydb_deps/absl
         cd ../../
 
         # Install protobuf
         wget -O protobuf-3.21.12.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz
         tar -xvzf protobuf-3.21.12.tar.gz
         cd protobuf-3.21.12
         mkdir build && cd build
-        cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package ..
-        sudo cmake --build . --config Release --target install
+        cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON ..
+        cmake --build . --config Release
+        cmake --install . --config Release --prefix ~/ydb_deps/protobuf
         cd ../../
 
         # Install gRPC
-        wget -O grpc-1.41.0.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v1.41.0.tar.gz
-        tar -xvzf grpc-1.41.0.tar.gz && cd grpc-1.41.0
+        wget -O grpc-1.54.3.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v1.54.3.tar.gz
+        tar -xvzf grpc-1.54.3.tar.gz && cd grpc-1.54.3
         mkdir build && cd build
-        cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \
+        cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_PREFIX_PATH="~/ydb_deps/absl;~/ydb_deps/protobuf" -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \
           -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_BUILD_CSHARP_EXT=OFF \
           -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_RE2_PROVIDER=package \
           -DgRPC_SSL_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ABSL_PROVIDER=package \
           -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \
           -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF ..
-        sudo cmake --build . --config Release --target install
+        cmake --build . --config Release
+        cmake --install . --config Release --prefix ~/ydb_deps/grpc
         cd ../../
 
         # Install base64
         wget -O base64-0.5.2.tar.gz https://github.com/aklomp/base64/archive/refs/tags/v0.5.2.tar.gz
         tar -xvzf base64-0.5.2.tar.gz && cd base64-0.5.2
         mkdir build && cd build
-        cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ..
-        sudo cmake --build . --config Release --target install
+        cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_BUILD_TYPE=Release ..
+        cmake --build . --config Release
+        cmake --install . --config Release --prefix ~/ydb_deps/base64
         cd ../../
 
         # Install brotli
         wget -O brotli-1.1.0.tar.gz https://github.com/google/brotli/archive/refs/tags/v1.1.0.tar.gz
         tar -xvzf brotli-1.1.0.tar.gz && cd brotli-1.1.0
         mkdir build && cd build
-        cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ..
-        sudo cmake --build . --config Release --target install
+        cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_BUILD_TYPE=Release ..
+        cmake --build . --config Release
+        cmake --install . --config Release --prefix ~/ydb_deps/brotli
         cd ../../
 
         # Install jwt-cpp
         wget -O jwt-cpp-0.7.0.tar.gz https://github.com/Thalhammer/jwt-cpp/archive/refs/tags/v0.7.0.tar.gz
         tar -xvzf jwt-cpp-0.7.0.tar.gz && cd jwt-cpp-0.7.0
         mkdir build && cd build
-        cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ..
-        sudo cmake --build . --config Release --target install
+        cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_BUILD_TYPE=Release ..
+        cmake --build . --config Release
+        cmake --install . --config Release --prefix ~/ydb_deps/jwt-cpp
         cd ../../
 
-        # Install ccache
-        (V=4.8.1; curl -L https://github.com/ccache/ccache/releases/download/v${V}/ccache-${V}-linux-x86_64.tar.xz | \
-        sudo tar -xJ -C /usr/local/bin/ --strip-components=1 --no-same-owner ccache-${V}-linux-x86_64/ccache)
-        sudo rm -rf llvm.sh abseil-cpp-20230802.0.tar.gz protobuf-3.21.12.tar.gz grpc-1.41.0.tar.gz \
+        # Clean up
+        ccache -s
+        sudo rm -rf llvm.sh abseil-cpp-20230802.0.tar.gz protobuf-3.21.12.tar.gz grpc-1.54.3.tar.gz \
           base64-0.5.2.tar.gz brotli-1.1.0.tar.gz jwt-cpp-0.7.0.tar.gz abseil-cpp-20230802.0 \
-          protobuf-3.21.12 grpc-1.41.0 base64-0.5.2 brotli-1.1.0 jwt-cpp-0.7.0
+          protobuf-3.21.12 grpc-1.54.3 base64-0.5.2 brotli-1.1.0 jwt-cpp-0.7.0

.github/import_generation.txt

@@ -1 +1 @@
-25
+28

.github/last_commit.txt

@@ -1 +1 @@
-b39a359f04c57875ea162524646ae879f06c7a33
+42606baa568dd1f0408b0651dce31577f28bcfe1

.github/workflows/examples.yaml

@@ -46,8 +46,6 @@ jobs:
       if: github.event.pull_request.head.sha == ''
       with:
         submodules: true
-    - name: Install dependencies
-      uses: ./.github/actions/prepare_vm
     - name: Prepare ccache timestamp
       id: ccache_cache_timestamp
       shell: cmake -P {0}
@@ -61,6 +59,8 @@ jobs:
         key: ubuntu-22.04-ccache-${{ matrix.compiler }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }}
         restore-keys: |
           ubuntu-22.04-ccache-${{ matrix.compiler }}-
+    - name: Install dependencies
+      uses: ./.github/actions/prepare_vm
     - name: Build
       uses: ./.github/actions/build
       with:
@@ -70,6 +70,9 @@ jobs:
       run: |
         cd build
         examples/basic_example/basic_example -e localhost:2136 -d /local
+        if [ ${{ matrix.ydb-version }} == "trunk" ]; then
+          examples/basic_example/basic_example -e localhost:2135 -d /local -c /tmp/ydb_certs/ca.pem
+        fi
         examples/bulk_upsert_simple/bulk_upsert_simple -e localhost:2136 -d /local -p /local/bulk
         examples/pagination/pagination -e localhost:2136 -d /local -p /local/pagination
         examples/ttl/ttl -e localhost:2136 -d /local -p /local/ttl

.github/workflows/tests.yaml

@@ -35,8 +35,6 @@ jobs:
       if: github.event.pull_request.head.sha == ''
       with:
         submodules: true
-    - name: Install dependencies
-      uses: ./.github/actions/prepare_vm
     - name: Prepare ccache timestamp
       id: ccache_cache_timestamp
       shell: cmake -P {0}
@@ -50,6 +48,8 @@ jobs:
         key: ubuntu-22.04-ccache-${{ matrix.compiler }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }}
         restore-keys: |
           ubuntu-22.04-ccache-${{ matrix.compiler }}-
+    - name: Install dependencies
+      uses: ./.github/actions/prepare_vm
     - name: Build
       uses: ./.github/actions/build
       with:
@@ -95,8 +95,6 @@ jobs:
       if: github.event.pull_request.head.sha == ''
       with:
         submodules: true
-    - name: Install dependencies
-      uses: ./.github/actions/prepare_vm
     - name: Prepare ccache timestamp
       id: ccache_cache_timestamp
       shell: cmake -P {0}
@@ -110,6 +108,8 @@ jobs:
         key: ubuntu-22.04-ccache-${{ matrix.compiler }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }}
         restore-keys: |
           ubuntu-22.04-ccache-${{ matrix.compiler }}-
+    - name: Install dependencies
+      uses: ./.github/actions/prepare_vm
     - name: Build
       uses: ./.github/actions/build
       with:

.github/workflows/warmup_cache.yaml

@@ -17,6 +17,13 @@ jobs:
       uses: actions/checkout@v4
       with:
         submodules: true
+    - name: Restore cache files
+      uses: actions/cache/restore@v4
+      with:
+        path: ~/.ccache
+        key: ubuntu-22.04-ccache-${{ matrix.compiler }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }}
+        restore-keys: |
+          ubuntu-22.04-ccache-${{ matrix.compiler }}-
     - name: Install dependencies
       uses: ./.github/actions/prepare_vm
     - name: Prepare ccache timestamp

CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.5)
 file(READ "src/version.h" YDB_SDK_VERSION_FILE_RAW)
 string(REGEX MATCH "YDB_SDK_VERSION = \"([0-9]+\\.[0-9]+\\.[0-9]+)\"" _ ${YDB_SDK_VERSION_FILE_RAW})
 set(YDB_SDK_VERSION ${CMAKE_MATCH_1})
-message(STATUS "YDB С++ SDK version: ${YDB_SDK_VERSION}")
+message(STATUS "YDB C++ SDK version: ${YDB_SDK_VERSION}")
 
 project(YDB-CPP-SDK VERSION ${YDB_SDK_VERSION} LANGUAGES C CXX ASM)
 
@@ -106,4 +106,6 @@ if (YDB_SDK_INSTALL)
     ${YDB_SDK_SOURCE_DIR}/cmake/FindZSTD.cmake
     DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ydb-cpp-sdk/Modules
   )
+
+  include(cmake/PackSDK.cmake)
 endif()

CMakePresets.json

@@ -11,7 +11,7 @@
       "displayName": "Generic Config",
       "cacheVariables": {
         "CMAKE_EXPORT_COMPILE_COMMANDS": "ON",
-        "CMAKE_PREFIX_PATH": "~/install"
+        "CMAKE_PREFIX_PATH": "~/ydb_deps/absl;~/ydb_deps/protobuf;~/ydb_deps/grpc;~/ydb_deps/base64;~/ydb_deps/brotli;~/ydb_deps/jwt-cpp"
       },
       "generator": "Ninja",
       "binaryDir": "${sourceDir}/build"

README.md

@@ -44,38 +44,74 @@
 
 ```bash
 sudo apt-get -y update
-sudo apt-get -y install git cmake ninja-build libidn11-dev ragel yasm protobuf-compiler \
-  protobuf-compiler-grpc libprotobuf-dev libgrpc++-dev libgrpc-dev libgrpc++1 libgrpc10 \
-  rapidjson-dev zlib1g-dev libxxhash-dev libzstd-dev libsnappy-dev liblz4-dev \
-  libgtest-dev libgmock-dev libbz2-dev libdouble-conversion-dev libssl-dev
+sudo apt-get -y install git gdb ninja-build libidn11-dev ragel yasm libc-ares-dev libre2-dev \
+  rapidjson-dev zlib1g-dev libxxhash-dev libzstd-dev libsnappy-dev libgtest-dev libgmock-dev \
+  libbz2-dev liblz4-dev libdouble-conversion-dev libssl-dev libstdc++-13-dev gcc-13 g++-13
 
 wget https://apt.llvm.org/llvm.sh
 chmod u+x llvm.sh
 sudo ./llvm.sh 16
 
-wget https://ftp.gnu.org/pub/gnu/libiconv/libiconv-1.15.tar.gz
-tar -xvzf libiconv-1.15.tar.gz && cd libiconv-1.15
-./configure --prefix=/usr/local
-make
-sudo make install
-
+# Install abseil-cpp
+wget -O abseil-cpp-20230802.0.tar.gz https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.0.tar.gz
+tar -xvzf abseil-cpp-20230802.0.tar.gz
+cd abseil-cpp-20230802.0
+mkdir build && cd build
+cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DABSL_PROPAGATE_CXX_STD=ON ..
+cmake --build . --config Release
+cmake --install . --config Release --prefix ~/ydb_deps/absl
+cd ../../
+
+# Install protobuf
+wget -O protobuf-3.21.12.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz
+tar -xvzf protobuf-3.21.12.tar.gz
+cd protobuf-3.21.12
+mkdir build && cd build
+cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON ..
+cmake --build . --config Release
+cmake --install . --config Release --prefix ~/ydb_deps/protobuf
+cd ../../
+
+# Install gRPC
+wget -O grpc-1.54.3.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v1.54.3.tar.gz
+tar -xvzf grpc-1.54.3.tar.gz && cd grpc-1.54.3
+mkdir build && cd build
+cmake -G Ninja -DCMAKE_PREFIX_PATH="~/ydb_deps/absl;~/ydb_deps/protobuf" -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \
+  -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_BUILD_CSHARP_EXT=OFF \
+  -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_RE2_PROVIDER=package \
+  -DgRPC_SSL_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ABSL_PROVIDER=package \
+  -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \
+  -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF ..
+cmake --build . --config Release
+cmake --install . --config Release --prefix ~/ydb_deps/grpc
+cd ../../
+
+# Install base64
 wget -O base64-0.5.2.tar.gz https://github.com/aklomp/base64/archive/refs/tags/v0.5.2.tar.gz
 tar -xvzf base64-0.5.2.tar.gz && cd base64-0.5.2
 mkdir build && cd build
-cmake -DCMAKE_BUILD_TYPE=Release ..
-sudo cmake --build . --config Release --target install
+cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ..
+cmake --build . --config Release
+cmake --install . --config Release --prefix ~/ydb_deps/base64
+cd ../../
 
+# Install brotli
 wget -O brotli-1.1.0.tar.gz https://github.com/google/brotli/archive/refs/tags/v1.1.0.tar.gz
 tar -xvzf brotli-1.1.0.tar.gz && cd brotli-1.1.0
 mkdir build && cd build
-cmake -DCMAKE_BUILD_TYPE=Release ..
-sudo cmake --build . --config Release --target install
+cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ..
+cmake --build . --config Release
+cmake --install . --config Release --prefix ~/ydb_deps/brotli
+cd ../../
 
+# Install jwt-cpp
 wget -O jwt-cpp-0.7.0.tar.gz https://github.com/Thalhammer/jwt-cpp/archive/refs/tags/v0.7.0.tar.gz
 tar -xvzf jwt-cpp-0.7.0.tar.gz && cd jwt-cpp-0.7.0
 mkdir build && cd build
-cmake -DCMAKE_BUILD_TYPE=Release ..
-sudo cmake --build . --config Release --target install
+cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ..
+cmake --build . --config Release
+cmake --install . --config Release --prefix ~/ydb_deps/jwt-cpp
+cd ../../
 ```
 
 ### Create the work directory