Add dht utils for troubleshooting complex deployments (#1685)

* not always gh mac runners have nproc utility

* deprecate Ubuntu 20.04

* fix linking issues of portable tonlibjson.dylib

* add dht-ping-servers and dht-resolve utilities to artifacts

* test gh builds

* minor win fix

* minor win fix

Author: neodix42

Dockerfile

@@ -29,7 +29,7 @@ RUN mkdir build && \
         cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DTON_USE_JEMALLOC=ON .. && \
         ninja storage-daemon storage-daemon-cli tonlibjson fift func validator-engine validator-engine-console \
     generate-random-id dht-server lite-client tolk rldp-http-proxy dht-server proxy-liteserver create-state \
-    blockchain-explorer emulator tonlibjson http-proxy adnl-proxy
+    blockchain-explorer emulator tonlibjson http-proxy adnl-proxy dht-ping-servers dht-resolve
 
 FROM ubuntu:22.04
 ARG DEBIAN_FRONTEND=noninteractive
@@ -50,6 +50,8 @@ COPY --from=builder /ton/build/blockchain-explorer/blockchain-explorer /usr/loca
 COPY --from=builder /ton/build/crypto/create-state /usr/local/bin/
 COPY --from=builder /ton/build/utils/proxy-liteserver /usr/local/bin/
 COPY --from=builder /ton/build/dht-server/dht-server /usr/local/bin/
+COPY --from=builder /ton/build/dht/dht-ping-servers /usr/local/bin/
+COPY --from=builder /ton/build/dht/dht-resolve /usr/local/bin/
 COPY --from=builder /ton/build/rldp-http-proxy/rldp-http-proxy /usr/local/bin/
 COPY --from=builder /ton/build/http/http-proxy  /usr/local/bin/
 COPY --from=builder /ton/build/adnl/adnl-proxy  /usr/local/bin/

assembly/native/build-macos-portable.sh

@@ -150,7 +150,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; }
 if [ "$with_tests" = true ]; then
   ninja storage-daemon storage-daemon-cli blockchain-explorer   \
   tonlib tonlibjson tonlib-cli validator-engine func tolk fift \
-  lite-client validator-engine-console generate-random-id json2tlo dht-server \
+  lite-client validator-engine-console generate-random-id json2tlo dht-server dht-ping-servers dht-resolve \
   http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \
   test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont \
   test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \
@@ -159,7 +159,7 @@ if [ "$with_tests" = true ]; then
 else
   ninja storage-daemon storage-daemon-cli blockchain-explorer   \
   tonlib tonlibjson tonlib-cli validator-engine func tolk fift \
-  lite-client validator-engine-console generate-random-id json2tlo dht-server \
+  lite-client validator-engine-console generate-random-id json2tlo dht-server dht-ping-servers dht-resolve \
   http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver
   test $? -eq 0 || { echo "Can't compile ton"; exit 1; }
 fi
@@ -184,6 +184,8 @@ if [ "$with_artifacts" = true ]; then
   cp build/http/http-proxy artifacts/
   cp build/rldp-http-proxy/rldp-http-proxy artifacts/
   cp build/dht-server/dht-server artifacts/
+  cp build/dht/dht-ping-servers artifacts/
+  cp build/dht/dht-resolve artifacts/
   cp build/lite-client/lite-client artifacts/
   cp build/validator-engine/validator-engine artifacts/
   cp build/utils/generate-random-id artifacts/

assembly/native/build-macos-shared.sh

@@ -61,6 +61,19 @@ else
   echo "Using compiled lz4"
 fi
 
+if [ ! -d "zlib" ]; then
+  git clone https://github.com/madler/zlib.git
+  cd zlib
+  zlibPath=`pwd`
+  ./configure --static
+  make -j4
+  test $? -eq 0 || { echo "Can't compile zlib"; exit 1; }
+  cd ..
+else
+  zlibPath=$(pwd)/zlib
+  echo "Using compiled zlib"
+fi
+
 brew unlink [email protected]
 brew install openssl@3
 brew unlink openssl@3 &&  brew link --overwrite openssl@3
@@ -75,7 +88,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; }
 if [ "$with_tests" = true ]; then
   ninja storage-daemon storage-daemon-cli blockchain-explorer   \
   tonlib tonlibjson tonlib-cli validator-engine func tolk fift \
-  lite-client validator-engine-console generate-random-id json2tlo dht-server \
+  lite-client validator-engine-console generate-random-id json2tlo dht-server dht-ping-servers dht-resolve \
   http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \
   test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont \
   test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \
@@ -84,7 +97,7 @@ if [ "$with_tests" = true ]; then
 else
   ninja storage-daemon storage-daemon-cli blockchain-explorer   \
   tonlib tonlibjson tonlib-cli validator-engine func tolk fift \
-  lite-client validator-engine-console generate-random-id json2tlo dht-server \
+  lite-client validator-engine-console generate-random-id json2tlo dht-server dht-ping-servers dht-resolve \
   http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver
   test $? -eq 0 || { echo "Can't compile ton"; exit 1; }
 fi
@@ -109,6 +122,8 @@ if [ "$with_artifacts" = true ]; then
   cp build/http/http-proxy artifacts/
   cp build/rldp-http-proxy/rldp-http-proxy artifacts/
   cp build/dht-server/dht-server artifacts/
+  cp build/dht/dht-ping-servers artifacts/
+  cp build/dht/dht-resolve artifacts/
   cp build/lite-client/lite-client artifacts/
   cp build/validator-engine/validator-engine artifacts/
   cp build/utils/generate-random-id artifacts/

assembly/native/build-ubuntu-appimages.sh

@@ -61,7 +61,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; }
 if [ "$with_tests" = true ]; then
 ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \
       validator-engine lite-client validator-engine-console blockchain-explorer \
-      generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \
+      generate-random-id json2tlo dht-server http-proxy rldp-http-proxy dht-ping-servers dht-resolve \
       adnl-proxy create-state emulator test-ed25519 test-bigint \
       test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \
       test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \
@@ -71,7 +71,7 @@ else
 ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \
       validator-engine lite-client validator-engine-console blockchain-explorer \
       generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \
-      adnl-proxy create-state emulator proxy-liteserver
+      adnl-proxy create-state emulator proxy-liteserver dht-ping-servers dht-resolve
       test $? -eq 0 || { echo "Can't compile ton"; exit 1; }
 fi
 
@@ -104,6 +104,7 @@ if [ "$with_artifacts" = true ]; then
      build/tonlib/libtonlibjson.so build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy \
      build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine \
      build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.so \
+     build/dht/dht-ping-servers build/dht/dht-resolve \
      artifacts
   test $? -eq 0 || { echo "Can't copy final binaries"; exit 1; }
   cp -R crypto/smartcont artifacts

assembly/native/build-ubuntu-portable.sh

@@ -135,7 +135,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; }
 if [ "$with_tests" = true ]; then
 ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \
       validator-engine lite-client validator-engine-console blockchain-explorer \
-      generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \
+      generate-random-id json2tlo dht-server http-proxy rldp-http-proxy dht-ping-servers dht-resolve \
       adnl-proxy create-state emulator test-ed25519 test-bigint \
       test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \
       test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \
@@ -145,7 +145,7 @@ else
 ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \
       validator-engine lite-client validator-engine-console blockchain-explorer \
       generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \
-      adnl-proxy create-state emulator proxy-liteserver
+      adnl-proxy create-state emulator proxy-liteserver dht-ping-servers dht-resolve
       test $? -eq 0 || { echo "Can't compile ton"; exit 1; }
 fi
 
@@ -167,6 +167,7 @@ if [ "$with_artifacts" = true ]; then
      build/tonlib/libtonlibjson.so build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy \
      build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine \
      build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.so \
+     build/dht/dht-ping-servers build/dht/dht-resolve \
      artifacts
   test $? -eq 0 || { echo "Can't copy final binaries"; exit 1; }
   cp -R crypto/smartcont artifacts

assembly/native/build-ubuntu-shared.sh

@@ -64,7 +64,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; }
 if [ "$with_tests" = true ]; then
 ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \
       validator-engine lite-client validator-engine-console blockchain-explorer \
-      generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \
+      generate-random-id json2tlo dht-server http-proxy rldp-http-proxy dht-ping-servers dht-resolve \
       adnl-proxy create-state emulator test-ed25519 test-bigint \
       test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \
       test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \
@@ -74,7 +74,7 @@ else
 ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \
       validator-engine lite-client validator-engine-console blockchain-explorer \
       generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \
-      adnl-proxy create-state emulator proxy-liteserver
+      adnl-proxy create-state emulator proxy-liteserver dht-ping-servers dht-resolve
       test $? -eq 0 || { echo "Can't compile ton"; exit 1; }
 fi
 
@@ -99,6 +99,7 @@ if [ "$with_artifacts" = true ]; then
      build/tonlib/libtonlibjson.so build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy \
      build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine \
      build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.so \
+     build/dht/dht-ping-servers build/dht/dht-resolve \
      artifacts
   test $? -eq 0 || { echo "Can't copy final binaries"; exit 1; }
   cp -R crypto/smartcont artifacts

assembly/native/build-windows-2019.bat

@@ -149,14 +149,14 @@ tonlib-cli validator-engine lite-client validator-engine-console generate-random
 json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^
 test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net ^
 test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^
-test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver
+test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver dht-ping-servers dht-resolve
 IF %errorlevel% NEQ 0 (
   echo Can't compile TON
   exit /b %errorlevel%
 )
 ) else (
 ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson  ^
-tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^
+tonlib-cli validator-engine lite-client validator-engine-console generate-random-id dht-ping-servers dht-resolve ^
 json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver
 IF %errorlevel% NEQ 0 (
   echo Can't compile TON
@@ -192,6 +192,8 @@ for %%I in (build\storage\storage-daemon\storage-daemon.exe ^
   build\http\http-proxy.exe ^
   build\rldp-http-proxy\rldp-http-proxy.exe ^
   build\dht-server\dht-server.exe ^
+  build\dht\dht-ping-servers.exe ^
+  build\dht\dht-resolve.exe ^
   build\lite-client\lite-client.exe ^
   build\validator-engine\validator-engine.exe ^
   build\utils\generate-random-id.exe ^

assembly/native/build-windows.bat

@@ -149,14 +149,14 @@ tonlib-cli validator-engine lite-client validator-engine-console generate-random
 json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^
 test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net ^
 test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^
-test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver
+test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver dht-ping-servers dht-resolve
 IF %errorlevel% NEQ 0 (
   echo Can't compile TON
   exit /b %errorlevel%
 )
 ) else (
 ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson  ^
-tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^
+tonlib-cli validator-engine lite-client validator-engine-console generate-random-id dht-ping-servers dht-resolve ^
 json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver
 IF %errorlevel% NEQ 0 (
   echo Can't compile TON
@@ -192,6 +192,8 @@ for %%I in (build\storage\storage-daemon\storage-daemon.exe ^
   build\http\http-proxy.exe ^
   build\rldp-http-proxy\rldp-http-proxy.exe ^
   build\dht-server\dht-server.exe ^
+  build\dht\dht-ping-servers.exe ^
+  build\dht\dht-resolve.exe ^
   build\lite-client\lite-client.exe ^
   build\validator-engine\validator-engine.exe ^
   build\utils\generate-random-id.exe ^