Merge bitcoin/bitcoin#30043: net: Replace libnatpmp with built-in PCP+NATPMP implementation

5c7cacf ci: Remove natpmp build option and libnatpmp dependency (laanwj)
7e7ec98 doc: Remove mention of natpmp build options (laanwj)
061c3e3 depends: Drop natpmp and associated option from depends (laanwj)
20a18bf build: Drop libnatpmp from build system (laanwj)
7b04709 qt: Changes for built-in PCP+NAT-PMP (laanwj)
52f8ef6 net: Replace libnatpmp with built-in NATPMP+PCP implementation in mapport (laanwj)
97c9717 net: Add PCP and NATPMP implementation (laanwj)
d72df63 net: Use GetLocalAddresses in Discover (laanwj)
e020304 net: Add netif utility (laanwj)
754e425 crypto: Add missing WriteBE16 function (laanwj)

Pull request description:

  Continues #30005. Closes #17012..

  This PR adds PCP (Port Control Protocol) from [RFC6887](https://datatracker.ietf.org/doc/html/rfc6887).  This adds, in addition to the existing IPv4 port mapping (which now uses PCP, with fallback to NAT-PMP), support for IPv6 pinholing-that is, opening a port on the firewall to make it reachable.

  PCP, like NAT-PMP is a simple UDP-based protocol, and the implementation is self-contained, so this gets rid of lthe libnatpnp dependency without adding a new one. It should otherwise be a drop-in replacement. NAT-PMP fallback is implemented so this will not make router support worse.

  For now it is disabled by default, though in the future (not in this PR) we could consider enable it by default to increase the number of connectable nodes without adding significant attack surface.

  To test:
  ```bash
  bitcoind -regtest -natpmp=1 -debug=net
  ```

  (most of the changes in this PR are, ironically, removing the libnatpmp dependency and associated build system and build docs)

  ## TODO

  - [x] Default gateway discovery on Linux / FreeBSD
  - [x] Default gateway discovery on Windows
  - [x] Default gateway discovery on MacOS
  - [x] Either solve FreeBSD compile issue (probably upstream issue) or remove FreeBSD support

  ## Things to consider for follow-up PRs

  - bitcoin/bitcoin#30043 (comment) avoid unreachable nets (not given to -onlynet=)

  - bitcoin/bitcoin#30043 (comment) could announce an addr:port where we do not listen (no -bind)

  - bitcoin/bitcoin#30043 (comment) could announce the wrong port because it uses GetListenPort()

  - bitcoin/bitcoin#30043 (comment) if we requested one port but another was assigned, then which one to use in the renewal?

  - bitcoin/bitcoin#30043 (comment) Use `GetAdapterAddresses` to discover local addresses for Windows

ACKs for top commit:
  Sjors:
    ACK 5c7cacf
  achow101:
    ACK 5c7cacf
  vasild:
    ACK 5c7cacf

Tree-SHA512: e35b69e56d5f5449a3d48a799f9b7b65107c65eeb3e245c2c1e9d42221e469ca5ead90afae423160601cd664dd553a51c859e04f4492f335b064aae3bf23e3bc

Author: achow101

.github/workflows/ci.yml

@@ -67,12 +67,12 @@ jobs:
           echo "TEST_BASE=$(git rev-list -n$((${{ env.MAX_COUNT }} + 1)) --reverse HEAD $EXCLUDE_MERGE_BASE_ANCESTORS | head -1)" >> "$GITHUB_ENV"
       - run: |
           sudo apt-get update
-          sudo apt-get install clang ccache build-essential cmake pkg-config python3-zmq libevent-dev libboost-dev libsqlite3-dev libdb++-dev systemtap-sdt-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev qtbase5-dev qttools5-dev qttools5-dev-tools qtwayland5 libqrencode-dev -y
+          sudo apt-get install clang ccache build-essential cmake pkg-config python3-zmq libevent-dev libboost-dev libsqlite3-dev libdb++-dev systemtap-sdt-dev libminiupnpc-dev libzmq3-dev qtbase5-dev qttools5-dev qttools5-dev-tools qtwayland5 libqrencode-dev -y
       - name: Compile and run tests
         run: |
           # Run tests on commits after the last merge commit and before the PR head commit
           # Use clang++, because it is a bit faster and uses less memory than g++
-          git rebase --exec "echo Running test-one-commit on \$( git log -1 ) && CC=clang CXX=clang++ cmake -B build -DWITH_ZMQ=ON -DBUILD_GUI=ON -DBUILD_BENCH=ON -DBUILD_FUZZ_BINARY=ON -DWITH_BDB=ON -DWITH_NATPMP=ON -DWITH_MINIUPNPC=ON -DWITH_USDT=ON && cmake --build build -j $(nproc) && ctest --test-dir build -j $(nproc) && ./build/test/functional/test_runner.py -j $(( $(nproc) * 2 ))" ${{ env.TEST_BASE }}
+          git rebase --exec "echo Running test-one-commit on \$( git log -1 ) && CC=clang CXX=clang++ cmake -B build -DWITH_ZMQ=ON -DBUILD_GUI=ON -DBUILD_BENCH=ON -DBUILD_FUZZ_BINARY=ON -DWITH_BDB=ON -DWITH_MINIUPNPC=ON -DWITH_USDT=ON && cmake --build build -j $(nproc) && ctest --test-dir build -j $(nproc) && ./build/test/functional/test_runner.py -j $(( $(nproc) * 2 ))" ${{ env.TEST_BASE }}
 
   macos-native-arm64:
     name: 'macOS 14 native, arm64, no depends, sqlite only, gui'
@@ -105,7 +105,7 @@ jobs:
         run: |
           # A workaround for "The `brew link` step did not complete successfully" error.
           brew install --quiet python@3 || brew link --overwrite python@3
-          brew install --quiet ninja pkg-config gnu-getopt ccache boost libevent miniupnpc libnatpmp zeromq qt@5 qrencode
+          brew install --quiet ninja pkg-config gnu-getopt ccache boost libevent miniupnpc zeromq qt@5 qrencode
 
       - name: Set Ccache directory
         run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV"

CMakeLists.txt

@@ -121,11 +121,6 @@ option(REDUCE_EXPORTS "Attempt to reduce exported symbols in the resulting execu
 option(WERROR "Treat compiler warnings as errors." OFF)
 option(WITH_CCACHE "Attempt to use ccache for compiling." ON)
 
-option(WITH_NATPMP "Enable NAT-PMP." OFF)
-if(WITH_NATPMP)
-  find_package(NATPMP MODULE REQUIRED)
-endif()
-
 option(WITH_MINIUPNPC "Enable UPnP." OFF)
 if(WITH_MINIUPNPC)
   find_package(MiniUPnPc MODULE REQUIRED)
@@ -239,7 +234,6 @@ if(BUILD_FOR_FUZZING)
   set(BUILD_WALLET_TOOL OFF)
   set(BUILD_GUI OFF)
   set(ENABLE_EXTERNAL_SIGNER OFF)
-  set(WITH_NATPMP OFF)
   set(WITH_MINIUPNPC OFF)
   set(WITH_ZMQ OFF)
   set(BUILD_TESTS OFF)
@@ -621,9 +615,7 @@ if(ENABLE_WALLET)
   message("   - legacy wallets (Berkeley DB) ..... ${WITH_BDB}")
 endif()
 message("  external signer ..................... ${ENABLE_EXTERNAL_SIGNER}")
-message("  port mapping:")
-message("   - using NAT-PMP .................... ${WITH_NATPMP}")
-message("   - using UPnP ....................... ${WITH_MINIUPNPC}")
+message("  port mapping using UPnP ............. ${WITH_MINIUPNPC}")
 message("  ZeroMQ .............................. ${WITH_ZMQ}")
 message("  USDT tracing ........................ ${WITH_USDT}")
 message("  QR code (GUI) ....................... ${WITH_QRENCODE}")

CMakePresets.json

@@ -86,7 +86,6 @@
         "WITH_BDB": "ON",
         "WITH_MINIUPNPC": "ON",
         "WITH_MULTIPROCESS": "ON",
-        "WITH_NATPMP": "ON",
         "WITH_QRENCODE": "ON",
         "WITH_SQLITE": "ON",
         "WITH_USDT": "ON",

ci/test/00_setup_env_mac_native.sh

@@ -11,7 +11,7 @@ export LC_ALL=C.UTF-8
 export PIP_PACKAGES="--break-system-packages zmq"
 export GOAL="install"
 export CMAKE_GENERATOR="Ninja"
-export BITCOIN_CONFIG="-DBUILD_GUI=ON -DWITH_ZMQ=ON -DWITH_MINIUPNPC=ON -DWITH_NATPMP=ON -DREDUCE_EXPORTS=ON"
+export BITCOIN_CONFIG="-DBUILD_GUI=ON -DWITH_ZMQ=ON -DWITH_MINIUPNPC=ON -DREDUCE_EXPORTS=ON"
 export CI_OS_NAME="macos"
 export NO_DEPENDS=1
 export OSX_SDK=""

ci/test/00_setup_env_native_asan.sh

@@ -19,7 +19,7 @@ else
 fi
 
 export CONTAINER_NAME=ci_native_asan
-export PACKAGES="systemtap-sdt-dev clang-18 llvm-18 libclang-rt-18-dev python3-zmq qtbase5-dev qttools5-dev qttools5-dev-tools libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}"
+export PACKAGES="systemtap-sdt-dev clang-18 llvm-18 libclang-rt-18-dev python3-zmq qtbase5-dev qttools5-dev qttools5-dev-tools libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}"
 export NO_DEPENDS=1
 export GOAL="install"
 export BITCOIN_CONFIG="\

ci/test/00_setup_env_native_previous_releases.sh

@@ -10,7 +10,7 @@ export CONTAINER_NAME=ci_native_previous_releases
 export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04"
 # Use minimum supported python3.9 (or best effort 3.10) and gcc-11, see doc/dependencies.md
 export PACKAGES="gcc-11 g++-11 python3-zmq"
-export DEP_OPTS="NO_UPNP=1 NO_NATPMP=1 DEBUG=1 CC=gcc-11 CXX=g++-11"
+export DEP_OPTS="NO_UPNP=1 DEBUG=1 CC=gcc-11 CXX=g++-11"
 export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash"  # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash
 export RUN_UNIT_TESTS_SEQUENTIAL="true"
 export RUN_UNIT_TESTS="false"

ci/test/00_setup_env_native_tidy.sh

@@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8
 export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04"
 export CONTAINER_NAME=ci_native_tidy
 export TIDY_LLVM_V="18"
-export PACKAGES="clang-${TIDY_LLVM_V} libclang-${TIDY_LLVM_V}-dev llvm-${TIDY_LLVM_V}-dev libomp-${TIDY_LLVM_V}-dev clang-tidy-${TIDY_LLVM_V} jq libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev qtbase5-dev qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev"
+export PACKAGES="clang-${TIDY_LLVM_V} libclang-${TIDY_LLVM_V}-dev llvm-${TIDY_LLVM_V}-dev libomp-${TIDY_LLVM_V}-dev clang-tidy-${TIDY_LLVM_V} jq libevent-dev libboost-dev libminiupnpc-dev libzmq3-dev systemtap-sdt-dev qtbase5-dev qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev"
 export NO_DEPENDS=1
 export RUN_UNIT_TESTS=false
 export RUN_FUNCTIONAL_TESTS=false
@@ -18,7 +18,7 @@ export RUN_CHECK_DEPS=true
 export RUN_TIDY=true
 export GOAL="install"
 export BITCOIN_CONFIG="\
- -DWITH_ZMQ=ON -DBUILD_GUI=ON -DBUILD_BENCH=ON -DWITH_NATPMP=ON -DWITH_MINIUPNPC=ON -DWITH_USDT=ON -DWITH_BDB=ON -DWARN_INCOMPATIBLE_BDB=OFF \
+ -DWITH_ZMQ=ON -DBUILD_GUI=ON -DBUILD_BENCH=ON -DWITH_MINIUPNPC=ON -DWITH_USDT=ON -DWITH_BDB=ON -DWARN_INCOMPATIBLE_BDB=OFF \
  -DENABLE_HARDENING=OFF \
  -DCMAKE_C_COMPILER=clang-${TIDY_LLVM_V} \
  -DCMAKE_CXX_COMPILER=clang++-${TIDY_LLVM_V} \

ci/test/00_setup_env_native_valgrind.sh

@@ -8,14 +8,14 @@ export LC_ALL=C.UTF-8
 
 export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04"
 export CONTAINER_NAME=ci_native_valgrind
-export PACKAGES="valgrind clang-16 llvm-16 libclang-rt-16-dev python3-zmq libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev"
+export PACKAGES="valgrind clang-16 llvm-16 libclang-rt-16-dev python3-zmq libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libsqlite3-dev"
 export USE_VALGRIND=1
 export NO_DEPENDS=1
 export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra"  # feature_init excluded for now, see https://github.com/bitcoin/bitcoin/issues/30011 ; bind tests excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547
 export GOAL="install"
 # TODO enable GUI
 export BITCOIN_CONFIG="\
- -DWITH_ZMQ=ON -DWITH_BDB=ON -DWITH_NATPMP=ON -DWITH_MINIUPNPC=ON -DWARN_INCOMPATIBLE_BDB=OFF -DBUILD_GUI=OFF \
+ -DWITH_ZMQ=ON -DWITH_BDB=ON -DWITH_MINIUPNPC=ON -DWARN_INCOMPATIBLE_BDB=OFF -DBUILD_GUI=OFF \
  -DCMAKE_C_COMPILER=clang-16 \
  -DCMAKE_CXX_COMPILER=clang++-16 \
 "

cmake/module/FindNATPMP.cmake

@@ -42,7 +42,6 @@ NO_WALLET ?=
 NO_ZMQ ?=
 NO_UPNP ?=
 NO_USDT ?=
-NO_NATPMP ?=
 MULTIPROCESS ?=
 LTO ?=
 NO_HARDEN ?=
@@ -159,13 +158,12 @@ sqlite_packages_$(NO_SQLITE) = $(sqlite_packages)
 wallet_packages_$(NO_WALLET) = $(bdb_packages_) $(sqlite_packages_)
 
 upnp_packages_$(NO_UPNP) = $(upnp_packages)
-natpmp_packages_$(NO_NATPMP) = $(natpmp_packages)
 
 zmq_packages_$(NO_ZMQ) = $(zmq_packages)
 multiprocess_packages_$(MULTIPROCESS) = $(multiprocess_packages)
 usdt_packages_$(NO_USDT) = $(usdt_$(host_os)_packages)
 
-packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(boost_packages_) $(libevent_packages_) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_) $(usdt_packages_)
+packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(boost_packages_) $(libevent_packages_) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(usdt_packages_)
 native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages)
 
 ifneq ($(zmq_packages_),)
@@ -233,7 +231,6 @@ $(host_prefix)/toolchain.cmake : toolchain.cmake.in $(host_prefix)/.stamp_$(fina
             -e 's|@bdb_packages@|$(bdb_packages_)|' \
             -e 's|@sqlite_packages@|$(sqlite_packages_)|' \
             -e 's|@upnp_packages@|$(upnp_packages_)|' \
-            -e 's|@natpmp_packages@|$(natpmp_packages_)|' \
             -e 's|@usdt_packages@|$(usdt_packages_)|' \
             -e 's|@no_harden@|$(NO_HARDEN)|' \
             -e 's|@multiprocess@|$(MULTIPROCESS)|' \