chore: migrate to conan 2 (#440)

* chore: bump to conan2

* gha: fix conan-cache-action

* chore: run python formatting

* chore: more formatting fixes

* chore: fix compilation warnings

* gha: fix conan caching

* chore: bump protobuf version

* ci: run dev.conan to generate the generators

* fix(dist-test): build conan first

* fix(gha): temporarily disable TSAN and ASAN

* fix: use different conan profiles for sanitisers

* fix: run cpp formatting

* fix(conan): update conan-release.lock

* fix(ci): add conan toolchain to example's build

* fix(gha): don't force --clean on the conan-cache step

Author: csegarragonz

.env

@@ -1,4 +1,4 @@
-FAABRIC_VERSION=0.21.0
-FAABRIC_CLI_IMAGE=ghcr.io/faasm/faabric:0.21.0
+FAABRIC_VERSION=0.22.0
+FAABRIC_CLI_IMAGE=ghcr.io/faasm/faabric:0.22.0
 COMPOSE_PROJECT_NAME=faabric-dev
 CONAN_CACHE_MOUNT_SOURCE=./conan-cache/

.github/workflows/tests.yml

@@ -20,21 +20,26 @@ jobs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/faasm/faabric:0.21.0
+      image: ghcr.io/faasm/faabric:0.22.0
     env:
       DEPLOYMENT_TYPE: gha-ci
     steps:
       - name: "Check-out code"
         uses: actions/checkout@v4
-      - uses: faasm/conan-cache-action@v3
-      - name: "Build Conan CMake deps to be shared by all runs"
-        run: ./bin/inv_wrapper.sh dev.cmake --build Debug --clean
+      - uses: faasm/conan-cache-action@main
+        with:
+          build-type: Debug
+          from: faabric
+      - name: "Build Conan deps to be shared by all runs"
+        run: |
+          ./bin/inv_wrapper.sh dev.conan --build Debug
+          ./bin/inv_wrapper.sh dev.conan --build Release
 
   docs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/faasm/faabric:0.21.0
+      image: ghcr.io/faasm/faabric:0.22.0
     steps:
       - name: "Check out code"
         uses: actions/checkout@v4
@@ -45,7 +50,7 @@ jobs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/faasm/faabric:0.21.0
+      image: ghcr.io/faasm/faabric:0.22.0
     steps:
       - name: "Check out code"
         uses: actions/checkout@v4
@@ -65,17 +70,22 @@ jobs:
       REDIS_QUEUE_HOST: redis
       REDIS_STATE_HOST: redis
     container:
-      image: ghcr.io/faasm/faabric:0.21.0
+      image: ghcr.io/faasm/faabric:0.22.0
       options: --privileged
     services:
       redis:
         image: redis
     steps:
       - name: "Check-out code"
         uses: actions/checkout@v4
-      - uses: faasm/conan-cache-action@v3
+      - uses: faasm/conan-cache-action@main
+        with:
+          build-type: Debug
+          from: faabric
       - name: "Ping redis"
         run: redis-cli -h redis ping
+      - name: "Run conan"
+        run: ./bin/inv_wrapper.sh dev.conan --build=Debug
       - name: "Run cmake for tests"
         run: ./bin/inv_wrapper.sh dev.cmake --clean --build=Debug --coverage
       - name: "Build tests"
@@ -98,23 +108,35 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        sanitiser: [None, Address, Thread, Undefined]
+        sanitiser: [None, Thread, Address, Undefined]
     env:
       DEPLOYMENT_TYPE: gha-ci
       REDIS_QUEUE_HOST: redis
       REDIS_STATE_HOST: redis
     container:
-      image: ghcr.io/faasm/faabric:0.21.0
+      image: ghcr.io/faasm/faabric:0.22.0
       options: --privileged
     services:
       redis:
         image: redis
     steps:
       - name: "Check-out code"
         uses: actions/checkout@v4
-      - uses: faasm/conan-cache-action@v3
+      - uses: faasm/conan-cache-action@main
+        with:
+          build-type: Debug
+          from: faabric
       - name: "Ping redis"
         run: redis-cli -h redis ping
+      # For sanitised builds, we need to also re-build all Conan dependencies
+      # so that they are also sanitized. Otherwise this causes unexpected
+      # crashes.
+      - name: "Re-build conan deps"
+        if: ${{ matrix.sanitiser != 'None' }}
+        run: ./bin/inv_wrapper.sh dev.conan --clean --build=Debug --sanitiser ${{ matrix.sanitiser }}
+      - name: "Run conan"
+        if: ${{ matrix.sanitiser == 'None' }}
+        run: ./bin/inv_wrapper.sh dev.conan --build=Debug --sanitiser ${{ matrix.sanitiser }}
       - name: "Run cmake for tests"
         run: ./bin/inv_wrapper.sh dev.cmake --clean --build=Debug --sanitiser ${{ matrix.sanitiser }}
       - name: "Build tests"
@@ -133,8 +155,10 @@ jobs:
       # --- Code update ---
       - name: "Check out code"
         uses: actions/checkout@v4
-      - name: "Conan cache"
-        uses: faasm/conan-cache-action@v3
+      - uses: faasm/conan-cache-action@main
+        with:
+          build-type: Debug
+          from: faabric
       - name: "Build distributed tests"
         run: ./dist-test/build.sh
       - name: "Run the distributed tests"
@@ -156,22 +180,27 @@ jobs:
       REDIS_QUEUE_HOST: redis
       REDIS_STATE_HOST: redis
     container:
-      image: ghcr.io/faasm/faabric:0.21.0
+      image: ghcr.io/faasm/faabric:0.22.0
     services:
       redis:
         image: redis
     steps:
       - name: "Check-out code"
         uses: actions/checkout@v4
-      - uses: faasm/conan-cache-action@v3
+      - uses: faasm/conan-cache-action@main
         with:
-          build-type: "release"
+          build-type: Release
+          from: faabric
+      - name: "Run conan"
+        run: ./bin/inv_wrapper.sh dev.conan --build=Release
       - name: "Run cmake shared"
         run: ./bin/inv_wrapper.sh dev.cmake --clean --shared --build=Release
       - name: "Build Faabric shared library"
         run: ./bin/inv_wrapper.sh dev.cc faabric --shared
       - name: "Install Faabric shared library"
         run: ./bin/inv_wrapper.sh dev.install faabric --shared
+      - name: "Re-run conan (needed for examples)"
+        run: ./bin/inv_wrapper.sh dev.conan --build=Release
       - name: "Build examples"
         run: ./bin/inv_wrapper.sh examples
       - name: "Run example to check"

.ycm_extra_conf.py

@@ -1 +1 @@
-0.21.0
+0.22.0

VERSION

@@ -4,86 +4,18 @@ include (ExternalProject)
 include (FetchContent)
 find_package (Threads REQUIRED)
 
-# Find conan-generated package descriptions
+# Find conan-generated package descriptions.
 list(PREPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_BINARY_DIR})
 list(PREPEND CMAKE_PREFIX_PATH ${CMAKE_CURRENT_BINARY_DIR})
 
-if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/conan.cmake")
-  message(STATUS "Downloading conan.cmake from https://github.com/conan-io/cmake-conan")
-  file(DOWNLOAD "https://raw.githubusercontent.com/conan-io/cmake-conan/0.18.1/conan.cmake"
-                "${CMAKE_CURRENT_BINARY_DIR}/conan.cmake"
-                TLS_VERIFY ON)
-endif()
-
-set(CONAN_CMAKE_SILENT_OUTPUT ON CACHE INTERNAL "")
-include(${CMAKE_CURRENT_BINARY_DIR}/conan.cmake)
-
-conan_check(VERSION 1.63.0 REQUIRED)
-
-# Enable revisions in the conan config
-execute_process(COMMAND ${CONAN_CMD} config set general.revisions_enabled=1
-                RESULT_VARIABLE RET_CODE)
-if(NOT "${RET_CODE}" STREQUAL "0")
-    message(FATAL_ERROR "Error setting revisions for Conan: '${RET_CODE}'")
-endif()
-
-# --------------------------------
-# Conan dependencies
-# --------------------------------
-
-conan_cmake_configure(
-    REQUIRES
-        "abseil/20220623.0@#732381dc99db29b4cfd293684891da56"
-        "boost/1.84.0@#7604ce1e7485780469dffb6430f232ea"
-        "catch2/2.13.9@#8793d3e6287d3684201418de556d98fe"
-        "flatbuffers/23.5.26@#b153646f6546daab4c7326970b6cd89c"
-        "hiredis/1.0.2@#370dad964286cadb1f15dc90252e8ef3"
-        "openssl/3.0.2@#269fa93e5afe8c34bd9a0030d2b8f0fe"
-        "protobuf/3.20.0@#8e4de7081bea093469c9e6076149b2b4"
-        "readerwriterqueue/1.0.6@#a95c8da3d68822dec4d4c13fff4b5c96"
-        "spdlog/1.10.0@#6406c337028e15e56cd6a070cbac54c4"
-        "zlib/1.2.12@#3b9e037ae1c615d045a06c67d88491ae"
-    GENERATORS
-        cmake_find_package
-        cmake_paths
-    OPTIONS
-        boost:error_code_header_only=True
-        boost:system_no_deprecated=True
-        boost:zlib=False
-        boost:bzip2=False
-        boost:lzma=False
-        boost:zstd=False
-        boost:without_locale=True
-        boost:without_log=True
-        boost:without_mpi=True
-        boost:without_python=True
-        boost:without_test=True
-        boost:without_wave=True
-)
-
-conan_cmake_autodetect(FAABRIC_CONAN_SETTINGS)
-
-conan_cmake_install(PATH_OR_REFERENCE .
-                    BUILD outdated
-                    UPDATE
-                    REMOTE conancenter
-                    PROFILE_HOST ${CMAKE_CURRENT_LIST_DIR}/../conan-profile.txt
-                    PROFILE_BUILD ${CMAKE_CURRENT_LIST_DIR}/../conan-profile.txt
-                    SETTINGS ${FAABRIC_CONAN_SETTINGS}
-)
-
-include(${CMAKE_CURRENT_BINARY_DIR}/conan_paths.cmake)
-
 find_package(absl QUIET REQUIRED)
-find_package(Boost 1.80.0 QUIET REQUIRED)
+find_package(Boost 1.84.0 QUIET REQUIRED COMPONENTS filesystem program_options system)
 find_package(Catch2 QUIET REQUIRED)
-find_package(flatbuffers QUIET REQUIRED)
+find_package(flatbuffers CONFIG QUIET REQUIRED)
 find_package(fmt QUIET REQUIRED)
 find_package(hiredis QUIET REQUIRED)
-# 27/01/2023 - Pin OpenSSL to a specific version to avoid incompatibilities
-# with the system's (i.e. Ubuntu 22.04) OpenSSL
-find_package(OpenSSL 3.0.2 QUIET REQUIRED)
-find_package(Protobuf 3.20.0 QUIET REQUIRED)
+find_package(OpenSSL 3.6.0 QUIET REQUIRED)
+find_package(Protobuf 6.30.1 QUIET REQUIRED)
 find_package(readerwriterqueue QUIET REQUIRED)
 find_package(spdlog QUIET REQUIRED)
 find_package(ZLIB QUIET REQUIRED)
@@ -121,7 +53,7 @@ FetchContent_Declare(nng_ext
 )
 FetchContent_Declare(zstd_ext
     GIT_REPOSITORY "https://github.com/facebook/zstd"
-    GIT_TAG "v1.5.2"
+    GIT_TAG "v1.5.7"
     SOURCE_SUBDIR "build/cmake"
 )
 
@@ -150,7 +82,9 @@ target_link_libraries(faabric_common_dependencies INTERFACE
     absl::flat_hash_map
     absl::strings
     atomic_queue::atomic_queue
-    Boost::Boost
+    Boost::headers
+    Boost::filesystem
+    Boost::program_options
     Boost::system
     flatbuffers::flatbuffers
     hiredis::hiredis

cmake/ExternalProjects.cmake

@@ -0,0 +1,23 @@
+{
+    "version": "0.5",
+    "requires": [
+        "zlib/1.2.12#2ea72a0bae8b680f8a282e103e0a6880%1743582313.324",
+        "spdlog/1.10.0#5aed29a37b544d0eb5812b8be74cd7a4%1731353179.805",
+        "readerwriterqueue/1.0.6#aaa5ff6fac60c2aee591e9e51b063b83%1679587692.891",
+        "protobuf/6.30.1#97bae23ef6d7f9fcfdb4ded9468ad6de%1747665968.233",
+        "openssl/3.6.0#89e8af1d4a21afcac0557079d23d8890%1759746682.365",
+        "libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1%1722218217.276",
+        "hiredis/1.0.2#6001a683c04da07565e97e76d0d841bd%1744279494.11",
+        "fmt/8.1.1#d66420384a943dbf1f292d34ef34c11c%1735899179.969",
+        "flatbuffers/23.5.26#0290575326fe9b2c39a0236ea81d8c30%1743154511.803",
+        "catch2/2.13.9#17ac9b0b78c63353e304c5744e862a77%1678135818.816",
+        "boost/1.84.0#40dd9fecacce0ef109851d0e38727fd0%1759416424.265",
+        "abseil/20250127.0#faefa3bbf31b5c32933e328d72e42cfe%1754142622.985"
+    ],
+    "build_requires": [
+        "cmake/3.31.9#2032c6471fe4f5a3e17f65fed518d545%1758832282.188",
+        "b2/5.3.3#107c15377719889654eb9a162a673975%1750340310.079"
+    ],
+    "python_requires": [],
+    "config_requires": []
+}

conan-debug.lock

@@ -0,0 +1,15 @@
+[settings]
+os=Linux
+arch=x86_64
+compiler=clang
+compiler.version=17
+compiler.libcxx=libstdc++11
+compiler.cppstd=20
+build_type=RelWithDebInfo
+
+[conf]
+tools.build:compiler_executables={"c": "/usr/bin/clang-17", "cpp": "/usr/bin/clang++-17"}
+tools.build:cflags=["-fsanitize=address","-fno-omit-frame-pointer","-g","-O1"]
+tools.build:cxxflags=["-fsanitize=address","-fno-omit-frame-pointer","-g","-O1"]
+tools.build:sharedlinkflags=["-fsanitize=address"]
+tools.build:exelinkflags=["-fsanitize=address"]

conan-profiles/asan.txt

@@ -1,15 +1,11 @@
 [settings]
 os=Linux
-os_build=Linux
 arch=x86_64
-arch_build=x86_64
 compiler=clang
 compiler.version=17
 compiler.libcxx=libstdc++11
 compiler.cppstd=20
 build_type=Release
-[options]
-[build_requires]
-[env]
-CC=/usr/bin/clang-17
-CXX=/usr/bin/clang++-17
+
+[conf]
+tools.build:compiler_executables={"c": "/usr/bin/clang-17", "cpp": "/usr/bin/clang++-17"}

conan-profile.txt conan-profiles/default.txtconan-profile.txt renamed to conan-profiles/default.txt

@@ -0,0 +1,15 @@
+[settings]
+os=Linux
+arch=x86_64
+compiler=clang
+compiler.version=17
+compiler.libcxx=libstdc++11
+compiler.cppstd=20
+build_type=RelWithDebInfo
+
+[conf]
+tools.build:compiler_executables={"c": "/usr/bin/clang-17", "cpp": "/usr/bin/clang++-17"}
+tools.build:cxxflags=["-fsanitize=thread","-fno-omit-frame-pointer","-g"]
+tools.build:cflags=["-fsanitize=thread","-fno-omit-frame-pointer","-g"]
+tools.build:sharedlinkflags=["-fsanitize=thread"]
+tools.build:exelinkflags=["-fsanitize=thread"]

conan-profiles/tsan.txt

@@ -0,0 +1,23 @@
+{
+    "version": "0.5",
+    "requires": [
+        "zlib/1.2.12#2ea72a0bae8b680f8a282e103e0a6880%1743582313.324",
+        "spdlog/1.10.0#5aed29a37b544d0eb5812b8be74cd7a4%1731353179.805",
+        "readerwriterqueue/1.0.6#aaa5ff6fac60c2aee591e9e51b063b83%1679587692.891",
+        "protobuf/6.30.1#97bae23ef6d7f9fcfdb4ded9468ad6de%1747665968.233",
+        "openssl/3.6.0#89e8af1d4a21afcac0557079d23d8890%1759746682.365",
+        "libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1%1722218217.276",
+        "hiredis/1.0.2#6001a683c04da07565e97e76d0d841bd%1744279494.11",
+        "fmt/8.1.1#d66420384a943dbf1f292d34ef34c11c%1735899179.969",
+        "flatbuffers/23.5.26#0290575326fe9b2c39a0236ea81d8c30%1743154511.803",
+        "catch2/2.13.9#17ac9b0b78c63353e304c5744e862a77%1678135818.816",
+        "boost/1.84.0#40dd9fecacce0ef109851d0e38727fd0%1759416424.265",
+        "abseil/20250127.0#faefa3bbf31b5c32933e328d72e42cfe%1754142622.985"
+    ],
+    "build_requires": [
+        "cmake/3.31.9#2032c6471fe4f5a3e17f65fed518d545%1758832282.188",
+        "b2/5.3.3#107c15377719889654eb9a162a673975%1750340310.079"
+    ],
+    "python_requires": [],
+    "config_requires": []
+}