Merge branch 'main' into poc_config_yaml

Author: marcalff

.github/workflows/ci.yml

@@ -73,7 +73,7 @@ jobs:
     - name: install dependencies
       run: |
         sudo -E apt-get update
-        sudo -E apt-get install -y zlib1g-dev libabsl-dev libprotobuf-dev libgrpc++-dev protobuf-compiler protobuf-compiler-grpc
+        sudo -E apt-get install -y zlib1g-dev
     - name: run fetch content cmake test
       run: |
         ./ci/do_ci.sh cmake.fetch_content.test

.github/workflows/codeql-analysis.yml

@@ -37,10 +37,10 @@ jobs:
       run: |
         sudo -E ./ci/setup_ci_environment.sh
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@181d5eefc20863364f96762470ba6f862bdef56b # v3.29.2
+      uses: github/codeql-action/init@d6bbdef45e766d081b84a2def353b0055f728d3e # v3.29.3
       with:
        languages: cpp
     - name: Autobuild
-      uses: github/codeql-action/autobuild@181d5eefc20863364f96762470ba6f862bdef56b # v3.29.2
+      uses: github/codeql-action/autobuild@d6bbdef45e766d081b84a2def353b0055f728d3e # v3.29.3
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@181d5eefc20863364f96762470ba6f862bdef56b # v3.29.2
+      uses: github/codeql-action/analyze@d6bbdef45e766d081b84a2def353b0055f728d3e # v3.29.3

.github/workflows/ossf-scorecard.yml

@@ -47,6 +47,6 @@ jobs:
       # Upload the results to GitHub's code scanning dashboard (optional).
       # Commenting out will disable upload of results to your repo's Code Scanning dashboard
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@181d5eefc20863364f96762470ba6f862bdef56b # v3.29.2
+        uses: github/codeql-action/upload-sarif@d6bbdef45e766d081b84a2def353b0055f728d3e # v3.29.3
         with:
           sarif_file: results.sarif

CMakeLists.txt

@@ -346,83 +346,23 @@ if(WITH_PROMETHEUS)
   include("${opentelemetry-cpp_SOURCE_DIR}/cmake/prometheus-cpp.cmake")
 endif()
 
+#
+# Do we need protobuf and/or gRPC ?
+#
+
 if(WITH_OTLP_GRPC
    OR WITH_OTLP_HTTP
    OR WITH_OTLP_FILE)
 
-  # Including the CMakeFindDependencyMacro resolves an error from
-  # gRPCConfig.cmake on some grpc versions. See
-  # https://github.com/grpc/grpc/pull/33361 for more details.
-  include(CMakeFindDependencyMacro)
-
-  # Protobuf 3.22+ depends on abseil-cpp and must be found using the cmake
-  # find_package CONFIG search mode. The following attempts to find Protobuf
-  # using the CONFIG mode first, and if not found, falls back to the MODULE
-  # mode. See https://gitlab.kitware.com/cmake/cmake/-/issues/24321 for more
-  # details.
-  find_package(Protobuf CONFIG)
-  if(NOT Protobuf_FOUND)
-    find_package(Protobuf MODULE)
-    if(Protobuf_FOUND AND Protobuf_VERSION VERSION_GREATER_EQUAL "3.22.0")
-      message(
-        WARNING
-          "Found Protobuf version ${Protobuf_VERSION} using MODULE mode. "
-          "Linking errors may occur. Protobuf 3.22+ depends on abseil-cpp "
-          "and should be found using the CONFIG mode.")
-    endif()
-  endif()
-
+  # find or fetch grpc before protobuf to allow protobuf to be built in-tree as
+  # a grpc submodule.
   if(WITH_OTLP_GRPC)
-    find_package(gRPC CONFIG)
+    include("${opentelemetry-cpp_SOURCE_DIR}/cmake/grpc.cmake")
   endif()
-  if((NOT Protobuf_FOUND) OR (WITH_OTLP_GRPC AND NOT gRPC_FOUND))
 
-    if(WIN32 AND (NOT DEFINED CMAKE_TOOLCHAIN_FILE))
-      message(FATAL_ERROR "Windows dependency installation failed!")
-    endif()
-    if(WIN32)
-      include(${CMAKE_TOOLCHAIN_FILE})
-    endif()
-
-    if(NOT Protobuf_FOUND)
-      find_package(Protobuf CONFIG REQUIRED)
-    endif()
-    if(NOT gRPC_FOUND AND WITH_OTLP_GRPC)
-      find_package(gRPC CONFIG)
-    endif()
-    if(WIN32)
-      # Always use x64 protoc.exe
-      if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}")
-        set(Protobuf_PROTOC_EXECUTABLE
-            ${CMAKE_CURRENT_SOURCE_DIR}/tools/vcpkg/packages/protobuf_x64-windows/tools/protobuf/protoc.exe
-        )
-      endif()
-    endif()
-  endif()
-  # Latest Protobuf imported targets and without legacy module support
-  if(TARGET protobuf::protoc)
-    if(CMAKE_CROSSCOMPILING AND Protobuf_PROTOC_EXECUTABLE)
-      set(PROTOBUF_PROTOC_EXECUTABLE ${Protobuf_PROTOC_EXECUTABLE})
-    else()
-      project_build_tools_get_imported_location(PROTOBUF_PROTOC_EXECUTABLE
-                                                protobuf::protoc)
-      # If protobuf::protoc is not a imported target, then we use the target
-      # directly for fallback
-      if(NOT PROTOBUF_PROTOC_EXECUTABLE)
-        set(PROTOBUF_PROTOC_EXECUTABLE protobuf::protoc)
-      endif()
-    endif()
-  elseif(Protobuf_PROTOC_EXECUTABLE)
-    # Some versions of FindProtobuf.cmake uses mixed case instead of uppercase
-    set(PROTOBUF_PROTOC_EXECUTABLE ${Protobuf_PROTOC_EXECUTABLE})
-  endif()
-  include(CMakeDependentOption)
+  include("${opentelemetry-cpp_SOURCE_DIR}/cmake/protobuf.cmake")
 
-  message(STATUS "PROTOBUF_PROTOC_EXECUTABLE=${PROTOBUF_PROTOC_EXECUTABLE}")
-  set(SAVED_CMAKE_CXX_CLANG_TIDY ${CMAKE_CXX_CLANG_TIDY})
-  set(CMAKE_CXX_CLANG_TIDY "")
   include("${opentelemetry-cpp_SOURCE_DIR}/cmake/opentelemetry-proto.cmake")
-  set(CMAKE_CXX_CLANG_TIDY ${SAVED_CMAKE_CXX_CLANG_TIDY})
 endif()
 
 #
@@ -658,11 +598,14 @@ if(opentelemetry-proto_VERSION)
       "opentelemetry-proto: ${opentelemetry-proto_VERSION} (${opentelemetry-proto_PROVIDER})"
   )
 endif()
-if(Protobuf_FOUND)
-  message(STATUS "Protobuf: ${Protobuf_VERSION}")
+if(Protobuf_VERSION)
+  message(
+    STATUS
+      "Protobuf: ${Protobuf_VERSION} (${Protobuf_PROVIDER} - ${protobuf_lib_type})"
+  )
 endif()
-if(gRPC_FOUND)
-  message(STATUS "gRPC: ${gRPC_VERSION}")
+if(gRPC_VERSION)
+  message(STATUS "gRPC: ${gRPC_VERSION} (${gRPC_PROVIDER} - ${grpc_lib_type})")
 endif()
 if(CURL_VERSION)
   message(STATUS "CURL: ${CURL_VERSION} (${CURL_PROVIDER})")

ci/do_ci.sh

@@ -425,6 +425,7 @@ elif [[ "$1" == "cmake.exporter.otprotocol.shared_libs.with_static_grpc.test" ]]
         -DWITH_OTLP_HTTP=ON \
         -DWITH_OTLP_FILE=ON \
         -DBUILD_SHARED_LIBS=ON \
+        -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
         "${SRC_DIR}"
   grpc_cpp_plugin=`which grpc_cpp_plugin`
   proto_make_file="CMakeFiles/opentelemetry_proto.dir/build.make"

cmake/grpc.cmake

@@ -0,0 +1,84 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+# Import gRPC targets (gRPC::grpc++ and gRPC::grpc_cpp_plugin).
+# 1. Find an installed gRPC package
+# 2. Use FetchContent to fetch and build gRPC (and its submodules) from GitHub
+
+# Including the CMakeFindDependencyMacro resolves an error from
+# gRPCConfig.cmake on some grpc versions. See
+# https://github.com/grpc/grpc/pull/33361 for more details.
+include(CMakeFindDependencyMacro)
+
+find_package(gRPC CONFIG QUIET)
+set(gRPC_PROVIDER "find_package")
+
+if(NOT gRPC_FOUND)
+  FetchContent_Declare(
+      "grpc"
+      GIT_REPOSITORY  "https://github.com/grpc/grpc.git"
+      GIT_TAG "${grpc_GIT_TAG}"
+      GIT_SUBMODULES
+          "third_party/re2"
+          "third_party/abseil-cpp"
+          "third_party/protobuf"
+          "third_party/cares/cares"
+          "third_party/boringssl-with-bazel"
+      )
+  set(gRPC_PROVIDER "fetch_repository")
+
+  set(gRPC_INSTALL ${OPENTELEMETRY_INSTALL} CACHE BOOL "" FORCE)
+  set(gRPC_BUILD_TESTS OFF CACHE BOOL "" FORCE)
+  set(gRPC_BUILD_GRPC_CPP_PLUGIN ON CACHE BOOL "" FORCE)
+  set(gRPC_BUILD_GRPC_CSHARP_PLUGIN OFF CACHE BOOL "" FORCE)
+  set(gRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN OFF CACHE BOOL "" FORCE)
+  set(gRPC_BUILD_GRPC_PHP_PLUGIN OFF CACHE BOOL "" FORCE)
+  set(gRPC_BUILD_GRPC_NODE_PLUGIN OFF CACHE BOOL "" FORCE)
+  set(gRPC_BUILD_GRPC_PYTHON_PLUGIN OFF CACHE BOOL "" FORCE)
+  set(gRPC_BUILD_GRPC_RUBY_PLUGIN OFF CACHE BOOL "" FORCE)
+  set(gRPC_BUILD_GRPCPP_OTEL_PLUGIN OFF CACHE BOOL "" FORCE)
+  set(gRPC_ZLIB_PROVIDER "package" CACHE STRING "" FORCE)
+  set(gRPC_RE2_PROVIDER "module"  CACHE STRING "" FORCE)
+  set(RE2_BUILD_TESTING OFF CACHE BOOL "" FORCE)
+  set(gRPC_PROTOBUF_PROVIDER "module" CACHE STRING "" FORCE)
+  set(gRPC_PROTOBUF_PACKAGE_TYPE "CONFIG" CACHE STRING "" FORCE)
+  set(gRPC_ABSL_PROVIDER "module" CACHE STRING "" FORCE)
+  set(gRPC_CARES_PROVIDER "module" CACHE STRING "" FORCE)
+
+  FetchContent_MakeAvailable(grpc)
+
+  # Set the gRPC_VERSION variable from the git tag.
+  string(REGEX REPLACE "^v([0-9]+\\.[0-9]+\\.[0-9]+)$" "\\1" gRPC_VERSION "${grpc_GIT_TAG}")
+
+  #Disable iwyu and clang-tidy
+  foreach(_grpc_target grpc++ grpc_cpp_plugin)
+    if(TARGET ${_grpc_target})
+      set_target_properties(${_grpc_target} PROPERTIES POSITION_INDEPENDENT_CODE ON CXX_INCLUDE_WHAT_YOU_USE ""
+                                                      CXX_CLANG_TIDY "")
+    endif()
+  endforeach()
+
+  if(TARGET grpc++ AND NOT TARGET gRPC::grpc++)
+    add_library(gRPC::grpc++ ALIAS grpc++)
+  endif()
+
+  if(TARGET grpc_cpp_plugin AND NOT TARGET gRPC::grpc_cpp_plugin)
+    add_executable(gRPC::grpc_cpp_plugin ALIAS grpc_cpp_plugin)
+  endif()
+
+endif()
+
+if(NOT TARGET gRPC::grpc++)
+  message(FATAL_ERROR "A required gRPC target (gRPC::grpc++) was not imported")
+endif()
+
+if(CMAKE_CROSSCOMPILING)
+  find_program(gRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin)
+else()
+  if(NOT TARGET gRPC::grpc_cpp_plugin)
+    message(FATAL_ERROR "A required gRPC target (gRPC::grpc_cpp_plugin) was not imported")
+  endif()
+  set(gRPC_CPP_PLUGIN_EXECUTABLE "$<TARGET_FILE:gRPC::grpc_cpp_plugin>")
+endif()
+
+message(STATUS "gRPC_CPP_PLUGIN_EXECUTABLE=${gRPC_CPP_PLUGIN_EXECUTABLE}")

cmake/opentelemetry-proto.cmake

@@ -174,20 +174,6 @@ foreach(IMPORT_DIR ${PROTOBUF_IMPORT_DIRS})
   list(APPEND PROTOBUF_INCLUDE_FLAGS "-I${IMPORT_DIR}")
 endforeach()
 
-if(WITH_OTLP_GRPC)
-  if(CMAKE_CROSSCOMPILING)
-    find_program(gRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin)
-  else()
-    if(TARGET gRPC::grpc_cpp_plugin)
-      project_build_tools_get_imported_location(gRPC_CPP_PLUGIN_EXECUTABLE
-                                                gRPC::grpc_cpp_plugin)
-    else()
-      find_program(gRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin)
-    endif()
-  endif()
-  message(STATUS "gRPC_CPP_PLUGIN_EXECUTABLE=${gRPC_CPP_PLUGIN_EXECUTABLE}")
-endif()
-
 set(PROTOBUF_COMMON_FLAGS "--proto_path=${PROTO_PATH}"
                           "--cpp_out=${GENERATED_PROTOBUF_PATH}")
 # --experimental_allow_proto3_optional is available from 3.13 and be stable and
@@ -239,7 +225,10 @@ set(PROTOBUF_GENERATED_FILES
     ${PROFILES_SERVICE_PB_H_FILE}
     ${PROFILES_SERVICE_PB_CPP_FILE})
 
+set(PROTOBUF_GENERATE_DEPENDS ${PROTOBUF_PROTOC_EXECUTABLE})
+
 if(WITH_OTLP_GRPC)
+  list(APPEND PROTOBUF_GENERATE_DEPENDS ${gRPC_CPP_PLUGIN_EXECUTABLE})
   list(APPEND PROTOBUF_COMMON_FLAGS
        "--grpc_out=generate_mock_code=true:${GENERATED_PROTOBUF_PATH}"
        --plugin=protoc-gen-grpc="${gRPC_CPP_PLUGIN_EXECUTABLE}")
@@ -284,7 +273,7 @@ add_custom_command(
     ${LOGS_PROTO} ${METRICS_PROTO} ${TRACE_SERVICE_PROTO} ${LOGS_SERVICE_PROTO}
     ${METRICS_SERVICE_PROTO} ${PROFILES_PROTO} ${PROFILES_SERVICE_PROTO}
   COMMENT "[Run]: ${PROTOBUF_RUN_PROTOC_COMMAND}"
-  DEPENDS ${PROTOBUF_PROTOC_EXECUTABLE})
+  DEPENDS ${PROTOBUF_GENERATE_DEPENDS})
 
 unset(OTELCPP_PROTO_TARGET_OPTIONS)
 if(CMAKE_SYSTEM_NAME MATCHES "Windows|MinGW|WindowsStore")

cmake/protobuf.cmake

@@ -0,0 +1,80 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+
+# Import Protobuf targets (protobuf::libprotobuf and protobuf::protoc) and set PROTOBUF_PROTOC_EXECUTABLE.
+# 1. If gRPC was fetched from github then use the Protobuf submodule built with gRPC
+# 2. Find an installed Protobuf package
+# 3. Use FetchContent to fetch and build Protobuf from GitHub
+
+if(DEFINED gRPC_PROVIDER AND NOT gRPC_PROVIDER STREQUAL "find_package" AND TARGET libprotobuf)
+  # gRPC was fetched and built Protobuf as a submodule
+
+  set(_Protobuf_VERSION_REGEX "\"cpp\"[ \t]*:[ \t]*\"([0-9]+\\.[0-9]+(\\.[0-9]+)?)\"")
+  set(_Protobuf_VERSION_FILE "${grpc_SOURCE_DIR}/third_party/protobuf/version.json")
+
+  file(READ "${_Protobuf_VERSION_FILE}" _Protobuf_VERSION_FILE_CONTENTS)
+  if(_Protobuf_VERSION_FILE_CONTENTS MATCHES ${_Protobuf_VERSION_REGEX})
+    set(Protobuf_VERSION "${CMAKE_MATCH_1}")
+  else()
+    message(WARNING "Failed to parse Protobuf version from ${_Protobuf_VERSION_FILE} using regex ${_Protobuf_VERSION_REGEX}")
+  endif()
+  set(Protobuf_PROVIDER "grpc_submodule")
+else()
+
+  # Search for an installed Protobuf package explicitly using the CONFIG search mode first followed by the MODULE search mode.
+  # Protobuf versions < 3.22.0 may be found using the module mode and some protobuf apt packages do not support the CONFIG search.
+
+  find_package(Protobuf CONFIG QUIET)
+  set(Protobuf_PROVIDER "find_package")
+
+  if(NOT Protobuf_FOUND)
+    find_package(Protobuf MODULE QUIET)
+  endif()
+
+  if(NOT Protobuf_FOUND)
+    FetchContent_Declare(
+      "protobuf"
+      GIT_REPOSITORY "https://github.com/protocolbuffers/protobuf.git"
+      GIT_TAG "${protobuf_GIT_TAG}"
+    )
+
+    set(protobuf_INSTALL ${OPENTELEMETRY_INSTALL} CACHE BOOL "" FORCE)
+    set(protobuf_BUILD_TESTS OFF CACHE BOOL "" FORCE)
+    set(protobuf_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
+
+    FetchContent_MakeAvailable(protobuf)
+
+    set(Protobuf_PROVIDER "fetch_repository")
+
+    # Set the Protobuf_VERSION variable from the git tag.
+    string(REGEX REPLACE "^v([0-9]+\\.[0-9]+\\.[0-9]+)$" "\\1" Protobuf_VERSION "${protobuf_GIT_TAG}")
+
+    if(TARGET libprotobuf)
+      set_target_properties(libprotobuf PROPERTIES POSITION_INDEPENDENT_CODE ON CXX_CLANG_TIDY "" CXX_INCLUDE_WHAT_YOU_USE "")
+    endif()
+
+  endif()
+endif()
+
+if(NOT TARGET protobuf::libprotobuf)
+  message(FATAL_ERROR "A required protobuf target (protobuf::libprotobuf) was not imported")
+endif()
+
+if(PROTOBUF_PROTOC_EXECUTABLE AND NOT Protobuf_PROTOC_EXECUTABLE)
+  message(WARNING "Use of PROTOBUF_PROTOC_EXECUTABLE is deprecated. Please use Protobuf_PROTOC_EXECUTABLE instead.")
+  set(Protobuf_PROTOC_EXECUTABLE "${PROTOBUF_PROTOC_EXECUTABLE}")
+endif()
+
+if(CMAKE_CROSSCOMPILING)
+  find_program(Protobuf_PROTOC_EXECUTABLE protoc)
+else()
+  if(NOT TARGET protobuf::protoc)
+    message(FATAL_ERROR "A required protobuf target (protobuf::protoc) was not imported")
+  endif()
+  set(Protobuf_PROTOC_EXECUTABLE "$<TARGET_FILE:protobuf::protoc>")
+endif()
+
+set(PROTOBUF_PROTOC_EXECUTABLE "${Protobuf_PROTOC_EXECUTABLE}")
+
+message(STATUS "PROTOBUF_PROTOC_EXECUTABLE=${PROTOBUF_PROTOC_EXECUTABLE}")

exporters/otlp/CMakeLists.txt

@@ -32,7 +32,6 @@ target_link_libraries(opentelemetry_otlp_recordable
                       PUBLIC opentelemetry_metrics)
 
 if(WITH_OTLP_GRPC)
-  find_package(gRPC REQUIRED)
   if(NOT DEFINED grpc_lib_type)
     message(
       FATAL_ERROR "cmake/opentelemetry-proto.cmake should be included first")

install/cmake/CMakeLists.txt

@@ -232,6 +232,7 @@ if(grpc IN_LIST _THIRDPARTY_PACKAGE_LIST)
                "-DRE2_BUILD_TESTING=OFF"
                "-DgRPC_ZLIB_PROVIDER=package"
                "-DgRPC_PROTOBUF_PROVIDER=package"
+               "-DgRPC_PROTOBUF_PACKAGE_TYPE=CONFIG"
                "-DgRPC_ABSL_PROVIDER=package")
 
   add_dependencies(grpc-build zlib-install abseil-install protobuf-install)