Bump opentelemetry-cpp version to 1.9.1, change LogExporter/LogProcessor to post-1.6 LogRecordExporter and LogRecordProcessor (#294)

niande-xbox · web-flow · commit 20721e512355 · 2023-06-30T14:15:40.000-07:00
diff --git a/exporters/fluentd/CHANGELOG.md b/exporters/fluentd/CHANGELOG.md
@@ -13,6 +13,10 @@ Increment the:
 * MINOR version when you add functionality in a backwards compatible manner, and
 * PATCH version when you make backwards compatible bug fixes.
 
+## [2.0.0] 2023-06-30
+
+* [EXPORTER] OpenTelemetry SDK v1.9.1 compatibility. Migrate to ObservedTimestamp from Timestamp.
+
 ## [1.2.0] 2022-08-29
 
 * [EXPORTER] OpenTelemetry SDK v1.6.0 compatibility
diff --git a/exporters/fluentd/CMakeLists.txt b/exporters/fluentd/CMakeLists.txt
@@ -26,8 +26,8 @@ add_definitions(-DHAVE_CONSOLE_LOG)
 add_definitions(-DENABLE_LOGS_PREVIEW)
 
 if (MAIN_PROJECT)
-  find_package(opentelemetry-cpp QUIET)
-  if(opentelemetry_cpp_FOUND)
+  find_package(opentelemetry-cpp CONFIG QUIET)
+  if(opentelemetry-cpp_FOUND)
     message("Using external opentelemetry-cpp")
   else()
     include(cmake/opentelemetry-cpp.cmake)
@@ -52,6 +52,7 @@ else()
 endif()
 
 find_package(CURL REQUIRED)
+find_package(Threads REQUIRED)
 
 include_directories(include)
 # create fluentd trace exporter
diff --git a/exporters/fluentd/README.md b/exporters/fluentd/README.md
@@ -12,7 +12,7 @@ Refer to install instructions [INSTALL.md](../../INSTALL.md#building-as-standalo
 Modify step 2 to create `cmake` build configuration for compiling fluentd as below:
 
 ```console
-   $ cmake -DWITH_fluentd=ON ..
+   $ cmake ..
    -- The C compiler identification is GNU 9.3.0
    -- The CXX compiler identification is GNU 9.3.0
    ...
@@ -22,6 +22,18 @@ Modify step 2 to create `cmake` build configuration for compiling fluentd as bel
    $
 ```
 
+### VCPKG Integration
+
+If integrating with VCPKG, make sure the cmake invocation defines the additional CMAKE_TOOLCHAIN_FILE.
+
+For example:
+
+```console
+    $ .../opentelemetry-cpp-contrib2/exporters/fluentd$ ~/vcpkg/vcpkg install
+    $ .../opentelemetry-cpp-contrib2/exporters/fluentd$ cmake -D CMAKE_TOOLCHAIN_FILE=/home/niande/vcpkg/scripts/buildsystems/vcpkg.cmake
+    $ .../opentelemetry-cpp-contrib2/exporters/fluentd$ make
+```
+
 ### Bazel Install Instructions
 
 TODO
diff --git a/exporters/fluentd/cmake/opentelemetry-cpp.cmake b/exporters/fluentd/cmake/opentelemetry-cpp.cmake
@@ -1,5 +1,5 @@
 if("${opentelemetry-cpp-tag}" STREQUAL "")
-	set(opentelemetry-cpp-tag "v1.6.0")
+	set(opentelemetry-cpp-tag "v1.9.1")
 endif()
 function(target_create _target _lib)
   add_library(${_target} STATIC IMPORTED)
@@ -11,7 +11,7 @@ endfunction()
 function(build_opentelemetry)
   set(opentelemetry_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/opentelemetry-cpp")
   set(opentelemetry_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/opentelemetry-cpp")
-  set(opentelemetry_cpp_targets opentelemetry_trace opentelemetry_logs opentelemetry_http_client_curl )
+  set(opentelemetry_cpp_targets opentelemetry_trace opentelemetry_logs)
   set(opentelemetry_CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON
           -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
           -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
@@ -24,7 +24,6 @@ function(build_opentelemetry)
     ${opentelemetry_BINARY_DIR}/sdk/src/logs/libopentelemetry_logs.a
     ${opentelemetry_BINARY_DIR}/sdk/src/resource/libopentelemetry_resources.a
     ${opentelemetry_BINARY_DIR}/sdk/src/common/libopentelemetry_common.a
-    ${opentelemetry_BINARY_DIR}/ext/src/http/client/curl/libopentelemetry_http_client_curl.a
     ${CURL_LIBRARIES}
   )
 
@@ -35,9 +34,7 @@ function(build_opentelemetry)
 
   include_directories(SYSTEM ${opentelemetry_include_dir})
 
-  set(opentelemetry_deps opentelemetry_trace opentelemetry_logs opentelemetry_resources opentelemetry_common
-        opentelemetry_http_client_curl
-        ${CURL_LIBRARIES})
+  set(opentelemetry_deps opentelemetry_trace opentelemetry_logs opentelemetry_resources opentelemetry_common ${CURL_LIBRARIES})
 
    set(make_cmd ${CMAKE_COMMAND} --build <BINARY_DIR> --target
      ${opentelemetry_cpp_targets})
@@ -64,8 +61,6 @@ function(build_opentelemetry)
                   "sdk/src/resource/libopentelemetry_resources.a")
     target_create("opentelemetry_common"
                   "sdk/src/common/libopentelemetry_common.a")
-    target_create("opentelemetry_http_client_curl"
-                  "ext/src/http/client/curl/libopentelemetry_http_client_curl.a")
     add_library(opentelemetry::libopentelemetry INTERFACE IMPORTED)
     add_dependencies(opentelemetry::libopentelemetry opentelemetry-cpp)
     set_target_properties(
diff --git a/exporters/fluentd/example/CMakeLists.txt b/exporters/fluentd/example/CMakeLists.txt
@@ -7,7 +7,7 @@ target_link_libraries(foo_library_trace opentelemetry_exporter_fluentd_trace
 
 add_executable(fluentd_example_trace trace/main.cc)
 target_link_libraries(fluentd_example_trace ${CMAKE_THREAD_LIBS_INIT}
-                      foo_library_trace opentelemetry_trace)
+                      foo_library_trace opentelemetry-cpp::trace)
 
 add_library(foo_library_logs log/foo_library/foo_library.cc)
 
@@ -16,4 +16,4 @@ target_link_libraries(foo_library_logs opentelemetry_exporter_fluentd_logs
 
 add_executable(fluentd_example_logs log/main.cc)
 target_link_libraries(fluentd_example_logs ${CMAKE_THREAD_LIBS_INIT}
-                      foo_library_logs opentelemetry_logs)
+                      foo_library_logs opentelemetry-cpp::logs)
diff --git a/exporters/fluentd/example/log/foo_library/foo_library.cc b/exporters/fluentd/example/log/foo_library/foo_library.cc
@@ -14,20 +14,36 @@ nostd::shared_ptr<logs::Logger> get_logger() {
 }
 
 void f1() {
-  get_logger()->Log(opentelemetry::logs::Severity::kDebug, "logName2",
-                    {{"k1", "v1"}, {"k2", "v2"}});
+  auto log_record = get_logger()->CreateLogRecord();
+  log_record->SetSeverity(opentelemetry::logs::Severity::kDebug);
+  log_record->SetAttribute("k1", "v1");
+  log_record->SetAttribute("k2", "v2");
+  log_record->SetTimestamp(std::chrono::system_clock::now());
+  log_record->SetEventId(2, "logName2");
+
+  get_logger()->EmitLogRecord(std::move(log_record));
 }
 
 void f2() {
-  get_logger()->Log(opentelemetry::logs::Severity::kDebug, "logName3",
-                    "log body - 3");
+  auto log_record = get_logger()->CreateLogRecord();
+  log_record->SetSeverity(opentelemetry::logs::Severity::kDebug);
+  log_record->SetBody("log body - 3");
+  log_record->SetTimestamp(std::chrono::system_clock::now());
+  log_record->SetEventId(3, "logName3");
+
+  get_logger()->EmitLogRecord(std::move(log_record));
 
   f1();
   f1();
 }
 } // namespace
 
 void foo_library() {
-  get_logger()->Log(opentelemetry::logs::Severity::kDebug, "logName1",
-                    "log body - 1");
+  auto log_record = get_logger()->CreateLogRecord();
+  log_record->SetSeverity(opentelemetry::logs::Severity::kDebug);
+  log_record->SetBody("log body - 1");
+  log_record->SetTimestamp(std::chrono::system_clock::now());
+  log_record->SetEventId(1, "logName1");
+
+  get_logger()->EmitLogRecord(std::move(log_record));
 }
diff --git a/exporters/fluentd/example/log/main.cc b/exporters/fluentd/example/log/main.cc
@@ -4,7 +4,7 @@
 #include "opentelemetry/exporters/fluentd/log/fluentd_exporter.h"
 #include "opentelemetry/logs/provider.h"
 #include "opentelemetry/sdk/logs/logger_provider.h"
-#include "opentelemetry/sdk/logs/simple_log_processor.h"
+#include "opentelemetry/sdk/logs/simple_log_record_processor.h"
 
 // Using an exporter that simply dumps span data to stdout.
 #include "foo_library/foo_library.h"
@@ -20,10 +20,10 @@ namespace {
 void initLogger() {
   opentelemetry::exporter::fluentd::common::FluentdExporterOptions options;
   options.endpoint = "tcp://localhost:24222";
-  auto exporter = std::unique_ptr<sdk_logs::LogExporter>(
+  auto exporter = std::unique_ptr<sdk_logs::LogRecordExporter>(
       new opentelemetry::exporter::fluentd::logs::FluentdExporter(options));
-  auto processor = std::unique_ptr<sdk_logs::LogProcessor>(
-      new sdk_logs::SimpleLogProcessor(std::move(exporter)));
+  auto processor = std::unique_ptr<sdk_logs::LogRecordProcessor>(
+      new sdk_logs::SimpleLogRecordProcessor(std::move(exporter)));
 
   auto provider = std::shared_ptr<opentelemetry::logs::LoggerProvider>(
       new opentelemetry::sdk::logs::LoggerProvider(std::move(processor)));
diff --git a/exporters/fluentd/include/opentelemetry/exporters/fluentd/common/fluentd_common.h b/exporters/fluentd/include/opentelemetry/exporters/fluentd/common/fluentd_common.h
@@ -95,6 +95,32 @@ void inline PopulateAttribute(
   }
 }
 
+inline std::string AttributeValueToString(
+    const opentelemetry::common::AttributeValue &value) {
+  std::string result;
+  if (nostd::holds_alternative<bool>(value)) {
+    result = nostd::get<bool>(value) ? "true" : "false";
+  } else if (nostd::holds_alternative<int>(value)) {
+    result = std::to_string(nostd::get<int>(value));
+  } else if (nostd::holds_alternative<int64_t>(value)) {
+    result = std::to_string(nostd::get<int64_t>(value));
+  } else if (nostd::holds_alternative<unsigned int>(value)) {
+    result = std::to_string(nostd::get<unsigned int>(value));
+  } else if (nostd::holds_alternative<uint64_t>(value)) {
+    result = std::to_string(nostd::get<uint64_t>(value));
+  } else if (nostd::holds_alternative<double>(value)) {
+    result = std::to_string(nostd::get<double>(value));
+  } else if (nostd::holds_alternative<const char*>(value)) {
+    result = std::string(nostd::get<const char*>(value));
+  } else if (nostd::holds_alternative<opentelemetry::v1::nostd::string_view>(value)) {
+    result = std::string(nostd::get<opentelemetry::v1::nostd::string_view>(value).data());
+  } else {
+    LOG_WARN("[Fluentd Exporter] AttributeValueToString - "
+             " Nested attributes not supported - ignored");
+  }
+  return result;
+}
+
 // Ref. https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1
 enum class TransportFormat {
   kMessage,
diff --git a/exporters/fluentd/include/opentelemetry/exporters/fluentd/common/fluentd_fields.h b/exporters/fluentd/include/opentelemetry/exporters/fluentd/common/fluentd_fields.h
@@ -19,39 +19,40 @@
  * - links                        - OT Span links array
  */
 
-#  define FLUENT_FIELD_VERSION         "env_ver"               /* Event version           */
-#  define FLUENT_FIELD_TYPE            "env_type"              /* Event type              */
-#  define FLUENT_FIELD_NAME            "name"                  /* Event name              */
+#  define FLUENT_FIELD_VERSION              "env_ver"               /* Event version           */
+#  define FLUENT_FIELD_TYPE                 "env_type"              /* Event type              */
+#  define FLUENT_FIELD_NAME                 "name"                  /* Event name              */
 
 #ifndef HAVE_FIELD_TIME
 #define HAVE_FIELD_TIME
 #endif
-#  define FLUENT_FIELD_TIME            "env_time"              /* Event time at envelope  */
+#  define FLUENT_FIELD_TIME                 "env_time"              /* Event time at envelope  */
 
-#  define FLUENT_FIELD_TRACE_ID        "env_dt_traceId"        /* Trace Id                */
-#  define FLUENT_FIELD_SPAN_ID         "env_dt_spanId"         /* Span Id                 */
-#  define FLUENT_FIELD_SPAN_PARENTID   "parentId"              /* Span ParentId           */
-#  define FLUENT_FIELD_SPAN_KIND       "kind"                  /* Span Kind               */
-#  define FLUENT_FIELD_SPAN_LINKS      "links"                 /* Span Links array        */
+#  define FLUENT_FIELD_TRACE_ID             "env_dt_traceId"        /* Trace Id                */
+#  define FLUENT_FIELD_SPAN_ID              "env_dt_spanId"         /* Span Id                 */
+#  define FLUENT_FIELD_SPAN_PARENTID        "parentId"              /* Span ParentId           */
+#  define FLUENT_FIELD_SPAN_KIND            "kind"                  /* Span Kind               */
+#  define FLUENT_FIELD_SPAN_LINKS           "links"                 /* Span Links array        */
 
-#  define FLUENT_FIELD_PROPERTIES      "env_properties"
+#  define FLUENT_FIELD_PROPERTIES           "env_properties"
 
 /* Span option constants */
-#  define FLUENT_FIELD_STARTTIME       "startTime"             /* Operation start time    */
-#  define FLUENT_FIELD_ENDTTIME        "env_time"              /* Operation end time      */
-#  define FLUENT_FIELD_DURATION        "duration"              /* Operation duration      */
-#  define FLUENT_FIELD_STATUSCODE      "statusCode"            /* OT Span status code     */
-#  define FLUENT_FIELD_STATUSMESSAGE   "statusMessage"         /* OT Span status message  */
-#  define FLUENT_FIELD_SUCCESS         "success"               /* OT Span success         */
+#  define FLUENT_FIELD_STARTTIME            "startTime"             /* Operation start time    */
+#  define FLUENT_FIELD_ENDTTIME             "env_time"              /* Operation end time      */
+#  define FLUENT_FIELD_DURATION             "duration"              /* Operation duration      */
+#  define FLUENT_FIELD_STATUSCODE           "statusCode"            /* OT Span status code     */
+#  define FLUENT_FIELD_STATUSMESSAGE        "statusMessage"         /* OT Span status message  */
+#  define FLUENT_FIELD_SUCCESS              "success"               /* OT Span success         */
 
 /*Log option constants */
-# define FLUENT_FIELD_TIMESTAMP       "Timestamp"              /* Log timestamp  */
+# define FLUENT_FIELD_TIMESTAMP             "Timestamp"              /* Log timestamp  */
+# define FLUENT_FIELD_OBSERVEDTIMESTAMP     "ObservedTimestamp"              /* Log observed timestamp  */
 
 /* Value constants */
-#  define FLUENT_VALUE_SPAN            "Span"                  /* Event name for Span     */
-#  define FLUENT_VALUE_LOG             "Log"                  /* Event name for Log     */
+#  define FLUENT_VALUE_SPAN                 "Span"                  /* Event name for Span     */
+#  define FLUENT_VALUE_LOG                  "Log"                  /* Event name for Log     */
 
-#  define FLUENT_VALUE_SPAN_START      "SpanStart"             /* Span Start              */
-#  define FLUENT_VALUE_SPAN_END        "SpanEnd"               /* Span Start              */
+#  define FLUENT_VALUE_SPAN_START           "SpanStart"             /* Span Start              */
+#  define FLUENT_VALUE_SPAN_END             "SpanEnd"               /* Span Start              */
 
 /* clang-format on */
diff --git a/exporters/fluentd/include/opentelemetry/exporters/fluentd/log/fluentd_exporter.h b/exporters/fluentd/include/opentelemetry/exporters/fluentd/log/fluentd_exporter.h
@@ -9,7 +9,7 @@
 #include "opentelemetry/exporters/fluentd/trace/recordable.h"
 #include "opentelemetry/ext/http/common/url_parser.h"
 #include "opentelemetry/sdk/logs/exporter.h"
-#include "opentelemetry/sdk/logs/log_record.h"
+#include "opentelemetry/logs/log_record.h"
 
 #include <vector>
 
@@ -25,7 +25,7 @@ namespace fluentd_common = opentelemetry::exporter::fluentd::common;
 /**
  * The fluentd exporter exports span data in JSON format as expected by fluentd
  */
-class FluentdExporter final : public logs_sdk::LogExporter {
+class FluentdExporter final : public logs_sdk::LogRecordExporter {
 public:
   /**
    * Create a FluentdExporter using all default options.
diff --git a/exporters/fluentd/include/opentelemetry/exporters/fluentd/log/recordable.h b/exporters/fluentd/include/opentelemetry/exporters/fluentd/log/recordable.h
@@ -39,7 +39,7 @@ class Recordable final : public opentelemetry::sdk::logs::Recordable {
    * Set body field for this log.
    * @param message the body to set
    */
-  void SetBody(nostd::string_view message) noexcept override;
+  void SetBody(const opentelemetry::common::AttributeValue &message) noexcept override;
 
   /**
    * Set a resource for this log.
@@ -57,25 +57,32 @@ class Recordable final : public opentelemetry::sdk::logs::Recordable {
       nostd::string_view key,
       const opentelemetry::common::AttributeValue &value) noexcept override;
 
+  /**
+   * Set the Event Id.
+   * @param id The event id to set
+   * @param name Optional event name to set
+   */
+  void SetEventId(int64_t id, nostd::string_view name = {}) noexcept override;
+
   /**
    * Set trace id for this log.
    * @param trace_id the trace id to set
    */
-  void SetTraceId(opentelemetry::trace::TraceId trace_id) noexcept override;
+  void SetTraceId(const opentelemetry::trace::TraceId &trace_id) noexcept override;
 
   /**
    * Set span id for this log.
    * @param span_id the span id to set
    */
   virtual void
-  SetSpanId(opentelemetry::trace::SpanId span_id) noexcept override;
+  SetSpanId(const opentelemetry::trace::SpanId &span_id) noexcept override;
 
   /**
    * Inject a trace_flags  for this log.
    * @param trace_flags the span id to set
    */
   void SetTraceFlags(
-      opentelemetry::trace::TraceFlags trace_flags) noexcept override {
+      const opentelemetry::trace::TraceFlags &trace_flags) noexcept override {
   } // Not Supported
 
   /**
@@ -85,6 +92,13 @@ class Recordable final : public opentelemetry::sdk::logs::Recordable {
   void SetTimestamp(
       opentelemetry::common::SystemTimestamp timestamp) noexcept override;
 
+  /**
+   * Set the observed timestamp for this log.
+   * @param timestamp the timestamp to set
+   */
+  void SetObservedTimestamp(
+      opentelemetry::common::SystemTimestamp timestamp) noexcept override;
+
   /**
    * Set instrumentation_scope for this log.
    * @param instrumentation_scope the instrumentation scope to set
diff --git a/exporters/fluentd/src/log/fluentd_exporter.cc b/exporters/fluentd/src/log/fluentd_exporter.cc
@@ -5,6 +5,7 @@
 #include "opentelemetry/exporters/fluentd/log/fluentd_exporter.h"
 #include "opentelemetry/exporters/fluentd/log/recordable.h"
 #include "opentelemetry/ext/http/common/url_parser.h"
+#include "opentelemetry/sdk/logs/read_write_log_record.h"
 
 #include "opentelemetry/exporters/fluentd/common/fluentd_logging.h"
 
@@ -60,7 +61,7 @@ FluentdExporter::FluentdExporter()
  */
 std::unique_ptr<logs_sdk::Recordable>
 FluentdExporter::MakeRecordable() noexcept {
-  return std::unique_ptr<sdk::logs::Recordable>(new Recordable);
+  return std::unique_ptr<logs_sdk::Recordable>(new opentelemetry::exporter::fluentd::logs::Recordable());
 }
 
 /**
@@ -90,7 +91,8 @@ sdk::common::ExportResult FluentdExporter::Export(
         auto log = rec->Log();
         // Emit "log" as fluentd event
         json record = json::array();
-        record.push_back(log[FLUENT_FIELD_TIMESTAMP]);
+        // ObservedTimestamp is now set when Log/EmitLogRecord is invoked rather than Timestamp.
+        record.push_back(log[FLUENT_FIELD_OBSERVEDTIMESTAMP]);
         json fields = {};
         for (auto &kv : log.items()) {
           fields[kv.key()] = kv.value();
@@ -150,7 +152,7 @@ bool FluentdExporter::Send(std::vector<uint8_t> &packet) {
       return true;
     }
 
-    LOG_WARN("send failed, retrying %u ...", retryCount);
+    LOG_WARN("send failed, retrying %lu ...", retryCount);
     // Retry to connect and/or send
   }
 
diff --git a/exporters/fluentd/src/log/recordable.cc b/exporters/fluentd/src/log/recordable.cc
diff --git a/exporters/fluentd/test/log/fluentd_recordable_test.cc b/exporters/fluentd/test/log/fluentd_recordable_test.cc
diff --git a/exporters/fluentd/vcpkg.json b/exporters/fluentd/vcpkg.json
