feat(trace):改进trace，使之支持module字段

Author: hevake

modules/trace/lib/sink.cpp

@@ -23,12 +23,12 @@
 #include <fcntl.h>
 
 #include <cstring>
-#include <iostream>
 #include <vector>
 #include <sstream>
 
 #include <sys/syscall.h>
 #include <tbox/base/defines.h>
+#include <tbox/base/log.h>
 #include <tbox/util/fs.h>
 #include <tbox/util/string.h>
 #include <tbox/util/scalable_integer.h>
@@ -39,7 +39,7 @@ namespace trace {
 void CommitRecordFunc(const char *name, const char *module, uint32_t line,
                       uint64_t end_timepoint_us, uint64_t duration_us)
 {
-    Sink::GetInstance().commitRecord(name, line, end_timepoint_us, duration_us);
+    Sink::GetInstance().commitRecord(name, module, line, end_timepoint_us, duration_us);
 }
 
 #define ENDLINE "\n"
@@ -72,12 +72,13 @@ bool Sink::setPathPrefix(const std::string &path_prefix)
     auto striped_path_prefix = util::string::Strip(path_prefix);
 
     if (striped_path_prefix.empty() || striped_path_prefix.back() == '/') {
-        std::cerr << "Err: path_prefix '"  << path_prefix << "' is invalid." << std::endl;
+        LogWarn("path_prefix '%s' is invalid.", path_prefix.c_str());
         return false;
     }
 
     dir_path_ = striped_path_prefix + '.' + GetLocalDateTimeStr() + '.' + std::to_string(::getpid());
     name_list_filename_ = dir_path_ + "/names.txt";
+    module_list_filename_ = dir_path_ + "/modules.txt";
     thread_list_filename_ = dir_path_ + "/threads.txt";
     CHECK_CLOSE_RESET_FD(curr_record_fd_);
 
@@ -96,7 +97,7 @@ bool Sink::enable()
         return true;
 
     if (dir_path_.empty()) {
-        std::cerr << "Warn: no path_prefix" << std::endl;
+        LogWarn("path_prefix is empty, can't enable.");
         return false;
     }
 
@@ -117,11 +118,10 @@ void Sink::disable()
         async_pipe_.cleanup();
         CHECK_CLOSE_RESET_FD(curr_record_fd_);
         buffer_.hasReadAll();
-        curr_record_filename_.clear();
     }
 }
 
-void Sink::commitRecord(const char *name, uint32_t line, uint64_t end_timepoint_us, uint64_t duration_us)
+void Sink::commitRecord(const char *name, const char *module, uint32_t line, uint64_t end_timepoint_us, uint64_t duration_us)
 {
     if (!is_enabled_)
         return;
@@ -131,12 +131,14 @@ void Sink::commitRecord(const char *name, uint32_t line, uint64_t end_timepoint_
         .end_ts_us = end_timepoint_us,
         .duration_us = duration_us,
         .line = line,
-        .name_size = ::strlen(name) + 1
+        .name_size = ::strlen(name) + 1,
+        .module_size = ::strlen(module) + 1
     };
 
     async_pipe_.appendLock();
     async_pipe_.appendLockless(&header, sizeof(header));
     async_pipe_.appendLockless(name, header.name_size);
+    async_pipe_.appendLockless(module, header.module_size);
     async_pipe_.appendUnlock();
 }
 
@@ -152,23 +154,27 @@ void Sink::onBackendRecvData(const void *data, size_t size)
         //! A: 因为直接指针引用会有对齐的问题。
 
         //! 如果长度不够，就跳过，等下一次
-        if ((header.name_size + sizeof(header)) > buffer_.readableSize())
+        if ((header.name_size + header.module_size + sizeof(header)) > buffer_.readableSize())
             break;
 
         buffer_.hasRead(sizeof(header));
 
-        onBackendRecvRecord(header, reinterpret_cast<const char*>(buffer_.readableBegin()));
-        buffer_.hasRead(header.name_size);
+        const char *name = reinterpret_cast<const char*>(buffer_.readableBegin());
+        const char *module = name + header.name_size;
+        onBackendRecvRecord(header, name, module);
+
+        buffer_.hasRead(header.name_size + header.module_size);
     }
 }
 
-void Sink::onBackendRecvRecord(const RecordHeader &record, const char *name)
+void Sink::onBackendRecvRecord(const RecordHeader &record, const char *name, const char *module)
 {
     if (!checkAndCreateRecordFile())
         return;
 
     auto thread_index = allocThreadIndex(record.thread_id);
     auto name_index = allocNameIndex(name, record.line);
+    auto module_index = allocModuleIndex(module);
     auto time_diff = record.end_ts_us - last_timepoint_us_;
 
     constexpr size_t kBufferSize = 40;
@@ -179,10 +185,11 @@ void Sink::onBackendRecvRecord(const RecordHeader &record, const char *name)
     data_size += util::DumpScalableInteger(record.duration_us, (buffer + data_size), (kBufferSize - data_size));
     data_size += util::DumpScalableInteger(thread_index, (buffer + data_size), (kBufferSize - data_size));
     data_size += util::DumpScalableInteger(name_index, (buffer + data_size), (kBufferSize - data_size));
+    data_size += util::DumpScalableInteger(module_index, (buffer + data_size), (kBufferSize - data_size));
 
     auto wsize = ::write(curr_record_fd_, buffer, data_size);
     if (wsize != static_cast<ssize_t>(data_size)) {
-        std::cerr << "Err: write file error" << std::endl;
+        LogErrno(errno, "write record file '%s' fail", curr_record_filename_.c_str());
         return;
     }
 
@@ -205,7 +212,7 @@ bool Sink::checkAndCreateRecordFile()
 
     //!检查并创建路径
     if (!util::fs::MakeDirectory(records_path, false)) {
-        std::cerr << "Err: create director " << records_path << " fail. error:" << errno << ',' << strerror(errno) << std::endl;
+        LogErrno(errno, "create directory '%s' fail", records_path);
         return false;
     }
 
@@ -228,7 +235,7 @@ bool Sink::checkAndCreateRecordFile()
 
     curr_record_fd_ = ::open(curr_record_filename_.c_str(), flags, S_IRUSR | S_IWUSR);
     if (curr_record_fd_ < 0) {
-        std::cerr << "Err: open file " << curr_record_filename_ << " fail. error:" << errno << ',' << strerror(errno) << std::endl;
+        LogErrno(errno, "open record file '%s' fail", curr_record_filename_.c_str());
         return false;
     }
 
@@ -265,6 +272,33 @@ Sink::Index Sink::allocNameIndex(const std::string &name, uint32_t line)
     return new_index;
 }
 
+Sink::Index Sink::allocModuleIndex(const std::string &module)
+{
+    //! 如果文件不存在了，则重写所有的名称列表
+    if (!util::fs::IsFileExist(module_list_filename_)) {
+        std::vector<std::string> module_vec;
+        module_vec.resize(module_to_index_map_.size());
+        for (auto &item : module_to_index_map_)
+            module_vec[item.second] = item.first;
+
+        std::ostringstream oss;
+        for (auto &module : module_vec)
+            oss << module << ENDLINE;
+
+        util::fs::WriteStringToTextFile(module_list_filename_, oss.str(), is_file_sync_enabled_);
+    }
+
+    auto iter = module_to_index_map_.find(module);
+    if (iter != module_to_index_map_.end())
+        return iter->second;
+
+    auto new_index = next_module_index_++;
+    module_to_index_map_[module] = new_index;
+
+    util::fs::AppendStringToTextFile(module_list_filename_, module + ENDLINE, is_file_sync_enabled_);
+    return new_index;
+}
+
 Sink::Index Sink::allocThreadIndex(long thread_id)
 {
     //! 如果文件不存在了，则重写所有的线程列表

modules/trace/lib/sink.h

@@ -44,6 +44,7 @@ class Sink {
      *       目录结构:
      *       .
      *       |-- names.txt    # 函数名列表
+     *       |-- modules.txt  # 模块名列表
      *       |-- threads.txt  # 线程名列表
      *       `-- records      # 记录文件目录，其下存在一个或多个记录文件
      *           `-- 20240530_041046.bin
@@ -76,7 +77,7 @@ class Sink {
      * \param end_ts        记录结束的时间点，单位: us
      * \param duration_us   记录持续时长，单位:us
      */
-    void commitRecord(const char *name, uint32_t line, uint64_t end_timepoint_us, uint64_t duration_us);
+    void commitRecord(const char *name, const char *module, uint32_t line, uint64_t end_timepoint_us, uint64_t duration_us);
 
   protected:
     ~Sink();
@@ -87,22 +88,25 @@ class Sink {
         uint64_t duration_us;
         uint32_t line;
         size_t name_size;
+        size_t module_size;
     };
 
     using Index = uint64_t;
 
     void onBackendRecvData(const void *data, size_t size);
-    void onBackendRecvRecord(const RecordHeader &record, const char *name);
+    void onBackendRecvRecord(const RecordHeader &record, const char *name, const char *module);
 
     bool checkAndCreateRecordFile();
 
     Index allocNameIndex(const std::string &name, uint32_t line);
+    Index allocModuleIndex(const std::string &module);
     Index allocThreadIndex(long thread_id);
 
   private:
     std::string dir_path_;
     size_t record_file_max_size_ = std::numeric_limits<size_t>::max();
     std::string name_list_filename_;
+    std::string module_list_filename_;
     std::string thread_list_filename_;
     bool is_file_sync_enabled_ = false;
 
@@ -120,6 +124,9 @@ class Sink {
     //! 名称编码
     std::map<std::string, Index> name_to_index_map_;
     uint32_t next_name_index_ = 0;
+    //! 模块编码
+    std::map<std::string, Index> module_to_index_map_;
+    uint32_t next_module_index_ = 0;
     //! 线程号编码
     std::map<long, Index> thread_to_index_map_;
     int next_thread_index_ = 0;

modules/trace/lib/sink_test.cpp

@@ -22,6 +22,8 @@
 #include <sys/syscall.h>
 #include <thread>
 
+#include <tbox/base/log.h>
+#include <tbox/base/log_output.h>
 #include <tbox/util/fs.h>
 #include <tbox/util/string.h>
 #include <tbox/util/timestamp.h>
@@ -38,44 +40,53 @@ TEST(Sink, Base) {
   ts.setPathPrefix(path_prefix);
 
   ts.enable();
-  ts.commitRecord("void hello()", 1, 100, 10);
-  ts.commitRecord("int world(int, std::string)", 2, 101, 1);
-  ts.commitRecord("void hello()", 1, 200, 10);
-  ts.commitRecord("bool bye(double)", 3, 201, 100);
+  ts.commitRecord("void hello()", "a", 1, 100, 10);
+  ts.commitRecord("int world(int, std::string)", "a", 2, 101, 1);
+  ts.commitRecord("void hello()", "b", 1, 200, 10);
+  ts.commitRecord("bool bye(double)", "b", 3, 201, 100);
   ts.disable();
 
   std::string path = ts.getDirPath();
-  std::string name_list_filename = path + "/name_list.txt";
-  std::string thread_list_filename = path + "/thread_list.txt";
+  std::string name_list_filename = path + "/names.txt";
+  std::string thread_list_filename = path + "/threads.txt";
+  std::string module_list_filename = path + "/modules.txt";
   std::string record_filename = ts.getCurrRecordFilename();
 
   ASSERT_TRUE(util::fs::IsFileExist(name_list_filename));
   ASSERT_TRUE(util::fs::IsFileExist(thread_list_filename));
+  ASSERT_TRUE(util::fs::IsFileExist(module_list_filename));
   ASSERT_TRUE(util::fs::IsFileExist(record_filename));
 
   {
     std::string name_list_content;
     ASSERT_TRUE(util::fs::ReadStringFromTextFile(name_list_filename, name_list_content));
-    std::string target_content = "void hello() at L1\r\nint world(int, std::string) at L2\r\nbool bye(double) at L3\r\n";
+    std::string target_content = "void hello() at L1\nint world(int, std::string) at L2\nbool bye(double) at L3\n";
     EXPECT_EQ(name_list_content, target_content);
   }
 
+  {
+    std::string module_list_content;
+    ASSERT_TRUE(util::fs::ReadStringFromTextFile(module_list_filename, module_list_content));
+    std::string target_content = "a\nb\n";
+    EXPECT_EQ(module_list_content, target_content);
+  }
+
   {
     std::string thread_list_content;
     ASSERT_TRUE(util::fs::ReadStringFromTextFile(thread_list_filename, thread_list_content));
-    std::string target_content = std::to_string(::syscall(SYS_gettid)) + "\r\n";
+    std::string target_content = std::to_string(::syscall(SYS_gettid)) + "\n";
     EXPECT_EQ(thread_list_content, target_content);
   }
 
   {
     std::string first_record_content;
     ASSERT_TRUE(util::fs::ReadBinaryFromFile(record_filename, first_record_content));
     auto first_record_content_hex = util::string::RawDataToHexStr(first_record_content.data(), first_record_content.size());
-    std::string target_content = "64 0a 00 00 01 01 00 01 63 0a 00 00 01 64 00 02";
+    std::string target_content = "64 0a 00 00 00 01 01 00 01 00 63 0a 00 00 01 01 64 00 02 01";
     EXPECT_EQ(first_record_content_hex, target_content);
   }
 
-  //util::fs::RemoveDirectory("/tmp/cpp-tbox-test");
+  util::fs::RemoveDirectory("/tmp/cpp-tbox-test");
 }
 
 TEST(Sink, MultiThread) {
@@ -87,7 +98,7 @@ TEST(Sink, MultiThread) {
 
   auto test_func = [&ts] (const std::string name) {
     for (int i = 0; i < 1000; ++i) {
-      ts.commitRecord(name.c_str(), 100, util::GetCurrentMicrosecondsFrom1970(), 10);
+      ts.commitRecord(name.c_str(), "A", 100, util::GetCurrentMicrosecondsFrom1970(), 10);
     }
   };
 
@@ -98,7 +109,7 @@ TEST(Sink, MultiThread) {
   ts.disable();
 
   //!TODO: 检查记录和条数是否有2000条
-  //util::fs::RemoveDirectory("/tmp/cpp-tbox-test");
+  util::fs::RemoveDirectory("/tmp/cpp-tbox-test");
 }
 
 }