feat: support data cache generation (#24)

* feat: memory pool supports cache function

* feat: cache data initialization

* test: test cases for data memory cache

* feat: data generator supports data cache model

* feat: add validity checks for cache-related parameters

* build: support sanitizer

* feat: enable mimalloc memory allocator by default

* feat: remove the force keyword from the database deletion statement

* test: fix test cases

* feat: use cache data by default

Author: YamingPei

CMakeLists.txt

@@ -7,11 +7,11 @@ list(APPEND CMAKE_PREFIX_PATH ${CMAKE_BINARY_DIR}/conan)
 set(CMAKE_FIND_PACKAGE_PREFER_CONFIG ON)
 
 # Options
-option(CMAKE_ENABLE_TEST "Enable testing" ON)
-option(CMAKE_ENABLE_COVERAGE "Enable code coverage" OFF)
-option(CMAKE_ENABLE_SANITIZER "Enable sanitizers" OFF)
-option(BUNDLE_JEMALLOC "Use jemalloc as allocator" ON)
-option(BUNDLE_MIMALLOC "Use mimalloc as allocator" OFF)
+option(TSGEN_ENABLE_TEST "Enable testing" ON)
+option(TSGEN_ENABLE_COVERAGE "Enable code coverage" OFF)
+option(TSGEN_ENABLE_SANITIZER "Enable sanitizers" OFF)
+option(TSGEN_BUNDLE_JEMALLOC "Use jemalloc as allocator" OFF)
+option(TSGEN_BUNDLE_MIMALLOC "Use mimalloc as allocator" ON)
 
 # Global settings
 set(CMAKE_CXX_STANDARD 17)
@@ -52,20 +52,33 @@ string(TOUPPER ${CMAKE_BUILD_TYPE} BUILD_TYPE_UPPER)
 string(TIMESTAMP TSGEN_BUILD_DATE "%Y-%m-%d %H:%M:%S %z")
 message(STATUS "Sys Info: ${TSGEN_BUILD_TARGET_OSTYPE}-${TSGEN_BUILD_TARGET_CPUTYPE} ${TSGEN_BUILD_DATE}")
 
+# Sanitizer
+if(TSGEN_ENABLE_SANITIZER AND CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
+  add_compile_options(-fsanitize=address)
+
+  if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.13")
+    add_link_options(-fsanitize=address)
+  else()
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address")
+  endif()
+
+  message(STATUS "AddressSanitizer enabled")
+endif()
+
 # Memory allocator settings
-if(BUNDLE_JEMALLOC AND BUNDLE_MIMALLOC)
+if(TSGEN_BUNDLE_JEMALLOC AND TSGEN_BUNDLE_MIMALLOC)
     message(FATAL_ERROR "Cannot use both jemalloc and mimalloc at the same time")
 endif()
 
 # Set link indicators
 set(ALLOCATOR_LINK_FLAGS "")
 set(ALLOCATOR_LIB "")
 
-if(BUNDLE_JEMALLOC)
+if(TSGEN_BUNDLE_JEMALLOC)
   message(STATUS "Using jemalloc memory allocator")
   find_package(jemalloc REQUIRED)
   set(ALLOCATOR_LIB jemalloc::jemalloc)
-elseif(BUNDLE_MIMALLOC)
+elseif(TSGEN_BUNDLE_MIMALLOC)
   message(STATUS "Using mimalloc memory allocator")
   find_package(mimalloc REQUIRED)
   set(ALLOCATOR_LIB mimalloc-static)
@@ -126,12 +139,12 @@ endif()
 message(STATUS "Git Info: ${TSGEN_BUILD_GIT}")
 
 # Test support
-if(CMAKE_ENABLE_TEST)
+if(TSGEN_ENABLE_TEST)
   enable_testing()
 endif()
 
 # Add main program and test directories
 add_subdirectory(src)
-# if(CMAKE_ENABLE_TEST)
+# if(TSGEN_ENABLE_TEST)
 #   add_subdirectory(test)
 # endif()

conf/mqtt-csv.yaml

@@ -36,8 +36,8 @@ schema:
       type: binary(24)
   generation:
     interlace: 1
-    per_table_rows: 100
-    per_batch_rows: 10000
+    rows_per_table: 100
+    rows_per_batch: 10000
 
 jobs:
   # MQTT publish job

conf/mqtt-gen-ev-history.yaml

@@ -75,8 +75,8 @@ schema:
       expr: ({"京AD12345", "京AF67890", "京AD98765", "京AF43210", "京AD56789", "京AF01234", "京AD23456", "京AF78901", "京AD34567", "京AF89012", "京AD45678", "京AD67890", "京AD78901", "京AD89012", "京AD90123", "京AD01234", "京AD13579", "京AD24680", "京AD35791", "京AD46802", "京AD57913", "京AF12345", "京AF23456", "京AF56789", "京AF90123", "京AF13579", "京AF24680", "京AF35791"})[tonumber(string.match(_table, "%d+"))]
   generation:
     interlace: 0
-    per_table_rows: 43200
-    per_batch_rows: 1000
+    rows_per_table: 43200
+    rows_per_batch: 1000
 
 jobs:
   # MQTT publish job

conf/mqtt-gen-ev-realtime.yaml

@@ -75,8 +75,8 @@ schema:
       expr: ({"京AD12345", "京AF67890", "京AD98765", "京AF43210", "京AD56789", "京AF01234", "京AD23456", "京AF78901", "京AD34567", "京AF89012", "京AD45678", "京AD67890", "京AD78901", "京AD89012", "京AD90123", "京AD01234", "京AD13579", "京AD24680", "京AD35791", "京AD46802", "京AD57913", "京AF12345", "京AF23456", "京AF56789", "京AF90123", "京AF13579", "京AF24680", "京AF35791"})[tonumber(string.match(_table, "%d+"))]
   generation:
     interlace: 1
-    per_table_rows: -1
-    per_batch_rows: 10000
+    rows_per_table: -1
+    rows_per_batch: 10000
 
 jobs:
   # MQTT publish job

conf/mqtt-gen.yaml

@@ -48,8 +48,8 @@ schema:
         - Austin
   generation:
     interlace: 1
-    per_table_rows: 100
-    per_batch_rows: 10000
+    rows_per_table: 100
+    rows_per_batch: 10000
 
 jobs:
   # MQTT publish job

conf/tdengine-csv.yaml

@@ -38,8 +38,8 @@ schema:
       type: binary(24)
   generation:
     interlace: 1
-    per_table_rows: 100
-    per_batch_rows: 10000
+    rows_per_table: 100
+    rows_per_batch: 10000
 
 jobs:
   # TDengine insert job

conf/tdengine-gen-2stb.yaml

@@ -46,8 +46,8 @@ schema:
         - Austin
   generation:
     interlace: 1
-    per_table_rows: 100
-    per_batch_rows: 10000
+    rows_per_table: 100
+    rows_per_batch: 10000
 
 ev_schema: &ev_schema
   name: ev
@@ -104,8 +104,8 @@ ev_schema: &ev_schema
         - Chengdu
   generation:
     interlace: 2
-    per_table_rows: 500
-    per_batch_rows: 5000
+    rows_per_table: 500
+    rows_per_batch: 5000
 
 concurrency: 2
 jobs:

conf/tdengine-gen-checkpoint.yaml

@@ -46,8 +46,8 @@ schema:
         - Austin
   generation:
     interlace: 1
-    per_table_rows: 100
-    per_batch_rows: 10000
+    rows_per_table: 100
+    rows_per_batch: 10000
 
 jobs:
   # TDengine insert job

conf/tdengine-gen-literal.yaml

@@ -46,8 +46,8 @@ schema:
         - Austin
   generation:
     interlace: 1
-    per_table_rows: 100
-    per_batch_rows: 10000
+    rows_per_table: 100
+    rows_per_batch: 10000
 
 jobs:
   # TDengine insert job

conf/tdengine-gen.yaml

@@ -46,8 +46,8 @@ schema:
         - Austin
   generation:
     interlace: 1
-    per_table_rows: 100
-    per_batch_rows: 10000
+    rows_per_table: 100
+    rows_per_batch: 10000
 
 jobs:
   # TDengine insert job