From c434ae33116361b82911543ff2123f1223879bdc Mon Sep 17 00:00:00 2001 From: Thiago Padilha Date: Mon, 2 Sep 2024 10:22:28 -0300 Subject: [PATCH 01/11] processor_calyptia: new lua processor for logs/metrics/traces The new processor has a similar API to filter lua, with the following differences: - logs/metrics/traces are supported - it is now possible to modify logs metadata - when splitting logs, it is possible to also specify the timestamp/metadata of each split log. - global variables for the return codes were added, these can make scripts more readable. The variables are: - DROP (-1) - KEEP (0) - MODIFY (1) - MODIFY_KEEP_TIMESTAMP (2) - For metrics/traces, timestamp is not used (since timestamps are specified using internal fields) - For metrics, metadata is not used for now Signed-off-by: Thiago Padilha --- cmake/plugins_options.cmake | 1 + plugins/CMakeLists.txt | 1 + plugins/processor_calyptia/CMakeLists.txt | 25 + plugins/processor_calyptia/calyptia.c | 279 +++++++++ plugins/processor_calyptia/calyptia_defs.h | 25 + plugins/processor_calyptia/calyptia_logs.c | 84 +++ plugins/processor_calyptia/calyptia_logs.h | 9 + .../calyptia_logs_from_lua.c | 89 +++ .../calyptia_logs_from_lua.h | 9 + .../processor_calyptia/calyptia_logs_to_lua.c | 46 ++ .../processor_calyptia/calyptia_logs_to_lua.h | 9 + plugins/processor_calyptia/calyptia_metrics.c | 75 +++ plugins/processor_calyptia/calyptia_metrics.h | 11 + .../calyptia_metrics_from_lua.c | 564 ++++++++++++++++++ .../calyptia_metrics_from_lua.h | 10 + .../calyptia_metrics_to_lua.c | 254 ++++++++ .../calyptia_metrics_to_lua.h | 10 + plugins/processor_calyptia/calyptia_traces.c | 79 +++ plugins/processor_calyptia/calyptia_traces.h | 11 + .../calyptia_traces_from_lua.c | 303 ++++++++++ .../calyptia_traces_from_lua.h | 9 + .../calyptia_traces_to_lua.c | 290 +++++++++ .../calyptia_traces_to_lua.h | 9 + plugins/processor_calyptia/cfl_to_lua.c | 94 +++ plugins/processor_calyptia/cfl_to_lua.h | 15 + .../generate_lua_helpers.cmake | 33 + plugins/processor_calyptia/lua_to_cfl.c | 129 ++++ plugins/processor_calyptia/lua_to_cfl.h | 18 + .../processor_calyptia/processor_helpers.lua | 277 +++++++++ 29 files changed, 2768 insertions(+) create mode 100644 plugins/processor_calyptia/CMakeLists.txt create mode 100644 plugins/processor_calyptia/calyptia.c create mode 100644 plugins/processor_calyptia/calyptia_defs.h create mode 100644 plugins/processor_calyptia/calyptia_logs.c create mode 100644 plugins/processor_calyptia/calyptia_logs.h create mode 100644 plugins/processor_calyptia/calyptia_logs_from_lua.c create mode 100644 plugins/processor_calyptia/calyptia_logs_from_lua.h create mode 100644 plugins/processor_calyptia/calyptia_logs_to_lua.c create mode 100644 plugins/processor_calyptia/calyptia_logs_to_lua.h create mode 100644 plugins/processor_calyptia/calyptia_metrics.c create mode 100644 plugins/processor_calyptia/calyptia_metrics.h create mode 100644 plugins/processor_calyptia/calyptia_metrics_from_lua.c create mode 100644 plugins/processor_calyptia/calyptia_metrics_from_lua.h create mode 100644 plugins/processor_calyptia/calyptia_metrics_to_lua.c create mode 100644 plugins/processor_calyptia/calyptia_metrics_to_lua.h create mode 100644 plugins/processor_calyptia/calyptia_traces.c create mode 100644 plugins/processor_calyptia/calyptia_traces.h create mode 100644 plugins/processor_calyptia/calyptia_traces_from_lua.c create mode 100644 plugins/processor_calyptia/calyptia_traces_from_lua.h create mode 100644 plugins/processor_calyptia/calyptia_traces_to_lua.c create mode 100644 plugins/processor_calyptia/calyptia_traces_to_lua.h create mode 100644 plugins/processor_calyptia/cfl_to_lua.c create mode 100644 plugins/processor_calyptia/cfl_to_lua.h create mode 100644 plugins/processor_calyptia/generate_lua_helpers.cmake create mode 100644 plugins/processor_calyptia/lua_to_cfl.c create mode 100644 plugins/processor_calyptia/lua_to_cfl.h create mode 100644 plugins/processor_calyptia/processor_helpers.lua diff --git a/cmake/plugins_options.cmake b/cmake/plugins_options.cmake index 5a63bbc7d68..87efe946093 100644 --- a/cmake/plugins_options.cmake +++ b/cmake/plugins_options.cmake @@ -65,6 +65,7 @@ DEFINE_OPTION(FLB_IN_WINSTAT "Enable Windows Stat input plugin" # Processors # ========== DEFINE_OPTION(FLB_PROCESSOR_CONTENT_MODIFIER "Enable content modifier processor" ON) +DEFINE_OPTION(FLB_PROCESSOR_CALYPTIA "Enable calyptia core lua processor" ON) DEFINE_OPTION(FLB_PROCESSOR_LABELS "Enable metrics label manipulation processor" ON) DEFINE_OPTION(FLB_PROCESSOR_METRICS_SELECTOR "Enable metrics selector processor" ON) DEFINE_OPTION(FLB_PROCESSOR_SQL "Enable SQL processor" ON) diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index ce8cae64d97..eb6ca630ce2 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -283,6 +283,7 @@ REGISTER_IN_PLUGIN("in_random") # PROCESSORS # ========== REGISTER_PROCESSOR_PLUGIN("processor_content_modifier") +REGISTER_PROCESSOR_PLUGIN("processor_calyptia") REGISTER_PROCESSOR_PLUGIN("processor_labels") REGISTER_PROCESSOR_PLUGIN("processor_metrics_selector") REGISTER_PROCESSOR_PLUGIN("processor_sql") diff --git a/plugins/processor_calyptia/CMakeLists.txt b/plugins/processor_calyptia/CMakeLists.txt new file mode 100644 index 00000000000..d02c5fce96f --- /dev/null +++ b/plugins/processor_calyptia/CMakeLists.txt @@ -0,0 +1,25 @@ +set(LUA_HELPERS_SRC ${CMAKE_CURRENT_SOURCE_DIR}/processor_helpers.lua) +set(LUA_HELPERS_DST ${CMAKE_CURRENT_BINARY_DIR}/calyptia_processor_helpers.c) + +add_custom_command( + OUTPUT ${LUA_HELPERS_DST} + COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/generate_lua_helpers.cmake ${LUA_HELPERS_SRC} ${LUA_HELPERS_DST} + DEPENDS generate_lua_helpers.cmake ${LUA_HELPERS_SRC} + ) + +set(src + calyptia.c + calyptia_logs.c + calyptia_logs_from_lua.c + calyptia_logs_to_lua.c + calyptia_metrics.c + calyptia_metrics_from_lua.c + calyptia_metrics_to_lua.c + calyptia_traces.c + calyptia_traces_to_lua.c + calyptia_traces_from_lua.c + cfl_to_lua.c + lua_to_cfl.c + ${LUA_HELPERS_DST}) + +FLB_PLUGIN(processor_calyptia "${src}" "") diff --git a/plugins/processor_calyptia/calyptia.c b/plugins/processor_calyptia/calyptia.c new file mode 100644 index 00000000000..503af473a83 --- /dev/null +++ b/plugins/processor_calyptia/calyptia.c @@ -0,0 +1,279 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "calyptia_defs.h" +#include "calyptia_logs.h" +#include "calyptia_metrics.h" +#include "calyptia_traces.h" +#include "cfl_to_lua.h" + +static void calyptia_config_destroy(struct calyptia_context *ctx) +{ + if (!ctx) { + return; + } + + if (ctx->code) { + flb_sds_destroy(ctx->code); + ctx->code = NULL; + } + + if (ctx->script) { + flb_sds_destroy(ctx->script); + ctx->script = NULL; + } + + if (ctx->lua) { + flb_luajit_destroy(ctx->lua); + ctx->lua = NULL; + } + + flb_free(ctx); +} + +static struct calyptia_context * +calyptia_config_create(struct flb_processor_instance *ins, + struct flb_config *config) +{ + int ret; + int err; + char buf[PATH_MAX]; + const char *tmp = NULL; + const char *script = NULL; + struct stat st; + struct flb_luajit *lj; + (void) config; + struct calyptia_context *ctx; + + /* Allocate context */ + ctx = flb_calloc(1, sizeof(struct calyptia_context)); + if (!ctx) { + flb_errno(); + return NULL; + } + ret = flb_processor_instance_config_map_set(ins, (void *) ctx); + if (ret < 0) { + flb_errno(); + flb_plg_error(ins, "configuration error"); + flb_free(ctx); + return NULL; + } + + ctx->ins = ins; + + /* config: code */ + tmp = flb_processor_instance_get_property("code", ins); + if (tmp) { + ctx->code = flb_sds_create(tmp); + } else { + /* config: script */ + script = flb_processor_instance_get_property("script", ins); + if (!script) { + flb_plg_error(ins, "either \"script\" or \"code\" must be set"); + calyptia_config_destroy(ctx); + return NULL; + } + + /* Compose path */ + ret = stat(script, &st); + if (ret == -1 && errno == ENOENT) { + if (script[0] == '/') { + flb_plg_error(ins, "cannot access script '%s'", script); + calyptia_config_destroy(ctx); + return NULL; + } + + if (config->conf_path) { + snprintf(buf, sizeof(buf) - 1, "%s%s", config->conf_path, + script); + script = buf; + } + } + + /* Validate script path */ + ret = access(script, R_OK); + if (ret == -1) { + flb_plg_error(ins, "cannot access script '%s'", script); + calyptia_config_destroy(ctx); + return NULL; + } + + ctx->script = flb_sds_create(script); + if (!ctx->script) { + flb_plg_error(ins, "could not allocate string"); + calyptia_config_destroy(ctx); + return NULL; + } + } + + if (!ctx->call) { + flb_plg_error(ctx->ins, "\"call\" is not set"); + calyptia_config_destroy(ctx); + return NULL; + } + + /* Create LuaJIT state/vm */ + lj = flb_luajit_create(config); + if (!lj) { + calyptia_config_destroy(ctx); + return NULL; + } + + ctx->lua = lj; + + /* Load the lua helpers */ + if (flb_luajit_load_buffer(ctx->lua, calyptia_processor_lua_helpers, + strlen(calyptia_processor_lua_helpers), + "processor_helpers.lua")) { + calyptia_config_destroy(ctx); + return NULL; + } + + /* this is here to allow passing options to the lua helpers */ + lua_createtable(ctx->lua->state, 0, 1); + lua_pushboolean(ctx->lua->state, ctx->disable_warnings); + lua_setfield(ctx->lua->state, -2, "disable_warnings"); + lua_setglobal(ctx->lua->state, "LUA_HELPERS_OPTS"); + + /* execute the lua helpers script, we expect 3 helper functions as return + * value */ + err = lua_pcall(ctx->lua->state, 0, 3, 0); + if (err) { + flb_error("[luajit] invalid lua content, error=%d: %s", err, + lua_tostring(lj->state, -1)); + calyptia_config_destroy(ctx); + return NULL; + } + + /* push registry key for logs helper */ + lua_pushlightuserdata(ctx->lua->state, (void *) LUA_LOGS_HELPER_KEY); + /* push the logs helper function */ + lua_pushvalue(ctx->lua->state, -4); + /* store it in the registry */ + lua_settable(ctx->lua->state, LUA_REGISTRYINDEX); + + /* push registry key for metrics helper */ + lua_pushlightuserdata(ctx->lua->state, (void *) LUA_METRICS_HELPER_KEY); + /* push the metrics helper function */ + lua_pushvalue(ctx->lua->state, -3); + /* store it in the registry */ + lua_settable(ctx->lua->state, LUA_REGISTRYINDEX); + + /* push registry key for traces helper */ + lua_pushlightuserdata(ctx->lua->state, (void *) LUA_TRACES_HELPER_KEY); + /* push the traces helper function */ + lua_pushvalue(ctx->lua->state, -2); + /* store it in the registry */ + lua_settable(ctx->lua->state, LUA_REGISTRYINDEX); + /* pop the helpers */ + lua_pop(ctx->lua->state, 3); + + /* Load the lua script */ + if (ctx->code) { + if (flb_luajit_load_buffer(ctx->lua, ctx->code, flb_sds_len(ctx->code), + "processor.lua")) { + calyptia_config_destroy(ctx); + return NULL; + } + } else if (flb_luajit_load_script(ctx->lua, ctx->script)) { + calyptia_config_destroy(ctx); + return NULL; + } + + if (ctx->opts) { + push_variant(ctx->lua->state, ctx->opts); + } + + /* Execute the lua script */ + err = lua_pcall(ctx->lua->state, ctx->opts ? 1 : 0, 0, 0); + if (err) { + flb_error("[luajit] invalid lua content, error=%d: %s", err, + lua_tostring(lj->state, -1)); + calyptia_config_destroy(ctx); + return NULL; + } + + if (flb_lua_is_valid_func(ctx->lua->state, ctx->call) != FLB_TRUE) { + flb_plg_error(ctx->ins, "function %s is not found", ctx->call); + calyptia_config_destroy(ctx); + return NULL; + } + + return ctx; +} + +static int cb_init(struct flb_processor_instance *ins, + void *source_plugin_instance, int source_plugin_type, + struct flb_config *config) +{ + struct calyptia_context *ctx; + + ctx = calyptia_config_create(ins, config); + if (!ctx) { + return -1; + } + + flb_processor_instance_set_context(ins, ctx); + + return FLB_PROCESSOR_SUCCESS; +} + +static int cb_exit(struct flb_processor_instance *ins, void *data) +{ + struct calyptia_context *ctx; + + if (!ins) { + return FLB_PROCESSOR_SUCCESS; + } + + ctx = data; + if (ctx) { + calyptia_config_destroy(ctx); + } + + return FLB_PROCESSOR_SUCCESS; +} + +static struct flb_config_map config_map[] + = { { FLB_CONFIG_MAP_STR, "code", NULL, 0, FLB_FALSE, 0, + "String that contains the Lua script source code" }, + { FLB_CONFIG_MAP_STR, "script", NULL, 0, FLB_FALSE, 0, + "The path of lua script." }, + { FLB_CONFIG_MAP_STR, "call", NULL, 0, FLB_TRUE, + offsetof(struct calyptia_context, call), + "Lua function that will be called to process logs." }, + { FLB_CONFIG_MAP_BOOL, "disable_warnings", "false", 0, FLB_TRUE, + offsetof(struct calyptia_context, disable_warnings), + "Disable warnings from lua helpers." }, + { FLB_CONFIG_MAP_VARIANT, "opts", NULL, 0, FLB_TRUE, + offsetof(struct calyptia_context, opts), + "Arguments object passed to Lua script" }, + { 0 } }; + +struct flb_processor_plugin processor_calyptia_plugin + = { .name = "calyptia", + .description = "Use lua to process logs, metrics and traces", + .cb_init = cb_init, + .cb_process_logs = calyptia_process_logs, + .cb_process_metrics = calyptia_process_metrics, + .cb_process_traces = calyptia_process_traces, + .cb_exit = cb_exit, + .config_map = config_map, + .flags = 0 }; diff --git a/plugins/processor_calyptia/calyptia_defs.h b/plugins/processor_calyptia/calyptia_defs.h new file mode 100644 index 00000000000..64ea623a38f --- /dev/null +++ b/plugins/processor_calyptia/calyptia_defs.h @@ -0,0 +1,25 @@ +#ifndef FLB_CALYPTIA_DEFS_H +#define FLB_CALYPTIA_DEFS_H + +#include +#include +#include + +extern char calyptia_processor_lua_helpers[]; + +#define LUA_LOGS_HELPER_KEY (calyptia_processor_lua_helpers + 1) +#define LUA_METRICS_HELPER_KEY (calyptia_processor_lua_helpers + 2) +#define LUA_TRACES_HELPER_KEY (calyptia_processor_lua_helpers + 3) + +struct calyptia_context { + flb_sds_t code; /* lua script source code */ + flb_sds_t script; /* lua script path */ + flb_sds_t call; /* lua callback to process the event */ + struct flb_luajit *lua; /* state context */ + struct flb_processor_instance *ins; /* processor instance */ + bool disable_warnings; /* disable warnings from lua helpers */ + struct cfl_variant *opts; /* arbitrary object passed to lua script */ +}; + + +#endif diff --git a/plugins/processor_calyptia/calyptia_logs.c b/plugins/processor_calyptia/calyptia_logs.c new file mode 100644 index 00000000000..6d2fe12e5a3 --- /dev/null +++ b/plugins/processor_calyptia/calyptia_logs.c @@ -0,0 +1,84 @@ +#include +#include + +#include + +#include "calyptia_logs.h" +#include "calyptia_defs.h" +#include "calyptia_logs_to_lua.h" +#include "calyptia_logs_from_lua.h" + +static void clear_logs(struct flb_mp_chunk_cobj *chunk_cobj) +{ + struct cfl_list *head; + struct cfl_list *tmp; + struct flb_mp_chunk_record *record = NULL; + + cfl_list_foreach_safe(head, tmp, &chunk_cobj->records) + { + record = cfl_list_entry(head, struct flb_mp_chunk_record, _head); + if (record->cobj_metadata) { + cfl_object_destroy(record->cobj_metadata); + } + if (record->cobj_record) { + cfl_object_destroy(record->cobj_record); + } + cfl_list_del(&record->_head); + flb_free(record); + } +} + +int calyptia_process_logs(struct flb_processor_instance *ins, void *chunk_data, + const char *tag, int tag_len) +{ + struct calyptia_context *ctx; + struct flb_mp_chunk_cobj *chunk_cobj; + int ret; + int l_code; + + ctx = ins->context; + chunk_cobj = (struct flb_mp_chunk_cobj *) chunk_data; + + ret = FLB_PROCESSOR_SUCCESS; + /* push the lua helper */ + lua_pushlightuserdata(ctx->lua->state, LUA_LOGS_HELPER_KEY); + lua_gettable(ctx->lua->state, LUA_REGISTRYINDEX); + /* push the lua callback */ + lua_getglobal(ctx->lua->state, ctx->call); + /* push the tag */ + lua_pushlstring(ctx->lua->state, tag, tag_len); + if (calyptia_logs_to_lua(ctx->lua->state, chunk_data)) { + flb_plg_error(ctx->ins, "Failed to encode logs"); + return FLB_PROCESSOR_FAILURE; + } + + ret = lua_pcall(ctx->lua->state, 3, 3, 0); + if (ret != 0) { + flb_plg_error(ctx->ins, "error code %d: %s", ret, + lua_tostring(ctx->lua->state, -1)); + lua_pop(ctx->lua->state, 1); + return FLB_PROCESSOR_FAILURE; + } + + /* index -2 is the "ingest" object, for which handling will only be + * implemented in the future */ + l_code = (int) lua_tointeger(ctx->lua->state, -3); + if (l_code == -1) { + clear_logs(chunk_cobj); + } else if (l_code == 0) { + /* nothing to do */ + } else if (l_code != 1) { + flb_plg_error(ctx->ins, "invalid return code %d", l_code); + ret = FLB_PROCESSOR_FAILURE; + } else { + clear_logs(chunk_cobj); + if (calyptia_logs_from_lua(ins, ctx->lua->state, chunk_cobj)) { + flb_plg_error(ctx->ins, "Failed to decode logs from lua"); + ret = FLB_PROCESSOR_FAILURE; + } + } + + /* clear lua stack */ + lua_settop(ctx->lua->state, 0); + return ret; +} diff --git a/plugins/processor_calyptia/calyptia_logs.h b/plugins/processor_calyptia/calyptia_logs.h new file mode 100644 index 00000000000..7172f8c8021 --- /dev/null +++ b/plugins/processor_calyptia/calyptia_logs.h @@ -0,0 +1,9 @@ +#ifndef FLB_CALYPTIA_LOGS_H +#define FLB_CALYPTIA_LOGS_H + +#include + +int calyptia_process_logs(struct flb_processor_instance *ins, void *chunk_data, + const char *tag, int tag_len); + +#endif diff --git a/plugins/processor_calyptia/calyptia_logs_from_lua.c b/plugins/processor_calyptia/calyptia_logs_from_lua.c new file mode 100644 index 00000000000..165772e226a --- /dev/null +++ b/plugins/processor_calyptia/calyptia_logs_from_lua.c @@ -0,0 +1,89 @@ +#include + +#include "calyptia_logs_from_lua.h" +#include "lua_to_cfl.h" + +int calyptia_logs_from_lua(struct flb_processor_instance *ins, lua_State *L, + struct flb_mp_chunk_cobj *chunk_cobj) +{ + int logs_length, metadata_length, timestamps_length; + struct flb_mp_chunk_record *record; + + if (lua_type(L, -1) != LUA_TTABLE) { + flb_plg_error(ins, "expected events object"); + return -1; + } + + lua_getfield(L, -1, "logs"); + if (lua_type(L, -1) != LUA_TTABLE) { + flb_plg_error(ins, "expected logs object"); + return -1; + } + + lua_getfield(L, -2, "metadata"); + if (lua_type(L, -1) != LUA_TTABLE) { + flb_plg_error(ins, "expected metadata object"); + return -1; + } + + lua_getfield(L, -3, "timestamps"); + if (lua_type(L, -1) != LUA_TTABLE) { + flb_plg_error(ins, "expected timestamps object"); + return -1; + } + + logs_length = lua_objlen(L, -3); + metadata_length = lua_objlen(L, -2); + timestamps_length = lua_objlen(L, -1); + + if (logs_length != metadata_length || logs_length != timestamps_length) { + flb_plg_error(ins, "logs, metadata, and timestamps must have the same length"); + return -1; + } + + for (int i = 1; i <= logs_length; i++) { + record = flb_mp_chunk_record_create(chunk_cobj); + if (!record) { + flb_plg_error(ins, "failed to create record"); + return -1; + } + + record->cobj_record = cfl_object_create(); + if (!record->cobj_record) { + flb_plg_error(ins, "failed to create record object"); + return -1; + } + + record->cobj_metadata = cfl_object_create(); + if (!record->cobj_metadata) { + flb_plg_error(ins, "failed to create metadata object"); + return -1; + } + + memset(&record->event, 0, sizeof(record->event)); + + /* get the timestamp */ + lua_rawgeti(L, -1, i); + flb_time_from_double(&record->event.timestamp, lua_to_double(L, -1)); + lua_pop(L, 1); + + /* get the metadata */ + lua_rawgeti(L, -2, i); + record->cobj_metadata->variant = lua_to_variant(L, -1); + record->cobj_metadata->type = CFL_OBJECT_VARIANT; + lua_pop(L, 1); + + /* get the log */ + lua_rawgeti(L, -3, i); + record->cobj_record->variant = lua_to_variant(L, -1); + record->cobj_record->type = CFL_OBJECT_VARIANT; + lua_pop(L, 1); + + cfl_list_add(&record->_head, &chunk_cobj->records); + } + + /* pop all */ + lua_pop(L, 3); + + return 0; +} diff --git a/plugins/processor_calyptia/calyptia_logs_from_lua.h b/plugins/processor_calyptia/calyptia_logs_from_lua.h new file mode 100644 index 00000000000..be3f5a9dd2a --- /dev/null +++ b/plugins/processor_calyptia/calyptia_logs_from_lua.h @@ -0,0 +1,9 @@ +#ifndef FLB_CALYPTIA_METRICS_FROM_LUA +#define FLB_CALYPTIA_METRICS_FROM_LUA + +#include +#include + +int calyptia_logs_from_lua(struct flb_processor_instance *ins, lua_State *L, struct flb_mp_chunk_cobj *chunk_cobj); + +#endif diff --git a/plugins/processor_calyptia/calyptia_logs_to_lua.c b/plugins/processor_calyptia/calyptia_logs_to_lua.c new file mode 100644 index 00000000000..a0fdc380986 --- /dev/null +++ b/plugins/processor_calyptia/calyptia_logs_to_lua.c @@ -0,0 +1,46 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "calyptia_logs_to_lua.h" +#include "cfl_to_lua.h" + +int calyptia_logs_to_lua(lua_State *L, struct flb_mp_chunk_cobj *chunk_cobj) +{ + double ts; + struct flb_mp_chunk_record *record; + + /* top-level table */ + lua_createtable(L, chunk_cobj->total_records, 0); + /* one array for records */ + lua_createtable(L, chunk_cobj->total_records, 0); + /* one array for metadata */ + lua_createtable(L, chunk_cobj->total_records, 0); + /* one array for timestamps */ + lua_createtable(L, chunk_cobj->total_records, 0); + + while (flb_mp_chunk_cobj_record_next(chunk_cobj, &record) == FLB_MP_CHUNK_RECORD_OK) { + push_variant(L, record->cobj_record->variant); + lua_rawseti(L, -4, lua_objlen(L, -4) + 1); + + push_variant(L, record->cobj_metadata->variant); + lua_rawseti(L, -3, lua_objlen(L, -3) + 1); + + ts = flb_time_to_double(&record->event.timestamp); + lua_pushnumber(L, ts); + lua_rawseti(L, -2, lua_objlen(L, -2) + 1); + } + + lua_setfield(L, -4, "timestamps"); + lua_setfield(L, -3, "metadata"); + lua_setfield(L, -2, "logs"); + + return 0; +} diff --git a/plugins/processor_calyptia/calyptia_logs_to_lua.h b/plugins/processor_calyptia/calyptia_logs_to_lua.h new file mode 100644 index 00000000000..98041f44dab --- /dev/null +++ b/plugins/processor_calyptia/calyptia_logs_to_lua.h @@ -0,0 +1,9 @@ +#ifndef FLB_CALYPTIA_METRICS_TO_LUA_H +#define FLB_CALYPTIA_METRICS_TO_LUA_H + +#include +#include + +int calyptia_logs_to_lua(lua_State *L, struct flb_mp_chunk_cobj *chunk_cobj); + +#endif diff --git a/plugins/processor_calyptia/calyptia_metrics.c b/plugins/processor_calyptia/calyptia_metrics.c new file mode 100644 index 00000000000..9b91969d01b --- /dev/null +++ b/plugins/processor_calyptia/calyptia_metrics.c @@ -0,0 +1,75 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "calyptia_metrics_to_lua.h" +#include "calyptia_metrics_from_lua.h" +#include "calyptia_defs.h" + +int calyptia_process_metrics(struct flb_processor_instance *ins, + struct cmt *metrics_context, + struct cmt **out_context, const char *tag, + int tag_len) +{ + struct calyptia_context *ctx; + int ret; + int l_code; + + ret = FLB_PROCESSOR_SUCCESS; + ctx = ins->context; + /* push the lua helper */ + lua_pushlightuserdata(ctx->lua->state, LUA_METRICS_HELPER_KEY); + lua_gettable(ctx->lua->state, LUA_REGISTRYINDEX); + /* push the lua callback */ + lua_getglobal(ctx->lua->state, ctx->call); + /* push the tag */ + lua_pushlstring(ctx->lua->state, tag, tag_len); + + if (calyptia_metrics_to_lua(ctx->lua->state, metrics_context)) { + flb_plg_error(ctx->ins, "Failed to encode metrics"); + return FLB_PROCESSOR_FAILURE; + } + + ret = lua_pcall(ctx->lua->state, 3, 3, 0); + if (ret != 0) { + flb_plg_error(ctx->ins, "error code %d: %s", ret, + lua_tostring(ctx->lua->state, -1)); + lua_pop(ctx->lua->state, 1); + return FLB_PROCESSOR_FAILURE; + } + + /* index -2 is the "ingest" object, for which handling will only be + * implemented in the future */ + l_code = (int) lua_tointeger(ctx->lua->state, -3); + if (l_code == -1) { + *out_context = cmt_create(); + } else if (l_code == 0) { + /* don't touch the metrics */ + *out_context = metrics_context; + } else if (l_code != 1) { + flb_plg_error(ctx->ins, "invalid return code %d", l_code); + ret = FLB_PROCESSOR_FAILURE; + } else { + struct cmt *new_metrics = cmt_create(); + if (calyptia_metrics_from_lua(ins, ctx->lua->state, new_metrics)) { + cmt_destroy(new_metrics); + flb_plg_error(ctx->ins, "Failed to decode metrics from lua"); + ret = FLB_PROCESSOR_FAILURE; + } else { + *out_context = new_metrics; + } + } + + /* clear lua stack */ + lua_settop(ctx->lua->state, 0); + return ret; +} diff --git a/plugins/processor_calyptia/calyptia_metrics.h b/plugins/processor_calyptia/calyptia_metrics.h new file mode 100644 index 00000000000..ad8098c8a7f --- /dev/null +++ b/plugins/processor_calyptia/calyptia_metrics.h @@ -0,0 +1,11 @@ +#ifndef FLB_CALYPTIA_METRICS_H +#define FLB_CALYPTIA_METRICS_H + +#include + +int calyptia_process_metrics(struct flb_processor_instance *ins, + struct cmt *metrics_context, + struct cmt **out_context, const char *tag, + int tag_len); + +#endif diff --git a/plugins/processor_calyptia/calyptia_metrics_from_lua.c b/plugins/processor_calyptia/calyptia_metrics_from_lua.c new file mode 100644 index 00000000000..411b8bca62f --- /dev/null +++ b/plugins/processor_calyptia/calyptia_metrics_from_lua.c @@ -0,0 +1,564 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "calyptia_metrics_from_lua.h" +#include "lua_to_cfl.h" + +struct metrics_header { + char fqname_buf[0xfff]; + char *ns; + char *subsystem; + char *name; +}; + +static void free_labels(char **labels, size_t label_count) +{ + if (!labels) { + return; + } + for (int i = 0; i < label_count; i++) { + if (labels[i]) { + free(labels[i]); + } + } + free(labels); +} + +static void split_fqname(const char *fqname, struct metrics_header *header) +{ + strcpy(header->fqname_buf, fqname); + header->ns = header->fqname_buf; + header->subsystem = strchr(header->fqname_buf, '_'); + if (!header->subsystem) { + header->name = header->fqname_buf; + header->ns = ""; + } else { + *header->subsystem = 0; /* split */ + header->subsystem++; + header->name = strchr(header->subsystem, '_'); + if (!header->name) { + header->name = header->subsystem; + header->subsystem = ""; + } else { + *header->name = 0; /* split */ + header->name++; + } + } +} + +static int assign_label(char **keys, size_t key_count, char **values, + const char *key, const char *value) +{ + for (size_t i = 0; i < key_count; i++) { + if (!strcmp(keys[i], key)) { + values[i] = strdup(value); + if (!values[i]) { + return -1; + } + return 0; + } + } + return -1; +} + +static char **lua_to_labels(struct flb_processor_instance *ins, lua_State *L, + char **label_keys, size_t label_count) +{ + if (lua_type(L, -1) != LUA_TTABLE) { + return NULL; + } + + char **labels = calloc(label_count, sizeof(char *)); + if (!labels) { + flb_plg_error(ins, "could not allocate memory for labels"); + return NULL; + } + + lua_pushnil(L); // first key + while (lua_next(L, -2) != 0) { + if (assign_label(label_keys, label_count, labels, lua_tostring(L, -2), + lua_tostring(L, -1))) { + flb_plg_error(ins, "could not assign label %s:%s", + lua_tostring(L, -2), lua_tostring(L, -1)); + lua_pop(L, 2); /* remove key/value */ + goto err; + } + /* removes 'value'; keeps 'key' for next iteration */ + lua_pop(L, 1); + } + + return labels; + +err: + free_labels(labels, label_count); + return NULL; +} + +static int double_cmp(const void *a, const void *b) +{ + double x = *(double *) a; + double y = *(double *) b; + + if (x < y) { + return -1; + } else if (x > y) { + return 1; + } else { + return 0; + } +} + +static double *lua_to_quantile_values(struct flb_processor_instance *ins, + lua_State *L, double *quantile_keys, + int count) +{ + double *quantile_values = calloc(count, sizeof(*quantile_values)); + if (!quantile_values) { + flb_plg_error(ins, "could not allocate memory for quantile values"); + return NULL; + } + + for (int i = 0; i < count; i++) { + lua_pushnumber(L, quantile_keys[i]); + lua_gettable(L, -2); + quantile_values[i] = lua_to_double(L, -1); + lua_pop(L, 1); + } + + return quantile_values; +} + +static uint64_t *lua_to_bucket_values(struct flb_processor_instance *ins, + lua_State *L, double *bucket_keys, + int count) +{ + uint64_t *values = calloc(count, sizeof(*values)); + if (!values) { + flb_plg_error(ins, "could not allocate memory for bucket values"); + return NULL; + } + + for (int i = 0; i < count; i++) { + lua_pushnumber(L, bucket_keys[i]); + lua_gettable(L, -2); + values[i] = lua_to_uint(L); + lua_pop(L, 1); + } + + return values; +} + +static double *lua_to_quantiles_buckets(struct flb_processor_instance *ins, + lua_State *L, int *count) +{ + double *keys; + *count = 0; + if (lua_type(L, -1) != LUA_TTABLE) { + return NULL; + } + + // assumes a quantiles or buckets table is at the top of the stack + lua_pushnil(L); // first key + while (lua_next(L, -2) != 0) { + lua_pop(L, 1); + *count += 1; + } + + keys = calloc(*count, sizeof(*keys)); + if (!keys) { + flb_plg_error(ins, "could not allocate memory for quantiles/buckets"); + return NULL; + } + + lua_pushnil(L); // first key + int i = 0; + while (lua_next(L, -2) != 0) { + keys[i] = lua_to_double(L, -2); + i++; + // removes 'value'; keeps 'key' for next iteration + lua_pop(L, 1); + } + + qsort(keys, *count, sizeof(*keys), double_cmp); + + return keys; +} + +static double *lua_to_quantile_bucket_keys(struct flb_processor_instance *ins, + lua_State *L, const char *kind, + int *count) +{ + int sample_count; + double *keys; + + *count = 0; + if (lua_type(L, -2) != LUA_TTABLE) { + flb_plg_error(ins, "expected metric to be a table"); + return NULL; + } + + lua_getfield(L, -2, "metrics"); + sample_count = lua_objlen(L, -1); + int found = 0; + + /* find the first sample that has quantiles */ + for (int i = 1; i <= sample_count; i++) { + lua_rawgeti(L, -1, i); + lua_getfield(L, -1, kind); + if (lua_type(L, -1) == LUA_TTABLE) { + found = 1; + break; + } + lua_pop(L, 2); /* pop "quantiles" table and the sample */ + } + + if (!found) { + lua_pop(L, 1); /* pop "metrics" */ + return NULL; + } + + keys = lua_to_quantiles_buckets(ins, L, count); + + lua_pop(L, + 3); /* pop "quantiles"/"buckets" table, metric and metrics array */ + + return keys; +} + +static char **append_label(char **labels, size_t *labels_size, + size_t *label_index, const char *label) +{ + for (size_t i = 0; i < *label_index; i++) { + if (!strcmp(labels[i], label)) { + /* don't do anything if the label is already in the array */ + return labels; + } + } + + if (*label_index == *labels_size) { + if (!*labels_size) { + *labels_size = 8; + } else { + *labels_size *= 2; + } + labels = realloc(labels, *labels_size * sizeof(char *)); + if (!labels) { + return NULL; + } + } + + labels[*label_index] = strdup(label); + if (!labels[*label_index]) { + return NULL; + } + + *label_index += 1; + return labels; +} + +static char **lua_to_label_keys(struct flb_processor_instance *ins, + lua_State *L, int *label_count) +{ + int sample_count; + char **label_keys; + size_t label_index; + size_t labels_size; + + *label_count = 0; + if (lua_type(L, -1) != LUA_TTABLE) { + flb_plg_error(ins, "expected metric to be a table"); + return NULL; + } + + lua_getfield(L, -1, "metrics"); + sample_count = lua_objlen(L, -1); + if (lua_type(L, -1) != LUA_TTABLE || !sample_count) { + flb_plg_error(ins, "samples should be in a \"metrics\" array with at " + "least one element"); + return NULL; + } + + label_keys = NULL; + labels_size = 0; + label_index = 0; + + for (int i = 1; i <= sample_count; i++) { + lua_rawgeti(L, -1, i); + if (lua_type(L, -1) != LUA_TTABLE) { + free_labels(label_keys, label_index); + flb_plg_error(ins, "expected sample to be a table"); + return NULL; + } + + lua_getfield(L, -1, "labels"); + if (lua_type(L, -1) == LUA_TTABLE) { + lua_pushnil(L); /* first key */ + while (lua_next(L, -2) != 0) { + label_keys = append_label(label_keys, &labels_size, + &label_index, lua_tostring(L, -2)); + lua_pop(L, 1); /* pop value, keep key for next iteration */ + } + } + + lua_pop(L, 2); /* pop "labels" table and the sample */ + } + lua_pop(L, 1); /* pop "metrics" */ + + *label_count = label_index; + + if (!label_index) { + return NULL; + } + + return label_keys; +} + +/* This is a big function because if I had split into multiple utility functions + * to conver the specific metric types, there would be a lot of repeated code + * for the common parts. Another option would be to use big macros to reduce + * repetition, but this is the more maintainable/debuggable option */ +int calyptia_metrics_from_lua(struct flb_processor_instance *ins, lua_State *L, + struct cmt *cmt) +{ + struct cmt_counter *counter; + struct cmt_gauge *gauge; + struct cmt_summary *summary; + struct cmt_histogram_buckets *cmt_buckets; + struct cmt_histogram *histogram; + struct cmt_untyped *untyped; + struct metrics_header header; + double *quantiles; + double *quantile_values; + double *buckets; + uint64_t *bucket_values; + double sum; + double value; + uint64_t count; + int quantile_count; + int bucket_count; + int type; + int sample_count; + int metric_count; + const char *help; + char **label_keys; + char **label_vals; + int label_count; + uint64_t timestamp; + + if (lua_type(L, -1) != LUA_TTABLE) { + flb_plg_error(ins, "expected metrics array"); + return -1; + } + + metric_count = lua_objlen(L, -1); + + for (int i = 1; i <= metric_count; i++) { + lua_rawgeti(L, -1, i); + + label_keys = lua_to_label_keys(ins, L, &label_count); + timestamp = 0; + + lua_getfield(L, -1, "type"); + if (lua_type(L, -1) != LUA_TSTRING) { + flb_plg_error(ins, "metric type must be a string"); + return -1; + } + + const char *metric_type = lua_tostring(L, -1); + + if (!strcasecmp(metric_type, "COUNTER")) { + type = CMT_COUNTER; + } else if (!strcasecmp(metric_type, "GAUGE")) { + type = CMT_GAUGE; + } else if (!strcasecmp(metric_type, "SUMMARY")) { + type = CMT_SUMMARY; + } else if (!strcasecmp(metric_type, "HISTOGRAM")) { + type = CMT_HISTOGRAM; + } else if (!strcasecmp(metric_type, "UNTYPED")) { + type = CMT_UNTYPED; + } else { + cmt_destroy(cmt); + flb_plg_error(ins, "invalid metric type: \"%s\"", metric_type); + return -1; + } + + lua_pop(L, 1); /* pop "type" */ + + lua_getfield(L, -1, "name"); + if (lua_type(L, -1) != LUA_TSTRING) { + flb_plg_error(ins, "metric name must be a string"); + return -1; + } + + const char *fqname = lua_tostring(L, -1); + split_fqname(fqname, &header); + lua_pop(L, 1); /* pop "name" */ + + lua_getfield(L, -1, "help"); + if (lua_type(L, -1) != LUA_TSTRING) { + flb_plg_error(ins, "metric help must be a string"); + return -1; + } + help = lua_tostring(L, -1); + + switch (type) { + case CMT_COUNTER: + counter = cmt_counter_create(cmt, header.ns, header.subsystem, + header.name, (char *) help, + label_count, label_keys); + break; + case CMT_GAUGE: + gauge = cmt_gauge_create(cmt, header.ns, header.subsystem, + header.name, (char *) help, label_count, + label_keys); + break; + case CMT_SUMMARY: + quantiles = lua_to_quantile_bucket_keys(ins, L, "quantiles", + &quantile_count); + summary = cmt_summary_create( + cmt, header.ns, header.subsystem, header.name, (char *) help, + quantile_count, quantiles, label_count, label_keys); + break; + case CMT_HISTOGRAM: + buckets + = lua_to_quantile_bucket_keys(ins, L, "buckets", &bucket_count); + cmt_buckets + = cmt_histogram_buckets_create_size(buckets, bucket_count - 1); + histogram = cmt_histogram_create( + cmt, header.ns, header.subsystem, header.name, (char *) help, + cmt_buckets, label_count, label_keys); + break; + case CMT_UNTYPED: + untyped = cmt_untyped_create(cmt, header.ns, header.subsystem, + header.name, (char *) help, + label_count, label_keys); + break; + } + + /* pop "help" only after creating the metric instance, as it was already + * copied */ + lua_pop(L, 1); + + /* load samples */ + lua_getfield(L, -1, "metrics"); + sample_count = lua_objlen(L, -1); + if (sample_count == 0) { + flb_plg_error(ins, "no samples found for metric \"%s\"", fqname); + return -1; + } + + for (int j = 1; j <= sample_count; j++) { + label_vals = NULL; + + /* get sample */ + lua_rawgeti(L, -1, j); + if (lua_type(L, -1) != LUA_TTABLE) { + flb_plg_error(ins, "expected sample to be a table"); + return -1; + } + + lua_getfield(L, -1, "labels"); + label_vals = lua_to_labels(ins, L, label_keys, label_count); + lua_pop(L, 1); /* pop labels */ + + lua_getfield(L, -1, "timestamp"); + timestamp = lua_to_uint(L); + lua_pop(L, 1); /* pop timestamp */ + + if (type == CMT_SUMMARY || type == CMT_HISTOGRAM) { + lua_getfield(L, -1, "sum"); + sum = lua_to_double(L, -1); + lua_pop(L, 1); /* pop sum */ + + lua_getfield(L, -1, "count"); + count = lua_to_uint(L); + lua_pop(L, 1); /* pop count */ + } + + if (type == CMT_SUMMARY) { + + lua_getfield(L, -1, "quantiles"); + quantile_values + = lua_to_quantile_values(ins, L, quantiles, quantile_count); + lua_pop(L, 1); /* pop quantiles */ + + if (cmt_summary_set_default( + summary, timestamp, quantile_values, sum, count, + label_vals ? label_count : 0, label_vals)) { + return -1; + } + + } else if (type == CMT_HISTOGRAM) { + + lua_getfield(L, -1, "buckets"); + bucket_values + = lua_to_bucket_values(ins, L, buckets, bucket_count); + lua_pop(L, 1); /* pop buckets */ + + if (cmt_histogram_set_default( + histogram, timestamp, bucket_values, sum, count, + label_vals ? label_count : 0, label_vals)) { + return -1; + } + + } else { + + lua_getfield(L, -1, "value"); + value = lua_to_double(L, -1); + lua_pop(L, 1); /* pop value */ + + if (type == CMT_COUNTER) { + if (cmt_counter_set(counter, timestamp, value, + label_vals ? label_count : 0, + label_vals)) { + return -1; + } + } else if (type == CMT_GAUGE) { + if (cmt_gauge_set(gauge, timestamp, value, + label_vals ? label_count : 0, + label_vals)) { + return -1; + } + } else { + if (cmt_untyped_set(untyped, timestamp, value, + label_vals ? label_count : 0, + label_vals)) { + return -1; + } + } + } + + if (label_vals) { + free_labels(label_vals, label_count); + } + + if (type == CMT_SUMMARY) { + free(quantile_values); + } else if (type == CMT_HISTOGRAM) { + free(bucket_values); + } + + lua_pop(L, 1); /* pop sample */ + } + + free_labels(label_keys, label_count); + + if (type == CMT_SUMMARY) { + free(quantiles); + } else if (type == CMT_HISTOGRAM) { + free(buckets); + } + + lua_pop(L, 2); /* pop samples and metric */ + } + + return 0; +} diff --git a/plugins/processor_calyptia/calyptia_metrics_from_lua.h b/plugins/processor_calyptia/calyptia_metrics_from_lua.h new file mode 100644 index 00000000000..91004246ae4 --- /dev/null +++ b/plugins/processor_calyptia/calyptia_metrics_from_lua.h @@ -0,0 +1,10 @@ +#ifndef FLB_CALYPTIA_METRICS_FROM_LUA +#define FLB_CALYPTIA_METRICS_FROM_LUA + +#include +#include +#include + +int calyptia_metrics_from_lua(struct flb_processor_instance *ins, lua_State *L, struct cmt *cmt); + +#endif diff --git a/plugins/processor_calyptia/calyptia_metrics_to_lua.c b/plugins/processor_calyptia/calyptia_metrics_to_lua.c new file mode 100644 index 00000000000..6e1768fc562 --- /dev/null +++ b/plugins/processor_calyptia/calyptia_metrics_to_lua.c @@ -0,0 +1,254 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "calyptia_metrics_to_lua.h" +#include "cfl_to_lua.h" + +#define DOUBLE_MAX_SAFE_INTEGER 9007199254740991 +#define DOUBLE_MIN_SAFE_INTEGER -9007199254740991 + +static void push_uint64(lua_State *L, uint64_t val) +{ + char buf[64]; + if (val > DOUBLE_MAX_SAFE_INTEGER || val < DOUBLE_MIN_SAFE_INTEGER) { + snprintf(buf, sizeof(buf), "%" PRIu64, val); + lua_pushstring(L, buf); + } else { + lua_pushnumber(L, val); + } +} + +static void push_timestamp(lua_State *L, struct cmt_map *map, + struct cmt_metric *metric) +{ + uint64_t timestamp; + + timestamp = cmt_metric_get_timestamp(metric); + if (timestamp) { + push_timestamp_as_string(L, cmt_metric_get_timestamp(metric)); + lua_setfield(L, -2, "timestamp"); + } +} + +static void push_counter_gauge_untyped(lua_State *L, struct cmt_map *map, + struct cmt_metric *metric) +{ + lua_pushnumber(L, cmt_metric_get_value(metric)); + lua_setfield(L, -2, "value"); +} + +static void push_histogram(lua_State *L, struct cmt_map *map, + struct cmt_metric *metric) +{ + int i; + struct cmt_histogram *histogram; + struct cmt_histogram_buckets *bucket; + struct cmt_opts *opts; + + histogram = (struct cmt_histogram *) map->parent; + bucket = histogram->buckets; + opts = map->opts; + + lua_createtable(L, bucket->count, 0); + for (i = 0; i <= bucket->count; i++) { + if (i < bucket->count) { + lua_pushnumber(L, bucket->upper_bounds[i]); + } else { + lua_pushnumber(L, INFINITY); + } + push_uint64(L, cmt_metric_hist_get_value(metric, i)); + lua_settable(L, -3); + } + lua_setfield(L, -2, "buckets"); + + lua_pushnumber(L, cmt_metric_hist_get_sum_value(metric)); + lua_setfield(L, -2, "sum"); + + lua_pushnumber(L, cmt_metric_hist_get_count_value(metric)); + lua_setfield(L, -2, "count"); +} + +static void push_summary(lua_State *L, struct cmt_map *map, + struct cmt_metric *metric) +{ + struct cmt_summary *summary; + struct cmt_opts *opts; + + summary = (struct cmt_summary *) map->parent; + opts = map->opts; + + if (metric->sum_quantiles_set) { + lua_createtable(L, summary->quantiles_count, 0); + for (int i = 0; i < summary->quantiles_count; i++) { + lua_pushnumber(L, summary->quantiles[i]); + lua_pushnumber(L, cmt_summary_quantile_get_value(metric, i)); + lua_settable(L, -3); + } + lua_setfield(L, -2, "quantiles"); + } + + lua_pushnumber(L, cmt_summary_get_sum_value(metric)); + lua_setfield(L, -2, "sum"); + + lua_pushnumber(L, cmt_summary_get_count_value(metric)); + lua_setfield(L, -2, "count"); +} + +static void push_metric(lua_State *L, struct cmt_map *map, + struct cmt_metric *metric) +{ + struct cfl_list *head; + struct cmt_map_label *label_k; + struct cmt_map_label *label_v; + + /* labels, value, timestamp, count, sum, quantiles, buckets */ + lua_createtable(L, 6, 0); + + push_timestamp(L, map, metric); + if (map->type == CMT_HISTOGRAM) { + push_histogram(L, map, metric); + } else if (map->type == CMT_SUMMARY) { + push_summary(L, map, metric); + } else { + push_counter_gauge_untyped(L, map, metric); + } + + if (cfl_list_size(&metric->labels) == 0) { + return; + } + + /* labels table */ + label_k + = cfl_list_entry_first(&map->label_keys, struct cmt_map_label, _head); + lua_createtable(L, cfl_list_size(&metric->labels), 0); + cfl_list_foreach(head, &metric->labels) + { + label_v = cfl_list_entry(head, struct cmt_map_label, _head); + + lua_pushlstring(L, label_k->name, cfl_sds_len(label_k->name)); + lua_pushlstring(L, label_v->name, cfl_sds_len(label_v->name)); + lua_settable(L, -3); + + label_k = cfl_list_entry_next(&label_k->_head, struct cmt_map_label, + _head, &map->label_keys); + } + lua_setfield(L, -2, "labels"); +} + +static void push_header(lua_State *L, cfl_sds_t fqname, cfl_sds_t description, + const char *type) +{ + lua_createtable(L, 4, 0); + + lua_pushlstring(L, fqname, cfl_sds_len(fqname)); + lua_setfield(L, -2, "name"); + + lua_pushlstring(L, description, cfl_sds_len(description)); + lua_setfield(L, -2, "help"); + + lua_pushstring(L, type); + lua_setfield(L, -2, "type"); +} + +static void push_metrics(lua_State *L, struct cmt *cmt, struct cmt_map *map, + const char *kind) +{ + struct cfl_list *head; + struct cmt_metric *metric; + int metric_count; + + metric_count = cfl_list_size(&map->metrics); + if (metric_count == 0 && !map->metric_static_set) { + return; + } + + /* counter table, 4 keys: name, help, type and metrics array */ + push_header(L, map->opts->fqname, map->opts->description, kind); + + /* metrics array */ + lua_createtable(L, metric_count + 1, 0); + + if (map->metric_static_set) { + push_metric(L, map, &map->metric); + lua_rawseti(L, -2, lua_objlen(L, -2) + 1); + } + + if (metric_count) { + cfl_list_foreach(head, &map->metrics) + { + metric = cfl_list_entry(head, struct cmt_metric, _head); + push_metric(L, map, metric); + lua_rawseti(L, -2, lua_objlen(L, -2) + 1); + } + } + + lua_setfield(L, -2, "metrics"); + lua_rawseti(L, -2, lua_objlen(L, -2) + 1); +} + +int calyptia_metrics_to_lua(lua_State *L, struct cmt *cmt) +{ + int count; + struct cfl_list *head; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + struct cmt_summary *summary; + struct cmt_histogram *histogram; + struct cmt_untyped *untyped; + + if (cmt == NULL) { + return -1; + } + + count = cfl_list_size(&cmt->counters) + cfl_list_size(&cmt->gauges) + + cfl_list_size(&cmt->summaries) + cfl_list_size(&cmt->histograms) + + cfl_list_size(&cmt->untypeds); + + /* metrics array */ + lua_createtable(L, count, 0); + + /* Counters */ + cfl_list_foreach(head, &cmt->counters) + { + counter = cfl_list_entry(head, struct cmt_counter, _head); + push_metrics(L, cmt, counter->map, "COUNTER"); + } + + /* Gauges */ + cfl_list_foreach(head, &cmt->gauges) + { + gauge = cfl_list_entry(head, struct cmt_gauge, _head); + push_metrics(L, cmt, gauge->map, "GAUGE"); + } + + /* Summaries */ + cfl_list_foreach(head, &cmt->summaries) + { + summary = cfl_list_entry(head, struct cmt_summary, _head); + push_metrics(L, cmt, summary->map, "SUMMARY"); + } + + /* Histograms */ + cfl_list_foreach(head, &cmt->histograms) + { + histogram = cfl_list_entry(head, struct cmt_histogram, _head); + push_metrics(L, cmt, histogram->map, "HISTOGRAM"); + } + + /* Untyped */ + cfl_list_foreach(head, &cmt->untypeds) + { + untyped = cfl_list_entry(head, struct cmt_untyped, _head); + push_metrics(L, cmt, untyped->map, "UNTYPED"); + } + + return 0; +} diff --git a/plugins/processor_calyptia/calyptia_metrics_to_lua.h b/plugins/processor_calyptia/calyptia_metrics_to_lua.h new file mode 100644 index 00000000000..cd370557195 --- /dev/null +++ b/plugins/processor_calyptia/calyptia_metrics_to_lua.h @@ -0,0 +1,10 @@ +#ifndef FLB_CALYPTIA_METRICS_TO_LUA_H +#define FLB_CALYPTIA_METRICS_TO_LUA_H + +#include +#include + + +int calyptia_metrics_to_lua(lua_State *L, struct cmt *cmt); + +#endif diff --git a/plugins/processor_calyptia/calyptia_traces.c b/plugins/processor_calyptia/calyptia_traces.c new file mode 100644 index 00000000000..e5b0bd5851a --- /dev/null +++ b/plugins/processor_calyptia/calyptia_traces.c @@ -0,0 +1,79 @@ +#include +#include + +#include "calyptia_defs.h" +#include "calyptia_traces.h" +#include "calyptia_traces_to_lua.h" +#include "calyptia_traces_from_lua.h" + +static void drop_traces(struct ctrace *ctx) +{ + struct cfl_list *head; + struct cfl_list *tmp; + struct ctrace_resource_span *resource_span; + + cfl_list_foreach_safe(head, tmp, &ctx->resource_spans) + { + resource_span + = cfl_list_entry(head, struct ctrace_resource_span, _head); + ctr_resource_span_destroy(resource_span); + } + cfl_list_init(&ctx->resource_spans); + cfl_list_init(&ctx->span_list); +} + +int calyptia_process_traces(struct flb_processor_instance *ins, + struct ctrace *traces_context, const char *tag, + int tag_len) +{ + struct calyptia_context *ctx; + int ret; + int l_code; + + ret = FLB_PROCESSOR_SUCCESS; + ctx = ins->context; + + /* push the lua helper */ + lua_pushlightuserdata(ctx->lua->state, LUA_TRACES_HELPER_KEY); + lua_gettable(ctx->lua->state, LUA_REGISTRYINDEX); + /* push the lua callback */ + lua_getglobal(ctx->lua->state, ctx->call); + /* push the tag */ + lua_pushlstring(ctx->lua->state, tag, tag_len); + if (calyptia_traces_to_lua(ctx->lua->state, traces_context) != 0) { + flb_plg_error(ctx->ins, "Failed to encode traces"); + ret = FLB_PROCESSOR_FAILURE; + goto cleanup; + } + + ret = lua_pcall(ctx->lua->state, 3, 3, 0); + if (ret != 0) { + flb_plg_error(ctx->ins, "error code %d: %s", ret, + lua_tostring(ctx->lua->state, -1)); + lua_pop(ctx->lua->state, 1); + ret = FLB_PROCESSOR_FAILURE; + } + + /* index -2 is the "ingest" object, for which handling will only be + * implemented in the future */ + l_code = (int) lua_tointeger(ctx->lua->state, -3); + if (l_code == -1) { + drop_traces(traces_context); + } else if (l_code == 0) { + /* don't touch the traces */ + goto cleanup; + } else { + assert(l_code == 1); + drop_traces(traces_context); + + if (calyptia_traces_from_lua(ctx->lua->state, traces_context)) { + flb_plg_error(ctx->ins, "Failed to decode traces from lua"); + ret = FLB_PROCESSOR_FAILURE; + } + } + +cleanup: + /* clear lua stack */ + lua_settop(ctx->lua->state, 0); + return ret; +} diff --git a/plugins/processor_calyptia/calyptia_traces.h b/plugins/processor_calyptia/calyptia_traces.h new file mode 100644 index 00000000000..9cf13d45a93 --- /dev/null +++ b/plugins/processor_calyptia/calyptia_traces.h @@ -0,0 +1,11 @@ +#ifndef CALYPTIA_TRACES_H +#define CALYPTIA_TRACES_H + +#include +#include + +int calyptia_process_traces(struct flb_processor_instance *ins, + struct ctrace *traces_context, const char *tag, + int tag_len); + +#endif diff --git a/plugins/processor_calyptia/calyptia_traces_from_lua.c b/plugins/processor_calyptia/calyptia_traces_from_lua.c new file mode 100644 index 00000000000..0e210f8ab65 --- /dev/null +++ b/plugins/processor_calyptia/calyptia_traces_from_lua.c @@ -0,0 +1,303 @@ +#include +#include +#include + +#include "calyptia_traces_from_lua.h" +#include "lua_to_cfl.h" + +static void lua_to_attributes(lua_State *L, struct ctrace_attributes *attr); + +static struct ctrace_id *lua_to_id(lua_State *L) +{ + cfl_sds_t tmp_sds; + struct ctrace_id *cid; + + if (lua_type(L, -1) != LUA_TSTRING) { + return NULL; + } + + tmp_sds = lua_to_sds(L); + cid = ctr_id_from_base16(tmp_sds); + cfl_sds_destroy(tmp_sds); + return cid; +} + +static void lua_to_links(lua_State *L, struct ctrace_span *span) +{ + struct ctrace_link *link; + size_t count; + struct ctrace_id *trace_id; + + if (lua_type(L, -1) != LUA_TTABLE) { + return; + } + + count = lua_objlen(L, -1); + for (size_t i = 1; i <= count; i++) { + lua_rawgeti(L, -1, i); + + lua_getfield(L, -1, "traceId"); + trace_id = lua_to_id(L); + lua_pop(L, 1); + + link = ctr_link_create_with_cid(span, trace_id, span->span_id); + ctr_id_destroy(trace_id); + if (!link) { + lua_pop(L, 1); + return; + } + + lua_getfield(L, -1, "droppedAttributesCount"); + link->dropped_attr_count = lua_to_uint(L); + lua_pop(L, 1); + + lua_getfield(L, -1, "traceState"); + if (lua_type(L, -1) == LUA_TSTRING) { + ctr_link_set_trace_state(link, (char *) lua_tostring(L, -1)); + } + lua_pop(L, 1); + + lua_getfield(L, -1, "attributes"); + lua_to_attributes(L, link->attr); + lua_pop(L, 1); + + lua_pop(L, 1); /* pop the link we just processed */ + } +} + +static void lua_to_events(lua_State *L, struct ctrace_span *span) +{ + struct ctrace_span_event *event; + size_t count; + const char *name; + + if (lua_type(L, -1) != LUA_TTABLE) { + return; + } + + count = lua_objlen(L, -1); + for (size_t i = 1; i <= count; i++) { + lua_rawgeti(L, -1, i); + + lua_getfield(L, -1, "name"); + name = lua_tostring(L, -1); + lua_pop(L, 1); + + event = ctr_span_event_add(span, (char *)name); + if (!event) { + lua_pop(L, 1); + return; + } + + lua_getfield(L, -1, "timeUnixNano"); + event->time_unix_nano = lua_to_uint(L); + lua_pop(L, 1); + + lua_getfield(L, -1, "attributes"); + lua_to_attributes(L, event->attr); + lua_pop(L, 1); + + lua_getfield(L, -1, "droppedAttributesCount"); + event->dropped_attr_count = lua_to_uint(L); + lua_pop(L, 1); + + lua_pop(L, 1); /* pop the event we just processed */ + } +} + +static struct ctrace_instrumentation_scope * +lua_to_instrumentation_scope(lua_State *L, int index) +{ + struct ctrace_instrumentation_scope *scope; + struct ctrace_attributes *attr; + cfl_sds_t name; + cfl_sds_t version; + uint32_t dropped_attr_count; + + lua_getfield(L, index, "name"); + name = lua_to_sds(L); + lua_pop(L, 1); /* pop name */ + + lua_getfield(L, index, "version"); + version = lua_to_sds(L); + lua_pop(L, 1); /* pop version */ + + lua_getfield(L, index, "attributes"); + attr = ctr_attributes_create(); + lua_to_attributes(L, attr); + lua_pop(L, 1); /* pop attributes */ + + lua_getfield(L, index, "droppedAttributesCount"); + dropped_attr_count = lua_to_uint(L); + lua_pop(L, 1); /* pop droppedAttributesCount */ + + scope = ctr_instrumentation_scope_create(name, version, dropped_attr_count, + attr); + cfl_sds_destroy(name); + cfl_sds_destroy(version); + + return scope; +} + +static void lua_to_attributes(lua_State *L, struct ctrace_attributes *attr) +{ + if (lua_type(L, -1) != LUA_TTABLE || !attr) { + return; + } + + cfl_kvlist_destroy(attr->kv); + attr->kv = lua_map_to_variant(L); +} + +static void lua_to_spans(lua_State *L, struct ctrace *ctx, + struct ctrace_scope_span *scope_span) +{ + size_t count; + struct ctrace_span *span; + cfl_sds_t name; + struct ctrace_id *parent_span_id; + + if (lua_type(L, -1) != LUA_TTABLE) { + return; + } + + count = lua_objlen(L, -1); + for (size_t i = 1; i <= count; i++) { + lua_rawgeti(L, -1, i); + + lua_getfield(L, -1, "name"); + name = lua_to_sds(L); + lua_pop(L, 1); + + span = ctr_span_create(ctx, scope_span, name, NULL); + if (!span) { + cfl_sds_destroy(name); + lua_pop(L, 1); + return; + } + cfl_sds_destroy(name); + + lua_getfield(L, -1, "traceId"); + span->trace_id = lua_to_id(L); + lua_pop(L, 1); + + lua_getfield(L, -1, "spanId"); + span->span_id = lua_to_id(L); + lua_pop(L, 1); + + lua_getfield(L, -1, "parentSpanId"); + parent_span_id = lua_to_id(L); + lua_pop(L, 1); + if (parent_span_id) { + ctr_span_set_parent_span_id_with_cid(span, parent_span_id); + ctr_id_destroy(parent_span_id); + } + + lua_getfield(L, -1, "kind"); + span->kind = lua_to_int(L); + lua_pop(L, 1); /* pop "kind" */ + + lua_getfield(L, -1, "startTimeUnixNano"); + span->start_time_unix_nano = lua_to_uint(L); + lua_pop(L, 1); /* pop "startTimeUnixNano" */ + + lua_getfield(L, -1, "endTimeUnixNano"); + span->end_time_unix_nano = lua_to_uint(L); + lua_pop(L, 1); /* pop "endTimeUnixNano" */ + + lua_getfield(L, -1, "attributes"); + lua_to_attributes(L, span->attr); + lua_pop(L, 1); + + lua_getfield(L, -1, "events"); + lua_to_events(L, span); + lua_pop(L, 1); /* pop events */ + + lua_getfield(L, -1, "links"); + lua_to_links(L, span); + lua_pop(L, 1); /* pop links */ + + lua_pop(L, 1); /* pop the span we just processed */ + } +} + +static void lua_to_scope_spans(lua_State *L, struct ctrace *ctx, + struct ctrace_resource_span *resource_span) +{ + size_t count; + struct ctrace_scope_span *scope_span; + + if (lua_type(L, -1) != LUA_TTABLE) { + return; + } + + count = lua_objlen(L, -1); + + for (size_t i = 1; i <= count; i++) { + scope_span = ctr_scope_span_create(resource_span); + + lua_rawgeti(L, -1, i); + + lua_getfield(L, -1, "schemaUrl"); + scope_span->schema_url = lua_to_sds(L); + lua_pop(L, 1); /* pop "schemaUrl" */ + + lua_getfield(L, -1, "scope"); + scope_span->instrumentation_scope + = lua_to_instrumentation_scope(L, lua_gettop(L)); + lua_pop(L, 1); /* pop "scope" */ + + lua_getfield(L, -1, "spans"); + lua_to_spans(L, ctx, scope_span); + lua_pop(L, 1); /* pop "spans" */ + + lua_pop(L, 1); /* pop the scope_span we just processed */ + } +} + +static void lua_to_resource_spans(lua_State *L, struct ctrace *ctx) +{ + size_t count; + struct ctrace_resource_span *resource_span; + + if (lua_type(L, -1) != LUA_TTABLE) { + return; + } + + count = lua_objlen(L, -1); + + for (size_t i = 1; i <= count; i++) { + resource_span = ctr_resource_span_create(ctx); + + lua_rawgeti(L, -1, i); + + lua_getfield(L, -1, "resource"); + + lua_getfield(L, -1, "attributes"); + lua_to_attributes(L, resource_span->resource->attr); + lua_pop(L, 1); /* pop "attributes" */ + + lua_getfield(L, -1, "droppedAttributesCount"); + resource_span->resource->dropped_attr_count = lua_to_uint(L); + lua_pop(L, 1); /* pop "droppedAttributesCount" */ + + lua_pop(L, 1); /* pop "resource" */ + + lua_getfield(L, -1, "schemaUrl"); + resource_span->schema_url = lua_to_sds(L); + lua_pop(L, 1); /* pop "schemaUrl" */ + + lua_getfield(L, -1, "scopeSpans"); + lua_to_scope_spans(L, ctx, resource_span); + lua_pop(L, 1); /* pop "scopeSpans" */ + + lua_pop(L, 1); /* pop the resourceSpan we just processed */ + } +} + +int calyptia_traces_from_lua(lua_State *L, struct ctrace *ctx) +{ + lua_to_resource_spans(L, ctx); + + return 0; +} diff --git a/plugins/processor_calyptia/calyptia_traces_from_lua.h b/plugins/processor_calyptia/calyptia_traces_from_lua.h new file mode 100644 index 00000000000..4ec35364465 --- /dev/null +++ b/plugins/processor_calyptia/calyptia_traces_from_lua.h @@ -0,0 +1,9 @@ +#ifndef CALYPTIA_TRACES_FROM_LUA_H +#define CALYPTIA_TRACES_FROM_LUA_H + +#include +#include + +int calyptia_traces_from_lua(lua_State *L, struct ctrace *ctx); + +#endif diff --git a/plugins/processor_calyptia/calyptia_traces_to_lua.c b/plugins/processor_calyptia/calyptia_traces_to_lua.c new file mode 100644 index 00000000000..83c91d8c92b --- /dev/null +++ b/plugins/processor_calyptia/calyptia_traces_to_lua.c @@ -0,0 +1,290 @@ +#include +#include +#include + +#include "cfl_to_lua.h" + +static void push_attributes(lua_State *L, struct ctrace_attributes *attr) +{ + struct cfl_kvlist *kvlist; + + kvlist = attr->kv; + push_kvlist(L, kvlist); +} + +static void push_instrumentation_scope(lua_State *L, struct ctrace_instrumentation_scope *ins_scope) +{ + lua_createtable(L, 0, 4); + + if (ins_scope->name) { + push_string(L, ins_scope->name, cfl_sds_len(ins_scope->name)); + lua_setfield(L, -2, "name"); + } + + if (ins_scope->version) { + push_string(L, ins_scope->version, cfl_sds_len(ins_scope->version)); + lua_setfield(L, -2, "version"); + } + + if (ins_scope->attr) { + push_attributes(L, ins_scope->attr); + lua_setfield(L, -2, "attributes"); + } + + lua_pushinteger(L, ins_scope->dropped_attr_count); + lua_setfield(L, -2, "droppedAttributesCount"); +} + +static void push_id(lua_State *L, struct ctrace_id *id) +{ + cfl_sds_t encoded_id; + + if (id) { + encoded_id = ctr_id_to_lower_base16(id); + + if (encoded_id != NULL) { + lua_pushstring(L, encoded_id); + cfl_sds_destroy(encoded_id); + } + else { + lua_pushnil(L); + } + } + else { + lua_pushnil(L); + } +} + +static void push_events(lua_State *L, struct cfl_list *events) +{ + int count; + struct cfl_list *head; + struct ctrace_span_event *event; + + count = cfl_list_size(events); + if (!count) { + lua_pushnil(L); + return; + } + + lua_createtable(L, count, 0); + + cfl_list_foreach(head, events) { + event = cfl_list_entry(head, struct ctrace_span_event, _head); + + lua_createtable(L, 0, 4); + + push_timestamp_as_string(L, event->time_unix_nano); + lua_setfield(L, -2, "timeUnixNano"); + + if (event->name) { + push_string(L, event->name, cfl_sds_len(event->name)); + } + else { + lua_pushnil(L); + } + lua_setfield(L, -2, "name"); + + if (event->attr) { + push_attributes(L, event->attr); + } + else { + lua_pushnil(L); + } + lua_setfield(L, -2, "attributes"); + + lua_pushinteger(L, event->dropped_attr_count); + lua_setfield(L, -2, "droppedAttributesCount"); + + lua_rawseti(L, -2, lua_objlen(L, -2) + 1); + } +} + +static void push_links(lua_State *L, struct cfl_list *links) +{ + int count; + struct cfl_list *head; + struct ctrace_link *link; + + count = cfl_list_size(links); + lua_createtable(L, count, 0); + + cfl_list_foreach(head, links) { + link = cfl_list_entry(head, struct ctrace_link, _head); + + lua_createtable(L, 0, 5); + + push_id(L, link->trace_id); + lua_setfield(L, -2, "traceId"); + + push_id(L, link->span_id); + lua_setfield(L, -2, "spanId"); + + if (link->trace_state) { + push_string(L, link->trace_state, cfl_sds_len(link->trace_state)); + lua_setfield(L, -2, "traceState"); + } + + if (link->attr) { + push_attributes(L, link->attr); + lua_setfield(L, -2, "attributes"); + } + + lua_pushinteger(L, link->dropped_attr_count); + lua_setfield(L, -2, "droppedAttributesCount"); + + lua_rawseti(L, -2, lua_objlen(L, -2) + 1); + } +} + +static void push_span(lua_State *L, struct ctrace_span *span) +{ + lua_createtable(L, 0, 13); + + push_id(L, span->trace_id); + lua_setfield(L, -2, "traceId"); + + push_id(L, span->span_id); + lua_setfield(L, -2, "spanId"); + + push_id(L, span->parent_span_id); + lua_setfield(L, -2, "parentSpanId"); + + if (span->trace_state) { + push_string(L, span->trace_state, cfl_sds_len(span->trace_state)); + lua_setfield(L, -2, "traceState"); + } + + if (span->name) { + push_string(L, span->name, cfl_sds_len(span->name)); + lua_setfield(L, -2, "name"); + } + + lua_pushinteger(L, span->kind); + lua_setfield(L, -2, "kind"); + + push_timestamp_as_string(L, span->start_time_unix_nano); + lua_setfield(L, -2, "startTimeUnixNano"); + + push_timestamp_as_string(L, span->end_time_unix_nano); + lua_setfield(L, -2, "endTimeUnixNano"); + + if (span->attr) { + push_attributes(L, span->attr); + lua_setfield(L, -2, "attributes"); + } + + lua_pushinteger(L, span->dropped_attr_count); + lua_setfield(L, -2, "droppedAttributesCount"); + + push_events(L, &span->events); + lua_setfield(L, -2, "events"); + + push_links(L, &span->links); + lua_setfield(L, -2, "links"); + + lua_createtable(L, 0, 2); + lua_pushinteger(L, span->status.code); + lua_setfield(L, -2, "code"); + if (span->status.message) { + push_string(L, span->status.message, cfl_sds_len(span->status.message)); + lua_setfield(L, -2, "message"); + } + lua_setfield(L, -2, "status"); +} + +static void push_spans(lua_State *L, struct ctrace_scope_span *scope_span) +{ + struct cfl_list *head; + struct ctrace_span *span; + size_t count; + + /* scopeSpans */ + count = cfl_list_size(&scope_span->spans); + lua_createtable(L, count, 0); + + cfl_list_foreach(head, &scope_span->spans) { + span = cfl_list_entry(head, struct ctrace_span, _head); + + push_span(L, span); + + size_t objlen = lua_objlen(L, -2); + lua_rawseti(L, -2, objlen + 1); + } +} + +static void push_scope_spans(lua_State *L, struct ctrace_resource_span *resource_span) +{ + struct cfl_list *head; + struct ctrace_scope_span *scope_span; + size_t count; + + /* scopeSpans */ + count = cfl_list_size(&resource_span->scope_spans); + lua_createtable(L, count, 0); + + cfl_list_foreach(head, &resource_span->scope_spans) { + scope_span = cfl_list_entry(head, struct ctrace_scope_span, _head); + + lua_createtable(L, 0, 3); + + if (scope_span->schema_url) { + push_string(L, scope_span->schema_url, cfl_sds_len(scope_span->schema_url)); + lua_setfield(L, -2, "schemaUrl"); + } + + push_instrumentation_scope(L, scope_span->instrumentation_scope); + lua_setfield(L, -2, "scope"); + + push_spans(L, scope_span); + lua_setfield(L, -2, "spans"); + + size_t objlen = lua_objlen(L, -2); + lua_rawseti(L, -2, objlen + 1); + } +} + +int calyptia_traces_to_lua(lua_State *L, struct ctrace *ctx) +{ + int count; + struct cfl_list *head; + struct ctrace_resource_span *resource_span; + struct ctrace_resource *resource; + + if (ctx == NULL) { + return luaL_error(L, "Invalid trace context provided."); + } + /* resourceSpans */ + count = cfl_list_size(&ctx->resource_spans); + lua_createtable(L, count, 0); + + cfl_list_foreach(head, &ctx->resource_spans) { + resource_span = cfl_list_entry(head, struct ctrace_resource_span, _head); + lua_createtable(L, 0, 3); + + resource = resource_span->resource; + lua_createtable(L, 0, 2); + + if (resource->attr) { + push_attributes(L, resource->attr); + lua_setfield(L, -2, "attributes"); + } + + lua_pushinteger(L, resource->dropped_attr_count); + lua_setfield(L, -2, "droppedAttributesCount"); + + lua_setfield(L, -2, "resource"); + + if (resource_span->schema_url) { + push_string(L, resource_span->schema_url, cfl_sds_len(resource_span->schema_url)); + lua_setfield(L, -2, "schemaUrl"); + } + + push_scope_spans(L, resource_span); + lua_setfield(L, -2, "scopeSpans"); + + lua_rawseti(L, -2, lua_objlen(L, -2) + 1); + } + + return 0; +} diff --git a/plugins/processor_calyptia/calyptia_traces_to_lua.h b/plugins/processor_calyptia/calyptia_traces_to_lua.h new file mode 100644 index 00000000000..8193271fbcd --- /dev/null +++ b/plugins/processor_calyptia/calyptia_traces_to_lua.h @@ -0,0 +1,9 @@ +#ifndef CALYPTIA_TRACES_TO_LUA_H +#define CALYPTIA_TRACES_TO_LUA_H + +#include +#include + +int calyptia_traces_to_lua(lua_State *L, struct ctrace *ctx); + +#endif diff --git a/plugins/processor_calyptia/cfl_to_lua.c b/plugins/processor_calyptia/cfl_to_lua.c new file mode 100644 index 00000000000..a94723ef73a --- /dev/null +++ b/plugins/processor_calyptia/cfl_to_lua.c @@ -0,0 +1,94 @@ +#include "cfl_to_lua.h" + + +void push_string(lua_State *L, const char *str, size_t size) +{ + lua_pushlstring(L, str, size); +} + +void push_array(lua_State *L, struct cfl_array *array) +{ + int i; + struct cfl_variant *entry; + + lua_createtable(L, array->entry_count, 0); + + for (i = 0; i < array->entry_count; i++) { + entry = array->entries[i]; + push_variant(L, entry); + lua_rawseti(L, -2, i + 1); + } +} + +void push_kvlist(lua_State *L, struct cfl_kvlist *kvlist) +{ + struct cfl_list *head; + struct cfl_list *list; + struct cfl_kvpair *kvpair; + + list = &kvlist->list; + + lua_createtable(L, 0, cfl_list_size(list)); + + cfl_list_foreach(head, list) + { + kvpair = cfl_list_entry(head, struct cfl_kvpair, _head); + + push_string(L, kvpair->key, cfl_sds_len(kvpair->key)); + push_variant(L, kvpair->val); + + lua_settable(L, -3); + } +} + +void push_timestamp_as_table(lua_State *L, uint64_t timestamp) +{ + lua_createtable(L, 0, 2); + lua_pushinteger(L, timestamp / 1000000); + lua_setfield(L, -2, "millis"); + lua_pushinteger(L, timestamp % 1000000); + lua_setfield(L, -2, "nanos"); +} + +void push_timestamp_as_string(lua_State *L, uint64_t timestamp) +{ + char buf[64]; + snprintf(buf, sizeof(buf), "%" PRIu64, timestamp); + lua_pushstring(L, buf); +} + +void push_variant(lua_State *L, struct cfl_variant *variant) +{ + int type = variant->type; + + switch (type) { + case CFL_VARIANT_STRING: + push_string(L, variant->data.as_string, cfl_variant_size_get(variant)); + break; + case CFL_VARIANT_BOOL: + lua_pushboolean(L, variant->data.as_bool); + break; + case CFL_VARIANT_INT: + lua_pushinteger(L, variant->data.as_int64); + break; + case CFL_VARIANT_UINT: + lua_pushinteger(L, variant->data.as_uint64); + break; + case CFL_VARIANT_DOUBLE: + lua_pushnumber(L, variant->data.as_double); + break; + case CFL_VARIANT_ARRAY: + push_array(L, variant->data.as_array); + break; + case CFL_VARIANT_KVLIST: + push_kvlist(L, variant->data.as_kvlist); + break; + case CFL_VARIANT_BYTES: + push_string(L, variant->data.as_bytes, cfl_variant_size_get(variant)); + break; + default: + /* unsupported type, push nil */ + lua_pushnil(L); + break; + } +} diff --git a/plugins/processor_calyptia/cfl_to_lua.h b/plugins/processor_calyptia/cfl_to_lua.h new file mode 100644 index 00000000000..aa6d8907d42 --- /dev/null +++ b/plugins/processor_calyptia/cfl_to_lua.h @@ -0,0 +1,15 @@ +#ifndef FLB_CALYPTIA_PROCESSOR_CFL_TO_LUA_H +#define FLB_CALYPTIA_PROCESSOR_CFL_TO_LUA_H + +#include +#include + +void push_variant(lua_State *L, struct cfl_variant *variant); +void push_string(lua_State *L, const char *str, size_t size); +void push_array(lua_State *L, struct cfl_array *array); +void push_kvlist(lua_State *L, struct cfl_kvlist *kvlist); +void push_timestamp_as_table(lua_State *L, uint64_t timestamp); +void push_timestamp_as_string(lua_State *L, uint64_t timestamp); +void push_variant(lua_State *L, struct cfl_variant *variant); + +#endif diff --git a/plugins/processor_calyptia/generate_lua_helpers.cmake b/plugins/processor_calyptia/generate_lua_helpers.cmake new file mode 100644 index 00000000000..5f8e5a8d68d --- /dev/null +++ b/plugins/processor_calyptia/generate_lua_helpers.cmake @@ -0,0 +1,33 @@ +# cmake script which converts a lua helpers file into a C header for embedding +# in the executable +set(INPUT_FILE ${CMAKE_ARGV3}) +set(OUTPUT_FILE ${CMAKE_ARGV4}) + +file(READ ${INPUT_FILE} content HEX) +string(REGEX MATCHALL "([A-Fa-f0-9][A-Fa-f0-9])" SEPARATED_HEX ${content}) + +# Create a counter so that we only have 16 hex bytes per line +set(counter 0) +# Iterate through each of the bytes from the source file +foreach (hex IN LISTS SEPARATED_HEX) + # Write the hex string to the line with an 0x prefix + # and a , postfix to seperate the bytes of the file. + string(APPEND output_c "0x${hex},") + # Increment the element counter before the newline. + math(EXPR counter "${counter}+1") + if (counter GREATER 16) + # Write a newline so that all of the array initializer + # gets spread across multiple lines. + string(APPEND output_c "\n ") + set(counter 0) + endif () +endforeach () + +set(c_name calyptia_processor_lua_helpers) +set(output_c " +char ${c_name}[] = { +${output_c} 0x0 +}\; +") + +file(WRITE ${OUTPUT_FILE} ${output_c}) diff --git a/plugins/processor_calyptia/lua_to_cfl.c b/plugins/processor_calyptia/lua_to_cfl.c new file mode 100644 index 00000000000..e5d0697ac38 --- /dev/null +++ b/plugins/processor_calyptia/lua_to_cfl.c @@ -0,0 +1,129 @@ +#include "lua_to_cfl.h" + +#include + + +cfl_sds_t lua_to_sds(lua_State *L) +{ + size_t len; + const char *str; + + if (lua_type(L, -1) != LUA_TSTRING) { + return NULL; + } + + str = lua_tolstring(L, -1, &len); + return cfl_sds_create_len(str, len); +} + +double lua_to_double(lua_State *L, int index) +{ + int type = lua_type(L, index); + if (type == LUA_TNUMBER) { + return lua_tonumber(L, index); + } else if (type == LUA_TSTRING) { + return atof(lua_tostring(L, index)); + } else { + return 0.0; + } +} + +uint64_t lua_to_uint(lua_State *L) +{ + int type = lua_type(L, -1); + if (type == LUA_TNUMBER) { + return lua_tointeger(L, -1); + } else if (type == LUA_TSTRING) { + return strtoull(lua_tostring(L, -1), NULL, 10); + } else { + return 0; + } +} + +long lua_to_int(lua_State *L) +{ + int type = lua_type(L, -1); + if (type == LUA_TNUMBER) { + return lua_tointeger(L, -1); + } else if (type == LUA_TSTRING) { + return strtol(lua_tostring(L, -1), NULL, 10); + } else { + return 0; + } +} + +struct cfl_variant *lua_string_to_variant(lua_State *L, int index) +{ + size_t len; + const char *str = lua_tolstring(L, index, &len); + return cfl_variant_create_from_string_s((char *) str, len, 0); +} + +bool lua_isinteger(lua_State *L, int index) +{ + if (lua_isnumber(L, index)) { + double val = lua_tonumber(L, index); + return val == (int64_t) val; + } + return false; +} + +struct cfl_array *lua_array_to_variant(lua_State *L, int array_len) +{ + struct cfl_array *array = cfl_array_create(array_len); + + for (int i = 1; i <= array_len; i++) { + lua_rawgeti(L, -1, i); + struct cfl_variant *variant = lua_to_variant(L, -1); + cfl_array_append(array, variant); + lua_pop(L, 1); + } + + return array; +} + +struct cfl_kvlist *lua_map_to_variant(lua_State *L) +{ + struct cfl_kvlist *kvlist = cfl_kvlist_create(); + + lua_pushnil(L); // first key + while (lua_next(L, -2) != 0) { + const char *key = lua_tostring(L, -2); + struct cfl_variant *value = lua_to_variant(L, -1); + cfl_kvlist_insert(kvlist, (char *) key, value); + + // removes 'value'; keeps 'key' for next iteration + lua_pop(L, 1); + } + + return kvlist; +} + +struct cfl_variant *lua_to_variant(lua_State *L, int index) +{ + int array_len; + int type = lua_type(L, index); + + switch (type) { + case LUA_TNUMBER: + if (lua_isinteger(L, index)) { + return cfl_variant_create_from_int64(lua_tointeger(L, index)); + } else { + return cfl_variant_create_from_double(lua_tonumber(L, index)); + } + case LUA_TBOOLEAN: + return cfl_variant_create_from_bool(lua_toboolean(L, index)); + case LUA_TSTRING: + return lua_string_to_variant(L, index); + case LUA_TTABLE: + array_len = flb_lua_arraylength(L, index); + if (array_len > 0) { + return cfl_variant_create_from_array(lua_array_to_variant(L, array_len)); + } else { + return cfl_variant_create_from_kvlist(lua_map_to_variant(L)); + } + default: + // nil or Unsupported types + return cfl_variant_create_from_null(); + } +} diff --git a/plugins/processor_calyptia/lua_to_cfl.h b/plugins/processor_calyptia/lua_to_cfl.h new file mode 100644 index 00000000000..c88a5ba3563 --- /dev/null +++ b/plugins/processor_calyptia/lua_to_cfl.h @@ -0,0 +1,18 @@ +#ifndef FLB_CALYPTIA_PROCESSOR_LUA_TO_CFL_H +#define FLB_CALYPTIA_PROCESSOR_LUA_TO_CFL_H + +#include +#include +#include + +cfl_sds_t lua_to_sds(lua_State *L); +double lua_to_double(lua_State *L, int index); +uint64_t lua_to_uint(lua_State *L); +long lua_to_int(lua_State *L); +struct cfl_variant *lua_string_to_variant(lua_State *L, int index); +bool lua_isinteger(lua_State *L, int index); +struct cfl_array *lua_array_to_variant(lua_State *L, int array_len); +struct cfl_kvlist *lua_map_to_variant(lua_State *L); +struct cfl_variant *lua_to_variant(lua_State *L, int index); + +#endif diff --git a/plugins/processor_calyptia/processor_helpers.lua b/plugins/processor_calyptia/processor_helpers.lua new file mode 100644 index 00000000000..80e340c6ffb --- /dev/null +++ b/plugins/processor_calyptia/processor_helpers.lua @@ -0,0 +1,277 @@ +-- allow C code to pass us options +local opts = LUA_HELPERS_OPTS +LUA_HELPERS_OPTS = nil + +local define_enums = function() + DROP = -1 + KEEP = 0 + MODIFY = 1 + MODIFY_KEEP_TIMESTAMP = 2 + return DROP, + KEEP, + MODIFY, + MODIFY_KEEP_TIMESTAMP +end + +local DROP, KEEP, MODIFY, MODIFY_KEEP_TIMESTAMP = define_enums() + +local function warn(msg, ...) + if opts.disable_warnings then + return + end + print(msg:format(...)) +end + +local function is_array(table) + return #table > 0 and next(table, #table) == nil +end + +local function concat_tables(t1, t2) + local result = {} + if type(t1) == 'table' then + for k, v in pairs(t1) do + result[k] = v + end + end + if type(t2) == 'table' then + for k, v in pairs(t2) do + result[k] = v + end + end + return result +end + +local function assign_table(t1, t2, exclude) + for k, v in pairs(t2) do + if k ~= exclude then + t1[k] = v + end + end + return t1 +end + +-- utility function to process a set of logs/metrics/traces using API similar to Lua classic filter +local function process_records(fn, tag, records, records_metadata, records_timestamps) + local new_records + local new_records_metadata + local new_records_timestamps + local ingest + local keep_cnt = 0 + local drop_cnt = 0 + local split_cnt = 0 + local orig_record_count = #records + local shared_metadata = type(records_metadata) ~= 'table' or not is_array(records_metadata) + local shared_timestamp = type(records_timestamps) ~= 'table' + local metadata_changed = false + local timestamps_changed = false + + for recIndex, record in ipairs(records) do + local metadata = records_metadata and (records_metadata[recIndex] or records_metadata) + local ts = records_timestamps and records_timestamps[recIndex] or 0 + local code, new_ts, new_record, new_metadata, new_ingest = fn(tag, ts, record, metadata) + + if code == KEEP then + -- use original record, only increase keep count + keep_cnt = keep_cnt + 1 + elseif code == MODIFY or code == MODIFY_KEEP_TIMESTAMP then + if type(new_record) ~= 'table' then + error('processor_helper: expected a table as the result of the user function') + end + + if type(new_metadata) == 'table' then + metadata_changed = true + if shared_metadata then + if (type(records_metadata) == 'table') then + assign_table(records_metadata, new_metadata) + end + else + records_metadata[recIndex] = new_metadata + end + end + + if new_ts and code ~= MODIFY_KEEP_TIMESTAMP then + if not shared_timestamp then + timestamps_changed = true + records_timestamps[recIndex] = new_ts + end + end + + if type(new_ingest) == 'table' and is_array(new_ingest) then + ingest = concat_tables(ingest, new_ingest) + end + + if is_array(new_record) then + split_cnt = split_cnt + 1 + end + + records[recIndex] = new_record + elseif code == DROP then + records[recIndex] = nil + drop_cnt = drop_cnt + 1 + else + warn('processor_helper: unexpected code returned by user function: %s', code) + end + end + + new_records = records + new_records_metadata = records_metadata + new_records_timestamps = records_timestamps + if split_cnt > 0 or drop_cnt > 0 then + -- we need to use a new array since the record count changed + new_records = {} + if not shared_metadata then + new_records_metadata = {} + end + if not shared_timestamp then + new_records_timestamps = {} + end + for i = 1, orig_record_count do + local record = records[i] + if type(record) == 'table' then + + if is_array(record) then + + for _, rec in ipairs(record) do + table.insert(new_records, rec) + end + + if not shared_metadata then + local rec_md = records_metadata[i] + if is_array(rec_md) then + for _, md in ipairs(rec_md) do + table.insert(new_records_metadata, md) + end + else + for _=1, #record do + table.insert(new_records_metadata, rec_md) + end + end + end + + if not shared_timestamp then + local rec_ts = records_timestamps[i] + if type(rec_ts) == 'table' and is_array(rec_ts) then + for _, ts in ipairs(rec_ts) do + table.insert(new_records_timestamps, ts) + end + else + for _=1, #record do + table.insert(new_records_timestamps, rec_ts) + end + end + end + + else + table.insert(new_records, record) + + if not shared_metadata then + table.insert(new_records_metadata, records_metadata[i]) + end + + if not shared_timestamp then + table.insert(new_records_timestamps, records_timestamps[i]) + end + end + end + end + end + + return new_records, new_records_metadata, new_records_timestamps, ingest, keep_cnt, drop_cnt +end + +local function logs_helper(fn, tag, events) + local orig_count = #events.logs + local new_logs, new_metadata, new_timestamps, ingest, keep_cnt, drop_cnt = process_records(fn, tag, events.logs, events.metadata, events.timestamps) + if keep_cnt == orig_count then + return 0 + elseif drop_cnt == orig_count then + return -1 + else + return 1, ingest, { + logs = new_logs, + metadata = new_metadata, + timestamps = new_timestamps + } + end +end + +local function metrics_helper(fn, tag, metrics) + local orig_count = #metrics + local new_metrics, _, _, ingest, keep_cnt, drop_cnt = process_records(fn, tag, metrics, nil, nil) + + if keep_cnt == orig_count then + return 0 + elseif drop_cnt == orig_count then + return -1 + else + return 1, ingest, new_metrics + end +end + +-- reuse this table to avoid unnecessary allocations between calls to traces_helper +local traces_metadata = {} +local traces_metadata_resourceSpan = {} +local traces_metadata_scopeSpan = {} + +local function traces_helper(fn, tag, resourceSpans) + local ingest + local resource_keep_cnt = 0 + local resource_drop_cnt = 0 + local orig_resource_spans_count = #resourceSpans + + for resourceSpanIndex, resourceSpan in ipairs(resourceSpans) do + local orig_scope_spans_count = #resourceSpan.scopeSpans + local scope_keep_cnt = 0 + local scope_drop_cnt = 0 + + for scopeSpanIndex, scopeSpan in ipairs(resourceSpan.scopeSpans) do + local orig_spans_count = #scopeSpan.spans + traces_metadata.resourceSpan = traces_metadata_resourceSpan + traces_metadata.scopeSpan = traces_metadata_scopeSpan + assign_table(traces_metadata_resourceSpan, resourceSpan, 'scopeSpans') + assign_table(traces_metadata_scopeSpan, scopeSpan, 'spans') + + local new_spans, new_metadata, _, new_ingest, keep_cnt, drop_cnt = process_records(fn, tag, scopeSpan.spans, traces_metadata, nil) + + if type(new_metadata) == 'table' then + -- metadata was returned, update the resourceSpan and scopeSpan + if type(new_metadata.scopeSpan) == 'table' then + assign_table(scopeSpan, new_metadata.scopeSpan) + resourceSpan.scopeSpans[scopeSpanIndex] = scopeSpan + end + if type(new_metadata.resourceSpan) == 'table' then + assign_table(resourceSpan, new_metadata.resourceSpan) + resourceSpans[resourceSpanIndex] = resourceSpan + end + end + + if type(new_ingest) == 'table' then + ingest = concat_tables(ingest, new_ingest) + end + + if keep_cnt == orig_spans_count then + scope_keep_cnt = scope_keep_cnt + 1 + elseif drop_cnt == orig_spans_count then + scope_drop_cnt = scope_drop_cnt + 1 + else + scopeSpan.spans = new_spans + end + end + + if scope_keep_cnt == orig_scope_spans_count then + resource_keep_cnt = resource_keep_cnt + 1 + elseif scope_drop_cnt == orig_scope_spans_count then + resource_drop_cnt = resource_drop_cnt + 1 + end + end + + if resource_keep_cnt == orig_resource_spans_count then + return 0 + elseif resource_drop_cnt == orig_resource_spans_count then + return -1 + else + return 1, ingest, resourceSpans + end +end + +return logs_helper, metrics_helper, traces_helper From 4eca76775e5cbd0d5c98064304f0191b10641f7f Mon Sep 17 00:00:00 2001 From: Thiago Padilha Date: Mon, 2 Sep 2024 10:37:14 -0300 Subject: [PATCH 02/11] processor_calyptia: fix if/else code style Signed-off-by: Thiago Padilha --- plugins/processor_calyptia/calyptia.c | 6 ++- plugins/processor_calyptia/calyptia_logs.c | 9 ++-- plugins/processor_calyptia/calyptia_metrics.c | 12 +++-- .../calyptia_metrics_from_lua.c | 48 ++++++++++++------- .../calyptia_metrics_to_lua.c | 12 +++-- plugins/processor_calyptia/calyptia_traces.c | 6 ++- plugins/processor_calyptia/lua_to_cfl.c | 24 ++++++---- 7 files changed, 78 insertions(+), 39 deletions(-) diff --git a/plugins/processor_calyptia/calyptia.c b/plugins/processor_calyptia/calyptia.c index 503af473a83..0dabae73468 100644 --- a/plugins/processor_calyptia/calyptia.c +++ b/plugins/processor_calyptia/calyptia.c @@ -82,7 +82,8 @@ calyptia_config_create(struct flb_processor_instance *ins, tmp = flb_processor_instance_get_property("code", ins); if (tmp) { ctx->code = flb_sds_create(tmp); - } else { + } + else { /* config: script */ script = flb_processor_instance_get_property("script", ins); if (!script) { @@ -192,7 +193,8 @@ calyptia_config_create(struct flb_processor_instance *ins, calyptia_config_destroy(ctx); return NULL; } - } else if (flb_luajit_load_script(ctx->lua, ctx->script)) { + } + else if (flb_luajit_load_script(ctx->lua, ctx->script)) { calyptia_config_destroy(ctx); return NULL; } diff --git a/plugins/processor_calyptia/calyptia_logs.c b/plugins/processor_calyptia/calyptia_logs.c index 6d2fe12e5a3..ab333088867 100644 --- a/plugins/processor_calyptia/calyptia_logs.c +++ b/plugins/processor_calyptia/calyptia_logs.c @@ -65,12 +65,15 @@ int calyptia_process_logs(struct flb_processor_instance *ins, void *chunk_data, l_code = (int) lua_tointeger(ctx->lua->state, -3); if (l_code == -1) { clear_logs(chunk_cobj); - } else if (l_code == 0) { + } + else if (l_code == 0) { /* nothing to do */ - } else if (l_code != 1) { + } + else if (l_code != 1) { flb_plg_error(ctx->ins, "invalid return code %d", l_code); ret = FLB_PROCESSOR_FAILURE; - } else { + } + else { clear_logs(chunk_cobj); if (calyptia_logs_from_lua(ins, ctx->lua->state, chunk_cobj)) { flb_plg_error(ctx->ins, "Failed to decode logs from lua"); diff --git a/plugins/processor_calyptia/calyptia_metrics.c b/plugins/processor_calyptia/calyptia_metrics.c index 9b91969d01b..eac00affa5c 100644 --- a/plugins/processor_calyptia/calyptia_metrics.c +++ b/plugins/processor_calyptia/calyptia_metrics.c @@ -52,19 +52,23 @@ int calyptia_process_metrics(struct flb_processor_instance *ins, l_code = (int) lua_tointeger(ctx->lua->state, -3); if (l_code == -1) { *out_context = cmt_create(); - } else if (l_code == 0) { + } + else if (l_code == 0) { /* don't touch the metrics */ *out_context = metrics_context; - } else if (l_code != 1) { + } + else if (l_code != 1) { flb_plg_error(ctx->ins, "invalid return code %d", l_code); ret = FLB_PROCESSOR_FAILURE; - } else { + } + else { struct cmt *new_metrics = cmt_create(); if (calyptia_metrics_from_lua(ins, ctx->lua->state, new_metrics)) { cmt_destroy(new_metrics); flb_plg_error(ctx->ins, "Failed to decode metrics from lua"); ret = FLB_PROCESSOR_FAILURE; - } else { + } + else { *out_context = new_metrics; } } diff --git a/plugins/processor_calyptia/calyptia_metrics_from_lua.c b/plugins/processor_calyptia/calyptia_metrics_from_lua.c index 411b8bca62f..059af879637 100644 --- a/plugins/processor_calyptia/calyptia_metrics_from_lua.c +++ b/plugins/processor_calyptia/calyptia_metrics_from_lua.c @@ -40,14 +40,16 @@ static void split_fqname(const char *fqname, struct metrics_header *header) if (!header->subsystem) { header->name = header->fqname_buf; header->ns = ""; - } else { + } + else { *header->subsystem = 0; /* split */ header->subsystem++; header->name = strchr(header->subsystem, '_'); if (!header->name) { header->name = header->subsystem; header->subsystem = ""; - } else { + } + else { *header->name = 0; /* split */ header->name++; } @@ -109,9 +111,11 @@ static int double_cmp(const void *a, const void *b) if (x < y) { return -1; - } else if (x > y) { + } + else if (x > y) { return 1; - } else { + } + else { return 0; } } @@ -246,7 +250,8 @@ static char **append_label(char **labels, size_t *labels_size, if (*label_index == *labels_size) { if (!*labels_size) { *labels_size = 8; - } else { + } + else { *labels_size *= 2; } labels = realloc(labels, *labels_size * sizeof(char *)); @@ -376,15 +381,20 @@ int calyptia_metrics_from_lua(struct flb_processor_instance *ins, lua_State *L, if (!strcasecmp(metric_type, "COUNTER")) { type = CMT_COUNTER; - } else if (!strcasecmp(metric_type, "GAUGE")) { + } + else if (!strcasecmp(metric_type, "GAUGE")) { type = CMT_GAUGE; - } else if (!strcasecmp(metric_type, "SUMMARY")) { + } + else if (!strcasecmp(metric_type, "SUMMARY")) { type = CMT_SUMMARY; - } else if (!strcasecmp(metric_type, "HISTOGRAM")) { + } + else if (!strcasecmp(metric_type, "HISTOGRAM")) { type = CMT_HISTOGRAM; - } else if (!strcasecmp(metric_type, "UNTYPED")) { + } + else if (!strcasecmp(metric_type, "UNTYPED")) { type = CMT_UNTYPED; - } else { + } + else { cmt_destroy(cmt); flb_plg_error(ins, "invalid metric type: \"%s\"", metric_type); return -1; @@ -496,7 +506,8 @@ int calyptia_metrics_from_lua(struct flb_processor_instance *ins, lua_State *L, return -1; } - } else if (type == CMT_HISTOGRAM) { + } + else if (type == CMT_HISTOGRAM) { lua_getfield(L, -1, "buckets"); bucket_values @@ -509,7 +520,8 @@ int calyptia_metrics_from_lua(struct flb_processor_instance *ins, lua_State *L, return -1; } - } else { + } + else { lua_getfield(L, -1, "value"); value = lua_to_double(L, -1); @@ -521,13 +533,15 @@ int calyptia_metrics_from_lua(struct flb_processor_instance *ins, lua_State *L, label_vals)) { return -1; } - } else if (type == CMT_GAUGE) { + } + else if (type == CMT_GAUGE) { if (cmt_gauge_set(gauge, timestamp, value, label_vals ? label_count : 0, label_vals)) { return -1; } - } else { + } + else { if (cmt_untyped_set(untyped, timestamp, value, label_vals ? label_count : 0, label_vals)) { @@ -542,7 +556,8 @@ int calyptia_metrics_from_lua(struct flb_processor_instance *ins, lua_State *L, if (type == CMT_SUMMARY) { free(quantile_values); - } else if (type == CMT_HISTOGRAM) { + } + else if (type == CMT_HISTOGRAM) { free(bucket_values); } @@ -553,7 +568,8 @@ int calyptia_metrics_from_lua(struct flb_processor_instance *ins, lua_State *L, if (type == CMT_SUMMARY) { free(quantiles); - } else if (type == CMT_HISTOGRAM) { + } + else if (type == CMT_HISTOGRAM) { free(buckets); } diff --git a/plugins/processor_calyptia/calyptia_metrics_to_lua.c b/plugins/processor_calyptia/calyptia_metrics_to_lua.c index 6e1768fc562..2fc090d9d52 100644 --- a/plugins/processor_calyptia/calyptia_metrics_to_lua.c +++ b/plugins/processor_calyptia/calyptia_metrics_to_lua.c @@ -21,7 +21,8 @@ static void push_uint64(lua_State *L, uint64_t val) if (val > DOUBLE_MAX_SAFE_INTEGER || val < DOUBLE_MIN_SAFE_INTEGER) { snprintf(buf, sizeof(buf), "%" PRIu64, val); lua_pushstring(L, buf); - } else { + } + else { lua_pushnumber(L, val); } } @@ -61,7 +62,8 @@ static void push_histogram(lua_State *L, struct cmt_map *map, for (i = 0; i <= bucket->count; i++) { if (i < bucket->count) { lua_pushnumber(L, bucket->upper_bounds[i]); - } else { + } + else { lua_pushnumber(L, INFINITY); } push_uint64(L, cmt_metric_hist_get_value(metric, i)); @@ -115,9 +117,11 @@ static void push_metric(lua_State *L, struct cmt_map *map, push_timestamp(L, map, metric); if (map->type == CMT_HISTOGRAM) { push_histogram(L, map, metric); - } else if (map->type == CMT_SUMMARY) { + } + else if (map->type == CMT_SUMMARY) { push_summary(L, map, metric); - } else { + } + else { push_counter_gauge_untyped(L, map, metric); } diff --git a/plugins/processor_calyptia/calyptia_traces.c b/plugins/processor_calyptia/calyptia_traces.c index e5b0bd5851a..a780bddbeac 100644 --- a/plugins/processor_calyptia/calyptia_traces.c +++ b/plugins/processor_calyptia/calyptia_traces.c @@ -59,10 +59,12 @@ int calyptia_process_traces(struct flb_processor_instance *ins, l_code = (int) lua_tointeger(ctx->lua->state, -3); if (l_code == -1) { drop_traces(traces_context); - } else if (l_code == 0) { + } + else if (l_code == 0) { /* don't touch the traces */ goto cleanup; - } else { + } + else { assert(l_code == 1); drop_traces(traces_context); diff --git a/plugins/processor_calyptia/lua_to_cfl.c b/plugins/processor_calyptia/lua_to_cfl.c index e5d0697ac38..4229508b8e8 100644 --- a/plugins/processor_calyptia/lua_to_cfl.c +++ b/plugins/processor_calyptia/lua_to_cfl.c @@ -21,9 +21,11 @@ double lua_to_double(lua_State *L, int index) int type = lua_type(L, index); if (type == LUA_TNUMBER) { return lua_tonumber(L, index); - } else if (type == LUA_TSTRING) { + } + else if (type == LUA_TSTRING) { return atof(lua_tostring(L, index)); - } else { + } + else { return 0.0; } } @@ -33,9 +35,11 @@ uint64_t lua_to_uint(lua_State *L) int type = lua_type(L, -1); if (type == LUA_TNUMBER) { return lua_tointeger(L, -1); - } else if (type == LUA_TSTRING) { + } + else if (type == LUA_TSTRING) { return strtoull(lua_tostring(L, -1), NULL, 10); - } else { + } + else { return 0; } } @@ -45,9 +49,11 @@ long lua_to_int(lua_State *L) int type = lua_type(L, -1); if (type == LUA_TNUMBER) { return lua_tointeger(L, -1); - } else if (type == LUA_TSTRING) { + } + else if (type == LUA_TSTRING) { return strtol(lua_tostring(L, -1), NULL, 10); - } else { + } + else { return 0; } } @@ -108,7 +114,8 @@ struct cfl_variant *lua_to_variant(lua_State *L, int index) case LUA_TNUMBER: if (lua_isinteger(L, index)) { return cfl_variant_create_from_int64(lua_tointeger(L, index)); - } else { + } + else { return cfl_variant_create_from_double(lua_tonumber(L, index)); } case LUA_TBOOLEAN: @@ -119,7 +126,8 @@ struct cfl_variant *lua_to_variant(lua_State *L, int index) array_len = flb_lua_arraylength(L, index); if (array_len > 0) { return cfl_variant_create_from_array(lua_array_to_variant(L, array_len)); - } else { + } + else { return cfl_variant_create_from_kvlist(lua_map_to_variant(L)); } default: From a6d891855d5b010bc7f016e2cc6479875328d763 Mon Sep 17 00:00:00 2001 From: Thiago Padilha Date: Mon, 2 Sep 2024 10:48:09 -0300 Subject: [PATCH 03/11] processor_calyptia: remove for loop initial declarations Signed-off-by: Thiago Padilha --- .../calyptia_logs_from_lua.c | 3 +- .../calyptia_metrics_from_lua.c | 30 ++++++++++++------- .../calyptia_metrics_to_lua.c | 3 +- .../calyptia_traces_from_lua.c | 15 ++++++---- plugins/processor_calyptia/lua_to_cfl.c | 3 +- 5 files changed, 36 insertions(+), 18 deletions(-) diff --git a/plugins/processor_calyptia/calyptia_logs_from_lua.c b/plugins/processor_calyptia/calyptia_logs_from_lua.c index 165772e226a..6bf4e610b6f 100644 --- a/plugins/processor_calyptia/calyptia_logs_from_lua.c +++ b/plugins/processor_calyptia/calyptia_logs_from_lua.c @@ -6,6 +6,7 @@ int calyptia_logs_from_lua(struct flb_processor_instance *ins, lua_State *L, struct flb_mp_chunk_cobj *chunk_cobj) { + int i; int logs_length, metadata_length, timestamps_length; struct flb_mp_chunk_record *record; @@ -41,7 +42,7 @@ int calyptia_logs_from_lua(struct flb_processor_instance *ins, lua_State *L, return -1; } - for (int i = 1; i <= logs_length; i++) { + for (i = 1; i <= logs_length; i++) { record = flb_mp_chunk_record_create(chunk_cobj); if (!record) { flb_plg_error(ins, "failed to create record"); diff --git a/plugins/processor_calyptia/calyptia_metrics_from_lua.c b/plugins/processor_calyptia/calyptia_metrics_from_lua.c index 059af879637..d2ec29a6687 100644 --- a/plugins/processor_calyptia/calyptia_metrics_from_lua.c +++ b/plugins/processor_calyptia/calyptia_metrics_from_lua.c @@ -21,10 +21,11 @@ struct metrics_header { static void free_labels(char **labels, size_t label_count) { + int i; if (!labels) { return; } - for (int i = 0; i < label_count; i++) { + for (i = 0; i < label_count; i++) { if (labels[i]) { free(labels[i]); } @@ -59,7 +60,8 @@ static void split_fqname(const char *fqname, struct metrics_header *header) static int assign_label(char **keys, size_t key_count, char **values, const char *key, const char *value) { - for (size_t i = 0; i < key_count; i++) { + size_t i; + for (i = 0; i < key_count; i++) { if (!strcmp(keys[i], key)) { values[i] = strdup(value); if (!values[i]) { @@ -124,13 +126,14 @@ static double *lua_to_quantile_values(struct flb_processor_instance *ins, lua_State *L, double *quantile_keys, int count) { + int i; double *quantile_values = calloc(count, sizeof(*quantile_values)); if (!quantile_values) { flb_plg_error(ins, "could not allocate memory for quantile values"); return NULL; } - for (int i = 0; i < count; i++) { + for (i = 0; i < count; i++) { lua_pushnumber(L, quantile_keys[i]); lua_gettable(L, -2); quantile_values[i] = lua_to_double(L, -1); @@ -144,13 +147,14 @@ static uint64_t *lua_to_bucket_values(struct flb_processor_instance *ins, lua_State *L, double *bucket_keys, int count) { + int i; uint64_t *values = calloc(count, sizeof(*values)); if (!values) { flb_plg_error(ins, "could not allocate memory for bucket values"); return NULL; } - for (int i = 0; i < count; i++) { + for (i = 0; i < count; i++) { lua_pushnumber(L, bucket_keys[i]); lua_gettable(L, -2); values[i] = lua_to_uint(L); @@ -163,6 +167,7 @@ static uint64_t *lua_to_bucket_values(struct flb_processor_instance *ins, static double *lua_to_quantiles_buckets(struct flb_processor_instance *ins, lua_State *L, int *count) { + int i; double *keys; *count = 0; if (lua_type(L, -1) != LUA_TTABLE) { @@ -183,7 +188,7 @@ static double *lua_to_quantiles_buckets(struct flb_processor_instance *ins, } lua_pushnil(L); // first key - int i = 0; + i = 0; while (lua_next(L, -2) != 0) { keys[i] = lua_to_double(L, -2); i++; @@ -200,6 +205,7 @@ static double *lua_to_quantile_bucket_keys(struct flb_processor_instance *ins, lua_State *L, const char *kind, int *count) { + int i; int sample_count; double *keys; @@ -214,7 +220,7 @@ static double *lua_to_quantile_bucket_keys(struct flb_processor_instance *ins, int found = 0; /* find the first sample that has quantiles */ - for (int i = 1; i <= sample_count; i++) { + for (i = 1; i <= sample_count; i++) { lua_rawgeti(L, -1, i); lua_getfield(L, -1, kind); if (lua_type(L, -1) == LUA_TTABLE) { @@ -240,7 +246,8 @@ static double *lua_to_quantile_bucket_keys(struct flb_processor_instance *ins, static char **append_label(char **labels, size_t *labels_size, size_t *label_index, const char *label) { - for (size_t i = 0; i < *label_index; i++) { + size_t i; + for (i = 0; i < *label_index; i++) { if (!strcmp(labels[i], label)) { /* don't do anything if the label is already in the array */ return labels; @@ -272,6 +279,7 @@ static char **append_label(char **labels, size_t *labels_size, static char **lua_to_label_keys(struct flb_processor_instance *ins, lua_State *L, int *label_count) { + int i; int sample_count; char **label_keys; size_t label_index; @@ -295,7 +303,7 @@ static char **lua_to_label_keys(struct flb_processor_instance *ins, labels_size = 0; label_index = 0; - for (int i = 1; i <= sample_count; i++) { + for (i = 1; i <= sample_count; i++) { lua_rawgeti(L, -1, i); if (lua_type(L, -1) != LUA_TTABLE) { free_labels(label_keys, label_index); @@ -357,6 +365,8 @@ int calyptia_metrics_from_lua(struct flb_processor_instance *ins, lua_State *L, char **label_vals; int label_count; uint64_t timestamp; + int i; + int j; if (lua_type(L, -1) != LUA_TTABLE) { flb_plg_error(ins, "expected metrics array"); @@ -365,7 +375,7 @@ int calyptia_metrics_from_lua(struct flb_processor_instance *ins, lua_State *L, metric_count = lua_objlen(L, -1); - for (int i = 1; i <= metric_count; i++) { + for (i = 1; i <= metric_count; i++) { lua_rawgeti(L, -1, i); label_keys = lua_to_label_keys(ins, L, &label_count); @@ -465,7 +475,7 @@ int calyptia_metrics_from_lua(struct flb_processor_instance *ins, lua_State *L, return -1; } - for (int j = 1; j <= sample_count; j++) { + for (j = 1; j <= sample_count; j++) { label_vals = NULL; /* get sample */ diff --git a/plugins/processor_calyptia/calyptia_metrics_to_lua.c b/plugins/processor_calyptia/calyptia_metrics_to_lua.c index 2fc090d9d52..5b2be51db74 100644 --- a/plugins/processor_calyptia/calyptia_metrics_to_lua.c +++ b/plugins/processor_calyptia/calyptia_metrics_to_lua.c @@ -81,6 +81,7 @@ static void push_histogram(lua_State *L, struct cmt_map *map, static void push_summary(lua_State *L, struct cmt_map *map, struct cmt_metric *metric) { + int i; struct cmt_summary *summary; struct cmt_opts *opts; @@ -89,7 +90,7 @@ static void push_summary(lua_State *L, struct cmt_map *map, if (metric->sum_quantiles_set) { lua_createtable(L, summary->quantiles_count, 0); - for (int i = 0; i < summary->quantiles_count; i++) { + for (i = 0; i < summary->quantiles_count; i++) { lua_pushnumber(L, summary->quantiles[i]); lua_pushnumber(L, cmt_summary_quantile_get_value(metric, i)); lua_settable(L, -3); diff --git a/plugins/processor_calyptia/calyptia_traces_from_lua.c b/plugins/processor_calyptia/calyptia_traces_from_lua.c index 0e210f8ab65..1df036aa55c 100644 --- a/plugins/processor_calyptia/calyptia_traces_from_lua.c +++ b/plugins/processor_calyptia/calyptia_traces_from_lua.c @@ -26,6 +26,7 @@ static void lua_to_links(lua_State *L, struct ctrace_span *span) { struct ctrace_link *link; size_t count; + size_t i; struct ctrace_id *trace_id; if (lua_type(L, -1) != LUA_TTABLE) { @@ -33,7 +34,7 @@ static void lua_to_links(lua_State *L, struct ctrace_span *span) } count = lua_objlen(L, -1); - for (size_t i = 1; i <= count; i++) { + for (i = 1; i <= count; i++) { lua_rawgeti(L, -1, i); lua_getfield(L, -1, "traceId"); @@ -69,6 +70,7 @@ static void lua_to_events(lua_State *L, struct ctrace_span *span) { struct ctrace_span_event *event; size_t count; + size_t i; const char *name; if (lua_type(L, -1) != LUA_TTABLE) { @@ -76,7 +78,7 @@ static void lua_to_events(lua_State *L, struct ctrace_span *span) } count = lua_objlen(L, -1); - for (size_t i = 1; i <= count; i++) { + for (i = 1; i <= count; i++) { lua_rawgeti(L, -1, i); lua_getfield(L, -1, "name"); @@ -155,6 +157,7 @@ static void lua_to_spans(lua_State *L, struct ctrace *ctx, size_t count; struct ctrace_span *span; cfl_sds_t name; + size_t i; struct ctrace_id *parent_span_id; if (lua_type(L, -1) != LUA_TTABLE) { @@ -162,7 +165,7 @@ static void lua_to_spans(lua_State *L, struct ctrace *ctx, } count = lua_objlen(L, -1); - for (size_t i = 1; i <= count; i++) { + for (i = 1; i <= count; i++) { lua_rawgeti(L, -1, i); lua_getfield(L, -1, "name"); @@ -225,6 +228,7 @@ static void lua_to_scope_spans(lua_State *L, struct ctrace *ctx, struct ctrace_resource_span *resource_span) { size_t count; + size_t i; struct ctrace_scope_span *scope_span; if (lua_type(L, -1) != LUA_TTABLE) { @@ -233,7 +237,7 @@ static void lua_to_scope_spans(lua_State *L, struct ctrace *ctx, count = lua_objlen(L, -1); - for (size_t i = 1; i <= count; i++) { + for (i = 1; i <= count; i++) { scope_span = ctr_scope_span_create(resource_span); lua_rawgeti(L, -1, i); @@ -258,6 +262,7 @@ static void lua_to_scope_spans(lua_State *L, struct ctrace *ctx, static void lua_to_resource_spans(lua_State *L, struct ctrace *ctx) { size_t count; + size_t i; struct ctrace_resource_span *resource_span; if (lua_type(L, -1) != LUA_TTABLE) { @@ -266,7 +271,7 @@ static void lua_to_resource_spans(lua_State *L, struct ctrace *ctx) count = lua_objlen(L, -1); - for (size_t i = 1; i <= count; i++) { + for (i = 1; i <= count; i++) { resource_span = ctr_resource_span_create(ctx); lua_rawgeti(L, -1, i); diff --git a/plugins/processor_calyptia/lua_to_cfl.c b/plugins/processor_calyptia/lua_to_cfl.c index 4229508b8e8..d96fd905cac 100644 --- a/plugins/processor_calyptia/lua_to_cfl.c +++ b/plugins/processor_calyptia/lua_to_cfl.c @@ -76,9 +76,10 @@ bool lua_isinteger(lua_State *L, int index) struct cfl_array *lua_array_to_variant(lua_State *L, int array_len) { + int i; struct cfl_array *array = cfl_array_create(array_len); - for (int i = 1; i <= array_len; i++) { + for (i = 1; i <= array_len; i++) { lua_rawgeti(L, -1, i); struct cfl_variant *variant = lua_to_variant(L, -1); cfl_array_append(array, variant); From a8c46d595de3e3cd0c93b755e7e653adb4ea8064 Mon Sep 17 00:00:00 2001 From: Thiago Padilha Date: Mon, 2 Sep 2024 11:23:05 -0300 Subject: [PATCH 04/11] ci: disable processor_calyptia Signed-off-by: Thiago Padilha --- fluent-bit_git.bb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fluent-bit_git.bb b/fluent-bit_git.bb index fd17c4f2565..f0308759457 100644 --- a/fluent-bit_git.bb +++ b/fluent-bit_git.bb @@ -36,8 +36,8 @@ OECMAKE_GENERATOR ?= "Unix Makefiles" # Host related setup EXTRA_OECMAKE += "-DGNU_HOST=${HOST_SYS} " -# Disable LuaJIT and filter_lua support -EXTRA_OECMAKE += "-DFLB_LUAJIT=Off -DFLB_FILTER_LUA=Off " +# Disable LuaJIT, filter_lua and processor_calyptia support +EXTRA_OECMAKE += "-DFLB_LUAJIT=Off -DFLB_FILTER_LUA=Off -DFLB_PROCESSOR_CALYPTIA=Off " # Disable Library and examples EXTRA_OECMAKE += "-DFLB_SHARED_LIB=Off -DFLB_EXAMPLES=Off " From fe69b05417172eb9791d84499f70030f0c6b6f3e Mon Sep 17 00:00:00 2001 From: Thiago Padilha Date: Mon, 2 Sep 2024 14:10:18 -0300 Subject: [PATCH 05/11] processor_calyptia: add examples Signed-off-by: Thiago Padilha --- .../logs_empty_key/expected_output | 1 + .../logs_empty_key/fluent-bit.yaml | 19 +++++ .../logs_empty_key/script.lua | 8 ++ .../logs_modify_key/expected_output | 1 + .../logs_modify_key/fluent-bit.yaml | 23 +++++ .../logs_modify_key/script.lua | 9 ++ .../logs_modify_metadata/expected_output | 1 + .../logs_modify_metadata/fluent-bit.yaml | 18 ++++ .../logs_modify_metadata/script.lua | 3 + .../logs_split_drop/expected_output | 4 + .../logs_split_drop/fluent-bit.yaml | 18 ++++ .../logs_split_drop/script.lua | 11 +++ .../expected_output | 9 ++ .../fluent-bit.yaml | 18 ++++ .../script.lua | 23 +++++ .../metrics_modify_name/expected_output | 8 ++ .../metrics_modify_name/fluent-bit.yaml | 18 ++++ .../metrics_modify_name/script.lua | 14 ++++ .../expected_output | 8 ++ .../fluent-bit.yaml | 18 ++++ .../metrics_modify_name_output/script.lua | 14 ++++ .../traces_modify_metadata/expected_output | 15 ++++ .../traces_modify_metadata/fluent-bit.yaml | 18 ++++ .../traces_modify_metadata/script.lua | 11 +++ .../expected_output | 15 ++++ .../fluent-bit.yaml | 18 ++++ .../traces_modify_metadata_output/script.lua | 11 +++ .../traces_modify_split_span/expected_output | 84 +++++++++++++++++++ .../traces_modify_split_span/fluent-bit.yaml | 18 ++++ .../traces_modify_split_span/script.lua | 40 +++++++++ 30 files changed, 476 insertions(+) create mode 100644 examples/processor_calyptia/logs_empty_key/expected_output create mode 100644 examples/processor_calyptia/logs_empty_key/fluent-bit.yaml create mode 100644 examples/processor_calyptia/logs_empty_key/script.lua create mode 100644 examples/processor_calyptia/logs_modify_key/expected_output create mode 100644 examples/processor_calyptia/logs_modify_key/fluent-bit.yaml create mode 100644 examples/processor_calyptia/logs_modify_key/script.lua create mode 100644 examples/processor_calyptia/logs_modify_metadata/expected_output create mode 100644 examples/processor_calyptia/logs_modify_metadata/fluent-bit.yaml create mode 100644 examples/processor_calyptia/logs_modify_metadata/script.lua create mode 100644 examples/processor_calyptia/logs_split_drop/expected_output create mode 100644 examples/processor_calyptia/logs_split_drop/fluent-bit.yaml create mode 100644 examples/processor_calyptia/logs_split_drop/script.lua create mode 100644 examples/processor_calyptia/metrics_add_remove_sample_label/expected_output create mode 100644 examples/processor_calyptia/metrics_add_remove_sample_label/fluent-bit.yaml create mode 100644 examples/processor_calyptia/metrics_add_remove_sample_label/script.lua create mode 100644 examples/processor_calyptia/metrics_modify_name/expected_output create mode 100644 examples/processor_calyptia/metrics_modify_name/fluent-bit.yaml create mode 100644 examples/processor_calyptia/metrics_modify_name/script.lua create mode 100644 examples/processor_calyptia/metrics_modify_name_output/expected_output create mode 100644 examples/processor_calyptia/metrics_modify_name_output/fluent-bit.yaml create mode 100644 examples/processor_calyptia/metrics_modify_name_output/script.lua create mode 100644 examples/processor_calyptia/traces_modify_metadata/expected_output create mode 100644 examples/processor_calyptia/traces_modify_metadata/fluent-bit.yaml create mode 100644 examples/processor_calyptia/traces_modify_metadata/script.lua create mode 100644 examples/processor_calyptia/traces_modify_metadata_output/expected_output create mode 100644 examples/processor_calyptia/traces_modify_metadata_output/fluent-bit.yaml create mode 100644 examples/processor_calyptia/traces_modify_metadata_output/script.lua create mode 100644 examples/processor_calyptia/traces_modify_split_span/expected_output create mode 100644 examples/processor_calyptia/traces_modify_split_span/fluent-bit.yaml create mode 100644 examples/processor_calyptia/traces_modify_split_span/script.lua diff --git a/examples/processor_calyptia/logs_empty_key/expected_output b/examples/processor_calyptia/logs_empty_key/expected_output new file mode 100644 index 00000000000..164acda0c2f --- /dev/null +++ b/examples/processor_calyptia/logs_empty_key/expected_output @@ -0,0 +1 @@ +reportingInstance"=>"EMPTY \ No newline at end of file diff --git a/examples/processor_calyptia/logs_empty_key/fluent-bit.yaml b/examples/processor_calyptia/logs_empty_key/fluent-bit.yaml new file mode 100644 index 00000000000..c241f1be034 --- /dev/null +++ b/examples/processor_calyptia/logs_empty_key/fluent-bit.yaml @@ -0,0 +1,19 @@ +service: + flush: 1 + log_level: error + +pipeline: + inputs: + - name: dummy + tag: event + dummy: |- + {"involvedObject":{"uid":"33146c89-69d9-4d99-b8f9-f256278732c9","kind":"Pod","resourceVersion":"381","apiVersion":"v1","name":"calyptia-core-controller-manager-7d64bbcdbc-fdpd9","namespace":"calyptia"},"reportingComponent":"default-scheduler","reportingInstance":"","reason":"FailedScheduling","type":"Warning","message":"0/1 nodes are available: 1 node(s) had untolerated taint {node.kubernetes.io/not-ready: }. preemption: 0/1 nodes are available: 1 Preemption is not helpful for scheduling.","source":{"component":"default-scheduler"},"lastTimestamp":"2024-07-04T12:14:48Z","firstTimestamp":"2024-07-04T12:14:48Z","metadata":{"uid":"0f537d3a-df6d-478f-9ead-54c0be200ea7","namespace":"calyptia","resourceVersion":"403","creationTimestamp":"2024-07-04T12:14:48Z","name":"calyptia-core-controller-manager-7d64bbcdbc-fdpd9.17df018d10d3fc8a","managedFields":[{"manager":"kube-scheduler","operation":"Update","apiVersion":"v1","fieldsType":"FieldsV1","time":"2024-07-04T12:14:48Z"}]}} + processors: + logs: + - name: calyptia + script: script.lua + call: process_logs + + outputs: + - name: stdout + match: event diff --git a/examples/processor_calyptia/logs_empty_key/script.lua b/examples/processor_calyptia/logs_empty_key/script.lua new file mode 100644 index 00000000000..be314b0c423 --- /dev/null +++ b/examples/processor_calyptia/logs_empty_key/script.lua @@ -0,0 +1,8 @@ +function process_logs(tag, ts, log) + for k, v in pairs(log) do + if v == "" then + log[k] = 'EMPTY' + end + end + return MODIFY, ts, log +end diff --git a/examples/processor_calyptia/logs_modify_key/expected_output b/examples/processor_calyptia/logs_modify_key/expected_output new file mode 100644 index 00000000000..b8092269622 --- /dev/null +++ b/examples/processor_calyptia/logs_modify_key/expected_output @@ -0,0 +1 @@ +{}], {"extra_data"=>["null", false, "false", true, nil, -4, "ba4", 4, 1]}] diff --git a/examples/processor_calyptia/logs_modify_key/fluent-bit.yaml b/examples/processor_calyptia/logs_modify_key/fluent-bit.yaml new file mode 100644 index 00000000000..9c56be9ade2 --- /dev/null +++ b/examples/processor_calyptia/logs_modify_key/fluent-bit.yaml @@ -0,0 +1,23 @@ +service: + flush: 0.2 + log_level: error + +pipeline: + inputs: + - name: event_type + tag: event + type: logs + processors: + logs: + - name: calyptia + script: script.lua + call: process_logs + # A JSON/YAML object can be passed as "opts" to the script. + # See "script.lua" for an example of how it can be used + opts: + key: extra_data + value: ["null", false, "false", true, null, -4, ba4, 4., 1] + + outputs: + - name: stdout + match: event diff --git a/examples/processor_calyptia/logs_modify_key/script.lua b/examples/processor_calyptia/logs_modify_key/script.lua new file mode 100644 index 00000000000..e4601a7aa8d --- /dev/null +++ b/examples/processor_calyptia/logs_modify_key/script.lua @@ -0,0 +1,9 @@ +-- Load the "opts" object passed in fluent-bit.yaml config into a local +-- variable +local opts = ... + +function process_logs(tag, ts, log) + log.event_type = nil + log[opts.key] = opts.value + return MODIFY, ts, log +end diff --git a/examples/processor_calyptia/logs_modify_metadata/expected_output b/examples/processor_calyptia/logs_modify_metadata/expected_output new file mode 100644 index 00000000000..617dd318f76 --- /dev/null +++ b/examples/processor_calyptia/logs_modify_metadata/expected_output @@ -0,0 +1 @@ +{"hello"=>"world"}], {"event_type"=>"some logs"}] diff --git a/examples/processor_calyptia/logs_modify_metadata/fluent-bit.yaml b/examples/processor_calyptia/logs_modify_metadata/fluent-bit.yaml new file mode 100644 index 00000000000..f1eeee60f1b --- /dev/null +++ b/examples/processor_calyptia/logs_modify_metadata/fluent-bit.yaml @@ -0,0 +1,18 @@ +service: + flush: 1 + log_level: error + +pipeline: + inputs: + - name: event_type + tag: event + type: logs + processors: + logs: + - name: calyptia + script: script.lua + call: process_logs + + outputs: + - name: stdout + match: event diff --git a/examples/processor_calyptia/logs_modify_metadata/script.lua b/examples/processor_calyptia/logs_modify_metadata/script.lua new file mode 100644 index 00000000000..c18fabcb933 --- /dev/null +++ b/examples/processor_calyptia/logs_modify_metadata/script.lua @@ -0,0 +1,3 @@ +function process_logs(tag, ts, log) + return MODIFY, ts, log, {hello = 'world'} +end diff --git a/examples/processor_calyptia/logs_split_drop/expected_output b/examples/processor_calyptia/logs_split_drop/expected_output new file mode 100644 index 00000000000..f28abd0c5a6 --- /dev/null +++ b/examples/processor_calyptia/logs_split_drop/expected_output @@ -0,0 +1,4 @@ +[0] event: [[0.000000000, {"index"=>6}], {"event_type"=>"some logs 0"}] +[1] event: [[10.000000000, {"index"=>6}], {"event_type"=>"some logs 0"}] +[0] event: [[2.000000000, {"index"=>8}], {"event_type"=>"some logs 2"}] +[1] event: [[12.000000000, {"index"=>8}], {"event_type"=>"some logs 2"}] diff --git a/examples/processor_calyptia/logs_split_drop/fluent-bit.yaml b/examples/processor_calyptia/logs_split_drop/fluent-bit.yaml new file mode 100644 index 00000000000..f1eeee60f1b --- /dev/null +++ b/examples/processor_calyptia/logs_split_drop/fluent-bit.yaml @@ -0,0 +1,18 @@ +service: + flush: 1 + log_level: error + +pipeline: + inputs: + - name: event_type + tag: event + type: logs + processors: + logs: + - name: calyptia + script: script.lua + call: process_logs + + outputs: + - name: stdout + match: event diff --git a/examples/processor_calyptia/logs_split_drop/script.lua b/examples/processor_calyptia/logs_split_drop/script.lua new file mode 100644 index 00000000000..9325eb74b1a --- /dev/null +++ b/examples/processor_calyptia/logs_split_drop/script.lua @@ -0,0 +1,11 @@ +local i = 0 + +function process_logs(tag, ts, log) + ts = i + log.event_type = log.event_type .. " " .. i + i = i + 1 + if i % 2 == 0 then + return DROP + end + return MODIFY, {ts, ts + 10}, {log, log}, {index = i + 5} +end diff --git a/examples/processor_calyptia/metrics_add_remove_sample_label/expected_output b/examples/processor_calyptia/metrics_add_remove_sample_label/expected_output new file mode 100644 index 00000000000..7eb769b1403 --- /dev/null +++ b/examples/processor_calyptia/metrics_add_remove_sample_label/expected_output @@ -0,0 +1,9 @@ +1970-01-01T00:00:00.000000000Z kubernetes_network_load_counter = 3 +1970-01-01T00:00:00.000000000Z kubernetes_network_load_counter{app="cmetrics"} = 1 +1970-01-01T00:00:00.000000000Z kubernetes_network_load_counter{app="test"} = 12.15 +1970-01-01T00:00:00.000000000Z kubernetes_network_load_gauge = 1 +1970-01-01T00:00:00.000000000Z kubernetes_network_load_gauge{hello="world"} = 42 +1970-01-01T00:00:00.000000000Z k8s_disk_load_summary = { quantiles = { 0.1=1.1, 0.2=2.2, 0.3=3.3, 0.4=4.4, 0.5=5.5 }, sum=51.6129, count=10 } +1970-01-01T00:00:00.000000000Z k8s_disk_load_summary{my_label="my_val"} = { quantiles = { 0.1=11.11, 0.2=0, 0.3=33.33, 0.4=44.44, 0.5=55.55 }, sum=51.6129, count=10 } +1970-01-01T00:00:00.000000000Z k8s_network_load_histogram = { buckets = { 0.05=2, 5=3, 10=4, +Inf=5 }, sum=1013.02, count=5 } +1970-01-01T00:00:00.000000000Z k8s_network_load_histogram{my_label="my_val"} = { buckets = { 0.05=2, 5=3, 10=4, +Inf=5 }, sum=1013.02, count=5 } diff --git a/examples/processor_calyptia/metrics_add_remove_sample_label/fluent-bit.yaml b/examples/processor_calyptia/metrics_add_remove_sample_label/fluent-bit.yaml new file mode 100644 index 00000000000..3fe7496521d --- /dev/null +++ b/examples/processor_calyptia/metrics_add_remove_sample_label/fluent-bit.yaml @@ -0,0 +1,18 @@ +service: + flush: 0.2 + log_level: error + +pipeline: + inputs: + - name: event_type + tag: event + type: metrics + processors: + metrics: + - name: calyptia + script: script.lua + call: process_metrics + + outputs: + - name: stdout + match: event diff --git a/examples/processor_calyptia/metrics_add_remove_sample_label/script.lua b/examples/processor_calyptia/metrics_add_remove_sample_label/script.lua new file mode 100644 index 00000000000..2302b923c79 --- /dev/null +++ b/examples/processor_calyptia/metrics_add_remove_sample_label/script.lua @@ -0,0 +1,23 @@ +function process_metrics(tag, ts, metric) + for _, sample in ipairs(metric.metrics) do + sample.timestamp = 0 + end + + if metric.name == 'kubernetes_network_load_counter' then + for _, sample in ipairs(metric.metrics) do + if sample.labels then + sample.labels.hostname = nil + end + end + end + if metric.name == 'kubernetes_network_load_gauge' then + table.insert(metric.metrics, { + timestamp = 0, + value = 42, + labels = { + hello = 'world' + } + }) + end + return MODIFY, ts, metric +end diff --git a/examples/processor_calyptia/metrics_modify_name/expected_output b/examples/processor_calyptia/metrics_modify_name/expected_output new file mode 100644 index 00000000000..5790aa7649e --- /dev/null +++ b/examples/processor_calyptia/metrics_modify_name/expected_output @@ -0,0 +1,8 @@ +1970-01-01T00:00:00.000000100Z metric_kubernetes_network_load_counter = 3 +1970-01-01T00:00:00.000000100Z metric_kubernetes_network_load_counter{app="cmetrics"} = 1 +1970-01-01T00:00:00.000000100Z metric_kubernetes_network_load_counter{app="test"} = 12.15 +1970-01-01T00:00:00.000000100Z metric_kubernetes_network_load_gauge = 1 +1970-01-01T00:00:00.000000100Z metric_k8s_disk_load_summary = { quantiles = { 0.1=1.1, 0.2=2.2, 0.3=3.3, 0.4=4.4, 0.5=5.5 }, sum=51.6129, count=10 } +1970-01-01T00:00:00.000000100Z metric_k8s_disk_load_summary{my_label="my_val"} = { quantiles = { 0.1=11.11, 0.2=0, 0.3=33.33, 0.4=44.44, 0.5=55.55 }, sum=51.6129, count=10 } +1970-01-01T00:00:00.000000100Z metric_k8s_network_load_histogram = { buckets = { 0.05=2, 5=3, 10=4, +Inf=5 }, sum=1013.02, count=5 } +1970-01-01T00:00:00.000000100Z metric_k8s_network_load_histogram{my_label="my_val"} = { buckets = { 0.05=2, 5=3, 10=4, +Inf=5 }, sum=1013.02, count=5 } diff --git a/examples/processor_calyptia/metrics_modify_name/fluent-bit.yaml b/examples/processor_calyptia/metrics_modify_name/fluent-bit.yaml new file mode 100644 index 00000000000..3fe7496521d --- /dev/null +++ b/examples/processor_calyptia/metrics_modify_name/fluent-bit.yaml @@ -0,0 +1,18 @@ +service: + flush: 0.2 + log_level: error + +pipeline: + inputs: + - name: event_type + tag: event + type: metrics + processors: + metrics: + - name: calyptia + script: script.lua + call: process_metrics + + outputs: + - name: stdout + match: event diff --git a/examples/processor_calyptia/metrics_modify_name/script.lua b/examples/processor_calyptia/metrics_modify_name/script.lua new file mode 100644 index 00000000000..3c082baee95 --- /dev/null +++ b/examples/processor_calyptia/metrics_modify_name/script.lua @@ -0,0 +1,14 @@ +function process_metrics(tag, ts, metric) + for _, sample in ipairs(metric.metrics) do + sample.timestamp = 100 + end + if metric.name == 'kubernetes_network_load_counter' then + for _, sample in ipairs(metric.metrics) do + if sample.labels then + sample.labels.hostname = nil + end + end + end + metric.name = 'metric_' .. metric.name + return MODIFY, ts, metric +end diff --git a/examples/processor_calyptia/metrics_modify_name_output/expected_output b/examples/processor_calyptia/metrics_modify_name_output/expected_output new file mode 100644 index 00000000000..5790aa7649e --- /dev/null +++ b/examples/processor_calyptia/metrics_modify_name_output/expected_output @@ -0,0 +1,8 @@ +1970-01-01T00:00:00.000000100Z metric_kubernetes_network_load_counter = 3 +1970-01-01T00:00:00.000000100Z metric_kubernetes_network_load_counter{app="cmetrics"} = 1 +1970-01-01T00:00:00.000000100Z metric_kubernetes_network_load_counter{app="test"} = 12.15 +1970-01-01T00:00:00.000000100Z metric_kubernetes_network_load_gauge = 1 +1970-01-01T00:00:00.000000100Z metric_k8s_disk_load_summary = { quantiles = { 0.1=1.1, 0.2=2.2, 0.3=3.3, 0.4=4.4, 0.5=5.5 }, sum=51.6129, count=10 } +1970-01-01T00:00:00.000000100Z metric_k8s_disk_load_summary{my_label="my_val"} = { quantiles = { 0.1=11.11, 0.2=0, 0.3=33.33, 0.4=44.44, 0.5=55.55 }, sum=51.6129, count=10 } +1970-01-01T00:00:00.000000100Z metric_k8s_network_load_histogram = { buckets = { 0.05=2, 5=3, 10=4, +Inf=5 }, sum=1013.02, count=5 } +1970-01-01T00:00:00.000000100Z metric_k8s_network_load_histogram{my_label="my_val"} = { buckets = { 0.05=2, 5=3, 10=4, +Inf=5 }, sum=1013.02, count=5 } diff --git a/examples/processor_calyptia/metrics_modify_name_output/fluent-bit.yaml b/examples/processor_calyptia/metrics_modify_name_output/fluent-bit.yaml new file mode 100644 index 00000000000..8795397bab6 --- /dev/null +++ b/examples/processor_calyptia/metrics_modify_name_output/fluent-bit.yaml @@ -0,0 +1,18 @@ +service: + flush: 0.2 + log_level: error + +pipeline: + inputs: + - name: event_type + tag: event + type: metrics + + outputs: + - name: stdout + match: event + processors: + metrics: + - name: calyptia + script: script.lua + call: process_metrics diff --git a/examples/processor_calyptia/metrics_modify_name_output/script.lua b/examples/processor_calyptia/metrics_modify_name_output/script.lua new file mode 100644 index 00000000000..3c082baee95 --- /dev/null +++ b/examples/processor_calyptia/metrics_modify_name_output/script.lua @@ -0,0 +1,14 @@ +function process_metrics(tag, ts, metric) + for _, sample in ipairs(metric.metrics) do + sample.timestamp = 100 + end + if metric.name == 'kubernetes_network_load_counter' then + for _, sample in ipairs(metric.metrics) do + if sample.labels then + sample.labels.hostname = nil + end + end + end + metric.name = 'metric_' .. metric.name + return MODIFY, ts, metric +end diff --git a/examples/processor_calyptia/traces_modify_metadata/expected_output b/examples/processor_calyptia/traces_modify_metadata/expected_output new file mode 100644 index 00000000000..44296bb8698 --- /dev/null +++ b/examples/processor_calyptia/traces_modify_metadata/expected_output @@ -0,0 +1,15 @@ +|-------------------- RESOURCE SPAN --------------------| + resource: + - attributes: + - service.name: 'Fluent Bit Test Service' + - dropped_attributes_count: 10 + schema_url: https://www.google2.com + [scope_span] + instrumentation scope: + - name : scope-span + - version : d.e.f + - dropped_attributes_count: 5 + - attributes: + + schema_url: https://www.google.com + [spans] diff --git a/examples/processor_calyptia/traces_modify_metadata/fluent-bit.yaml b/examples/processor_calyptia/traces_modify_metadata/fluent-bit.yaml new file mode 100644 index 00000000000..98e35b47f8c --- /dev/null +++ b/examples/processor_calyptia/traces_modify_metadata/fluent-bit.yaml @@ -0,0 +1,18 @@ +service: + flush: 0.2 + log_level: error + +pipeline: + inputs: + - name: event_type + tag: event + type: traces + processors: + traces: + - name: calyptia + script: script.lua + call: process_traces + + outputs: + - name: stdout + match: event diff --git a/examples/processor_calyptia/traces_modify_metadata/script.lua b/examples/processor_calyptia/traces_modify_metadata/script.lua new file mode 100644 index 00000000000..c7e9028a681 --- /dev/null +++ b/examples/processor_calyptia/traces_modify_metadata/script.lua @@ -0,0 +1,11 @@ +function process_traces(tag, ts, span, metadata) + metadata.resourceSpan.resource.droppedAttributesCount = 10 + metadata.resourceSpan.schemaUrl = 'https://www.google2.com' + metadata.scopeSpan.schemaUrl = 'https://www.google.com' + metadata.scopeSpan.scope.name = 'scope-span' + metadata.scopeSpan.scope.version = 'd.e.f' + metadata.scopeSpan.scope.droppedAttributesCount = 5 + span.startTimeUnixNano = 10 + span.endTimeUnixNano = 20 + return MODIFY, ts, span, metadata +end diff --git a/examples/processor_calyptia/traces_modify_metadata_output/expected_output b/examples/processor_calyptia/traces_modify_metadata_output/expected_output new file mode 100644 index 00000000000..44296bb8698 --- /dev/null +++ b/examples/processor_calyptia/traces_modify_metadata_output/expected_output @@ -0,0 +1,15 @@ +|-------------------- RESOURCE SPAN --------------------| + resource: + - attributes: + - service.name: 'Fluent Bit Test Service' + - dropped_attributes_count: 10 + schema_url: https://www.google2.com + [scope_span] + instrumentation scope: + - name : scope-span + - version : d.e.f + - dropped_attributes_count: 5 + - attributes: + + schema_url: https://www.google.com + [spans] diff --git a/examples/processor_calyptia/traces_modify_metadata_output/fluent-bit.yaml b/examples/processor_calyptia/traces_modify_metadata_output/fluent-bit.yaml new file mode 100644 index 00000000000..6371aec42a0 --- /dev/null +++ b/examples/processor_calyptia/traces_modify_metadata_output/fluent-bit.yaml @@ -0,0 +1,18 @@ +service: + flush: 0.2 + log_level: error + +pipeline: + inputs: + - name: event_type + tag: event + type: traces + + outputs: + - name: stdout + match: event + processors: + traces: + - name: calyptia + script: script.lua + call: process_traces diff --git a/examples/processor_calyptia/traces_modify_metadata_output/script.lua b/examples/processor_calyptia/traces_modify_metadata_output/script.lua new file mode 100644 index 00000000000..c7e9028a681 --- /dev/null +++ b/examples/processor_calyptia/traces_modify_metadata_output/script.lua @@ -0,0 +1,11 @@ +function process_traces(tag, ts, span, metadata) + metadata.resourceSpan.resource.droppedAttributesCount = 10 + metadata.resourceSpan.schemaUrl = 'https://www.google2.com' + metadata.scopeSpan.schemaUrl = 'https://www.google.com' + metadata.scopeSpan.scope.name = 'scope-span' + metadata.scopeSpan.scope.version = 'd.e.f' + metadata.scopeSpan.scope.droppedAttributesCount = 5 + span.startTimeUnixNano = 10 + span.endTimeUnixNano = 20 + return MODIFY, ts, span, metadata +end diff --git a/examples/processor_calyptia/traces_modify_split_span/expected_output b/examples/processor_calyptia/traces_modify_split_span/expected_output new file mode 100644 index 00000000000..48a2034309f --- /dev/null +++ b/examples/processor_calyptia/traces_modify_split_span/expected_output @@ -0,0 +1,84 @@ +|-------------------- RESOURCE SPAN --------------------| + resource: + - attributes: + - service.name: 'Fluent Bit Test Service' + - dropped_attributes_count: 5 + schema_url: https://ctraces/resource_span_schema_url + [scope_span] + instrumentation scope: + - name : ctrace + - version : a.b.c + - dropped_attributes_count: 3 + - attributes: + + schema_url: https://ctraces/scope_span_schema_url + [spans] + [span 'main'] + - trace_id : abababab + - span_id : cdcdcdcd + - parent_span_id : efefefef + - kind : 1 (internal) + - start_time : 100 + - end_time : 200 + - dropped_attributes_count: 0 + - dropped_events_count : 0 + - status: + - code : 0 + - attributes: + - my_array: [ + 'first', + 2, + false, + [ + 3.1000000000000001, + 5.2000000000000002, + 6.2999999999999998 + ] + ] + - events: + - name: connect to remote server + - timestamp : 300 + - dropped_attributes_count: 0 + - attributes: + - syscall 2: 'connect()' + - [links] + [span 'hello-2'] + - trace_id : abababab + - span_id : cdcdcdcd + - parent_span_id : efefefef + - kind : 3 (client) + - start_time : 100 + - end_time : 200 + - dropped_attributes_count: 0 + - dropped_events_count : 0 + - status: + - code : 0 + - attributes: none + - events: none + - [links] + - link: + - trace_id : aaaaaaaa + - span_id : cdcdcdcd + - trace_state : aaabbbccc + - dropped_events_count : 2 + - attributes : none + [span 'hello-2'] + - trace_id : abababab + - span_id : cdcdcdcd + - parent_span_id : efefefef + - kind : 3 (client) + - start_time : 100 + - end_time : 200 + - dropped_attributes_count: 0 + - dropped_events_count : 0 + - status: + - code : 0 + - attributes: none + - events: none + - [links] + - link: + - trace_id : aaaaaaaa + - span_id : cdcdcdcd + - trace_state : aaabbbccc + - dropped_events_count : 2 + - attributes : none diff --git a/examples/processor_calyptia/traces_modify_split_span/fluent-bit.yaml b/examples/processor_calyptia/traces_modify_split_span/fluent-bit.yaml new file mode 100644 index 00000000000..98e35b47f8c --- /dev/null +++ b/examples/processor_calyptia/traces_modify_split_span/fluent-bit.yaml @@ -0,0 +1,18 @@ +service: + flush: 0.2 + log_level: error + +pipeline: + inputs: + - name: event_type + tag: event + type: traces + processors: + traces: + - name: calyptia + script: script.lua + call: process_traces + + outputs: + - name: stdout + match: event diff --git a/examples/processor_calyptia/traces_modify_split_span/script.lua b/examples/processor_calyptia/traces_modify_split_span/script.lua new file mode 100644 index 00000000000..41ce26c44b1 --- /dev/null +++ b/examples/processor_calyptia/traces_modify_split_span/script.lua @@ -0,0 +1,40 @@ +local i = 0 +function process_traces(tag, ts, span, metadata) + i = i + 1 + local split = span.name ~= 'main' + if split then + span.name = 'hello-'..tostring(i) + end + -- modify some random attributes so that the output is deterministic + span.traceId = 'abababab' + span.spanId = 'cdcdcdcd' + span.parentSpanId = 'efefefef' + span.startTimeUnixNano = 100 + span.endTimeUnixNano = 200 + if span.attributes then + for k, _ in pairs(span.attributes) do + -- we can only have 1 key in a map or the output will not be deterministic + -- due to random table iteration order when converting back to C + if k ~= 'my_array' then + span.attributes[k] = nil + end + end + end + if span.events then + span.events[1].timeUnixNano = 300 + for k, _ in pairs(span.events[1].attributes) do + if k ~= 'syscall 2' then + -- we can only have 1 key in a map or the output will not be deterministic + -- due to random table iteration order when converting back to C + span.events[1].attributes[k] = nil + end + end + end + if span.links and span.links[1] then + span.links[1].traceId = 'AAAAAAAA' + end + if split then + return MODIFY, ts, {span, span} + end + return MODIFY, ts, span +end From 7d708e70a6464d04409bd309914911a905cb7ca0 Mon Sep 17 00:00:00 2001 From: Thiago Padilha Date: Mon, 2 Sep 2024 14:47:09 -0300 Subject: [PATCH 06/11] processor_calyptia: fix includes for windows Signed-off-by: Thiago Padilha --- plugins/processor_calyptia/calyptia.c | 10 ++++------ plugins/processor_calyptia/calyptia_logs_to_lua.c | 1 - plugins/processor_calyptia/calyptia_metrics_to_lua.c | 3 ++- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/plugins/processor_calyptia/calyptia.c b/plugins/processor_calyptia/calyptia.c index 0dabae73468..3ac05f8ba84 100644 --- a/plugins/processor_calyptia/calyptia.c +++ b/plugins/processor_calyptia/calyptia.c @@ -1,9 +1,3 @@ -#include -#include -#include - -#include -#include #include #include #include @@ -24,6 +18,10 @@ #include "calyptia_traces.h" #include "cfl_to_lua.h" +#include +#include +#include + static void calyptia_config_destroy(struct calyptia_context *ctx) { if (!ctx) { diff --git a/plugins/processor_calyptia/calyptia_logs_to_lua.c b/plugins/processor_calyptia/calyptia_logs_to_lua.c index a0fdc380986..d4fccbc0338 100644 --- a/plugins/processor_calyptia/calyptia_logs_to_lua.c +++ b/plugins/processor_calyptia/calyptia_logs_to_lua.c @@ -1,4 +1,3 @@ -#include #include #include diff --git a/plugins/processor_calyptia/calyptia_metrics_to_lua.c b/plugins/processor_calyptia/calyptia_metrics_to_lua.c index 5b2be51db74..e15d469374c 100644 --- a/plugins/processor_calyptia/calyptia_metrics_to_lua.c +++ b/plugins/processor_calyptia/calyptia_metrics_to_lua.c @@ -1,4 +1,3 @@ -#include #include #include @@ -9,6 +8,8 @@ #include #include +#include + #include "calyptia_metrics_to_lua.h" #include "cfl_to_lua.h" From 2f8310d4157ddd09e7b33958a50aef462e943bb8 Mon Sep 17 00:00:00 2001 From: Thiago Padilha Date: Mon, 2 Sep 2024 17:39:10 -0300 Subject: [PATCH 07/11] processor_calyptia: add license information Signed-off-by: Thiago Padilha --- plugins/CMakeLists.txt | 1 + plugins/processor_calyptia/CMakeLists.txt | 6 ++++++ plugins/processor_calyptia/calyptia.c | 19 +++++++++++++++++ plugins/processor_calyptia/calyptia_defs.h | 19 +++++++++++++++++ plugins/processor_calyptia/calyptia_logs.c | 19 +++++++++++++++++ plugins/processor_calyptia/calyptia_logs.h | 19 +++++++++++++++++ .../calyptia_logs_from_lua.c | 19 +++++++++++++++++ .../calyptia_logs_from_lua.h | 19 +++++++++++++++++ .../processor_calyptia/calyptia_logs_to_lua.c | 21 ++++++++++++++++++- .../processor_calyptia/calyptia_logs_to_lua.h | 19 +++++++++++++++++ plugins/processor_calyptia/calyptia_metrics.c | 19 +++++++++++++++++ plugins/processor_calyptia/calyptia_metrics.h | 19 +++++++++++++++++ .../calyptia_metrics_from_lua.c | 19 +++++++++++++++++ .../calyptia_metrics_from_lua.h | 19 +++++++++++++++++ .../calyptia_metrics_to_lua.c | 18 ++++++++++++++++ .../calyptia_metrics_to_lua.h | 19 +++++++++++++++++ plugins/processor_calyptia/calyptia_traces.c | 19 +++++++++++++++++ plugins/processor_calyptia/calyptia_traces.h | 19 +++++++++++++++++ .../calyptia_traces_from_lua.c | 19 +++++++++++++++++ .../calyptia_traces_from_lua.h | 19 +++++++++++++++++ .../calyptia_traces_to_lua.c | 18 ++++++++++++++++ .../calyptia_traces_to_lua.h | 19 +++++++++++++++++ plugins/processor_calyptia/cfl_to_lua.c | 19 +++++++++++++++++ plugins/processor_calyptia/cfl_to_lua.h | 19 +++++++++++++++++ plugins/processor_calyptia/lua_to_cfl.c | 19 +++++++++++++++++ plugins/processor_calyptia/lua_to_cfl.h | 19 +++++++++++++++++ 26 files changed, 462 insertions(+), 1 deletion(-) diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index eb6ca630ce2..9c4cd3515a0 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -377,6 +377,7 @@ endif() if(FLB_LUAJIT) REGISTER_FILTER_PLUGIN("filter_lua") + REGISTER_PROCESSOR_PLUGIN("processor_calyptia") endif() REGISTER_FILTER_PLUGIN("filter_stdout") diff --git a/plugins/processor_calyptia/CMakeLists.txt b/plugins/processor_calyptia/CMakeLists.txt index d02c5fce96f..9afca940d08 100644 --- a/plugins/processor_calyptia/CMakeLists.txt +++ b/plugins/processor_calyptia/CMakeLists.txt @@ -22,4 +22,10 @@ set(src lua_to_cfl.c ${LUA_HELPERS_DST}) +if(MSVC) + FLB_PLUGIN(processor_calyptia "${src}" "") +else() + FLB_PLUGIN(processor_calyptia "${src}" "m") +endif() + FLB_PLUGIN(processor_calyptia "${src}" "") diff --git a/plugins/processor_calyptia/calyptia.c b/plugins/processor_calyptia/calyptia.c index 3ac05f8ba84..9233f480570 100644 --- a/plugins/processor_calyptia/calyptia.c +++ b/plugins/processor_calyptia/calyptia.c @@ -1,3 +1,22 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #include #include #include diff --git a/plugins/processor_calyptia/calyptia_defs.h b/plugins/processor_calyptia/calyptia_defs.h index 64ea623a38f..01c342dccfe 100644 --- a/plugins/processor_calyptia/calyptia_defs.h +++ b/plugins/processor_calyptia/calyptia_defs.h @@ -1,3 +1,22 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #ifndef FLB_CALYPTIA_DEFS_H #define FLB_CALYPTIA_DEFS_H diff --git a/plugins/processor_calyptia/calyptia_logs.c b/plugins/processor_calyptia/calyptia_logs.c index ab333088867..b556bc11113 100644 --- a/plugins/processor_calyptia/calyptia_logs.c +++ b/plugins/processor_calyptia/calyptia_logs.c @@ -1,3 +1,22 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #include #include diff --git a/plugins/processor_calyptia/calyptia_logs.h b/plugins/processor_calyptia/calyptia_logs.h index 7172f8c8021..ac1d47f6f00 100644 --- a/plugins/processor_calyptia/calyptia_logs.h +++ b/plugins/processor_calyptia/calyptia_logs.h @@ -1,3 +1,22 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #ifndef FLB_CALYPTIA_LOGS_H #define FLB_CALYPTIA_LOGS_H diff --git a/plugins/processor_calyptia/calyptia_logs_from_lua.c b/plugins/processor_calyptia/calyptia_logs_from_lua.c index 6bf4e610b6f..feb4331c12e 100644 --- a/plugins/processor_calyptia/calyptia_logs_from_lua.c +++ b/plugins/processor_calyptia/calyptia_logs_from_lua.c @@ -1,3 +1,22 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #include #include "calyptia_logs_from_lua.h" diff --git a/plugins/processor_calyptia/calyptia_logs_from_lua.h b/plugins/processor_calyptia/calyptia_logs_from_lua.h index be3f5a9dd2a..12f15f0da2b 100644 --- a/plugins/processor_calyptia/calyptia_logs_from_lua.h +++ b/plugins/processor_calyptia/calyptia_logs_from_lua.h @@ -1,3 +1,22 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #ifndef FLB_CALYPTIA_METRICS_FROM_LUA #define FLB_CALYPTIA_METRICS_FROM_LUA diff --git a/plugins/processor_calyptia/calyptia_logs_to_lua.c b/plugins/processor_calyptia/calyptia_logs_to_lua.c index d4fccbc0338..72750c7d453 100644 --- a/plugins/processor_calyptia/calyptia_logs_to_lua.c +++ b/plugins/processor_calyptia/calyptia_logs_to_lua.c @@ -1,5 +1,22 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ -#include #include #include #include @@ -8,6 +25,8 @@ #include #include +#include + #include "calyptia_logs_to_lua.h" #include "cfl_to_lua.h" diff --git a/plugins/processor_calyptia/calyptia_logs_to_lua.h b/plugins/processor_calyptia/calyptia_logs_to_lua.h index 98041f44dab..0bf73db268b 100644 --- a/plugins/processor_calyptia/calyptia_logs_to_lua.h +++ b/plugins/processor_calyptia/calyptia_logs_to_lua.h @@ -1,3 +1,22 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #ifndef FLB_CALYPTIA_METRICS_TO_LUA_H #define FLB_CALYPTIA_METRICS_TO_LUA_H diff --git a/plugins/processor_calyptia/calyptia_metrics.c b/plugins/processor_calyptia/calyptia_metrics.c index eac00affa5c..f87f440a4f3 100644 --- a/plugins/processor_calyptia/calyptia_metrics.c +++ b/plugins/processor_calyptia/calyptia_metrics.c @@ -1,3 +1,22 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #include #include diff --git a/plugins/processor_calyptia/calyptia_metrics.h b/plugins/processor_calyptia/calyptia_metrics.h index ad8098c8a7f..0c7be9f01a4 100644 --- a/plugins/processor_calyptia/calyptia_metrics.h +++ b/plugins/processor_calyptia/calyptia_metrics.h @@ -1,3 +1,22 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #ifndef FLB_CALYPTIA_METRICS_H #define FLB_CALYPTIA_METRICS_H diff --git a/plugins/processor_calyptia/calyptia_metrics_from_lua.c b/plugins/processor_calyptia/calyptia_metrics_from_lua.c index d2ec29a6687..97425e875fc 100644 --- a/plugins/processor_calyptia/calyptia_metrics_from_lua.c +++ b/plugins/processor_calyptia/calyptia_metrics_from_lua.c @@ -1,3 +1,22 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #include #include diff --git a/plugins/processor_calyptia/calyptia_metrics_from_lua.h b/plugins/processor_calyptia/calyptia_metrics_from_lua.h index 91004246ae4..6b04987c420 100644 --- a/plugins/processor_calyptia/calyptia_metrics_from_lua.h +++ b/plugins/processor_calyptia/calyptia_metrics_from_lua.h @@ -1,3 +1,22 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #ifndef FLB_CALYPTIA_METRICS_FROM_LUA #define FLB_CALYPTIA_METRICS_FROM_LUA diff --git a/plugins/processor_calyptia/calyptia_metrics_to_lua.c b/plugins/processor_calyptia/calyptia_metrics_to_lua.c index e15d469374c..423912bd120 100644 --- a/plugins/processor_calyptia/calyptia_metrics_to_lua.c +++ b/plugins/processor_calyptia/calyptia_metrics_to_lua.c @@ -1,3 +1,21 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #include #include diff --git a/plugins/processor_calyptia/calyptia_metrics_to_lua.h b/plugins/processor_calyptia/calyptia_metrics_to_lua.h index cd370557195..9bf62ea7d55 100644 --- a/plugins/processor_calyptia/calyptia_metrics_to_lua.h +++ b/plugins/processor_calyptia/calyptia_metrics_to_lua.h @@ -1,3 +1,22 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #ifndef FLB_CALYPTIA_METRICS_TO_LUA_H #define FLB_CALYPTIA_METRICS_TO_LUA_H diff --git a/plugins/processor_calyptia/calyptia_traces.c b/plugins/processor_calyptia/calyptia_traces.c index a780bddbeac..836a4278675 100644 --- a/plugins/processor_calyptia/calyptia_traces.c +++ b/plugins/processor_calyptia/calyptia_traces.c @@ -1,3 +1,22 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #include #include diff --git a/plugins/processor_calyptia/calyptia_traces.h b/plugins/processor_calyptia/calyptia_traces.h index 9cf13d45a93..4efae3ac07e 100644 --- a/plugins/processor_calyptia/calyptia_traces.h +++ b/plugins/processor_calyptia/calyptia_traces.h @@ -1,3 +1,22 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #ifndef CALYPTIA_TRACES_H #define CALYPTIA_TRACES_H diff --git a/plugins/processor_calyptia/calyptia_traces_from_lua.c b/plugins/processor_calyptia/calyptia_traces_from_lua.c index 1df036aa55c..6f584e9b145 100644 --- a/plugins/processor_calyptia/calyptia_traces_from_lua.c +++ b/plugins/processor_calyptia/calyptia_traces_from_lua.c @@ -1,3 +1,22 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #include #include #include diff --git a/plugins/processor_calyptia/calyptia_traces_from_lua.h b/plugins/processor_calyptia/calyptia_traces_from_lua.h index 4ec35364465..e3aaf98e275 100644 --- a/plugins/processor_calyptia/calyptia_traces_from_lua.h +++ b/plugins/processor_calyptia/calyptia_traces_from_lua.h @@ -1,3 +1,22 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #ifndef CALYPTIA_TRACES_FROM_LUA_H #define CALYPTIA_TRACES_FROM_LUA_H diff --git a/plugins/processor_calyptia/calyptia_traces_to_lua.c b/plugins/processor_calyptia/calyptia_traces_to_lua.c index 83c91d8c92b..ed23a5994d0 100644 --- a/plugins/processor_calyptia/calyptia_traces_to_lua.c +++ b/plugins/processor_calyptia/calyptia_traces_to_lua.c @@ -1,3 +1,21 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #include #include #include diff --git a/plugins/processor_calyptia/calyptia_traces_to_lua.h b/plugins/processor_calyptia/calyptia_traces_to_lua.h index 8193271fbcd..a4b62ea3d27 100644 --- a/plugins/processor_calyptia/calyptia_traces_to_lua.h +++ b/plugins/processor_calyptia/calyptia_traces_to_lua.h @@ -1,3 +1,22 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #ifndef CALYPTIA_TRACES_TO_LUA_H #define CALYPTIA_TRACES_TO_LUA_H diff --git a/plugins/processor_calyptia/cfl_to_lua.c b/plugins/processor_calyptia/cfl_to_lua.c index a94723ef73a..7901e2223b6 100644 --- a/plugins/processor_calyptia/cfl_to_lua.c +++ b/plugins/processor_calyptia/cfl_to_lua.c @@ -1,3 +1,22 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #include "cfl_to_lua.h" diff --git a/plugins/processor_calyptia/cfl_to_lua.h b/plugins/processor_calyptia/cfl_to_lua.h index aa6d8907d42..03729a30bba 100644 --- a/plugins/processor_calyptia/cfl_to_lua.h +++ b/plugins/processor_calyptia/cfl_to_lua.h @@ -1,3 +1,22 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #ifndef FLB_CALYPTIA_PROCESSOR_CFL_TO_LUA_H #define FLB_CALYPTIA_PROCESSOR_CFL_TO_LUA_H diff --git a/plugins/processor_calyptia/lua_to_cfl.c b/plugins/processor_calyptia/lua_to_cfl.c index d96fd905cac..d01138ef3e2 100644 --- a/plugins/processor_calyptia/lua_to_cfl.c +++ b/plugins/processor_calyptia/lua_to_cfl.c @@ -1,3 +1,22 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #include "lua_to_cfl.h" #include diff --git a/plugins/processor_calyptia/lua_to_cfl.h b/plugins/processor_calyptia/lua_to_cfl.h index c88a5ba3563..7183d18b7a7 100644 --- a/plugins/processor_calyptia/lua_to_cfl.h +++ b/plugins/processor_calyptia/lua_to_cfl.h @@ -1,3 +1,22 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #ifndef FLB_CALYPTIA_PROCESSOR_LUA_TO_CFL_H #define FLB_CALYPTIA_PROCESSOR_LUA_TO_CFL_H From 92539d6e8d745606f32576f9183c51d1f1858821 Mon Sep 17 00:00:00 2001 From: Thiago Padilha Date: Tue, 10 Sep 2024 09:53:52 -0300 Subject: [PATCH 08/11] processor_calyptia: Fix compilation error Signed-off-by: Thiago Padilha --- plugins/CMakeLists.txt | 5 +++-- plugins/processor_calyptia/CMakeLists.txt | 2 -- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 9c4cd3515a0..9a63f2ab11a 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -283,11 +283,13 @@ REGISTER_IN_PLUGIN("in_random") # PROCESSORS # ========== REGISTER_PROCESSOR_PLUGIN("processor_content_modifier") -REGISTER_PROCESSOR_PLUGIN("processor_calyptia") REGISTER_PROCESSOR_PLUGIN("processor_labels") REGISTER_PROCESSOR_PLUGIN("processor_metrics_selector") REGISTER_PROCESSOR_PLUGIN("processor_sql") REGISTER_PROCESSOR_PLUGIN("processor_opentelemetry_envelope") +if(FLB_LUAJIT) + REGISTER_PROCESSOR_PLUGIN("processor_calyptia") +endif() # OUTPUTS # ======= @@ -377,7 +379,6 @@ endif() if(FLB_LUAJIT) REGISTER_FILTER_PLUGIN("filter_lua") - REGISTER_PROCESSOR_PLUGIN("processor_calyptia") endif() REGISTER_FILTER_PLUGIN("filter_stdout") diff --git a/plugins/processor_calyptia/CMakeLists.txt b/plugins/processor_calyptia/CMakeLists.txt index 9afca940d08..b9a00d4b557 100644 --- a/plugins/processor_calyptia/CMakeLists.txt +++ b/plugins/processor_calyptia/CMakeLists.txt @@ -27,5 +27,3 @@ if(MSVC) else() FLB_PLUGIN(processor_calyptia "${src}" "m") endif() - -FLB_PLUGIN(processor_calyptia "${src}" "") From bb76d8a0e948daaf18530bb6a094af870136d82e Mon Sep 17 00:00:00 2001 From: Thiago Padilha Date: Tue, 10 Sep 2024 11:30:19 -0300 Subject: [PATCH 09/11] processor_calyptia: fix windows build Signed-off-by: Thiago Padilha --- plugins/processor_calyptia/calyptia_logs_from_lua.h | 2 +- plugins/processor_calyptia/calyptia_logs_to_lua.c | 10 ---------- plugins/processor_calyptia/calyptia_logs_to_lua.h | 2 +- plugins/processor_calyptia/calyptia_metrics_from_lua.c | 9 --------- plugins/processor_calyptia/calyptia_metrics_from_lua.h | 2 +- plugins/processor_calyptia/calyptia_metrics_to_lua.c | 7 ++++--- plugins/processor_calyptia/calyptia_metrics_to_lua.h | 3 +-- plugins/processor_calyptia/calyptia_traces.c | 2 ++ plugins/processor_calyptia/calyptia_traces_from_lua.c | 1 - plugins/processor_calyptia/calyptia_traces_from_lua.h | 2 +- plugins/processor_calyptia/calyptia_traces_to_lua.c | 2 +- plugins/processor_calyptia/calyptia_traces_to_lua.h | 2 +- plugins/processor_calyptia/cfl_to_lua.c | 1 - plugins/processor_calyptia/cfl_to_lua.h | 2 +- 14 files changed, 14 insertions(+), 33 deletions(-) diff --git a/plugins/processor_calyptia/calyptia_logs_from_lua.h b/plugins/processor_calyptia/calyptia_logs_from_lua.h index 12f15f0da2b..e4c8d8d15c7 100644 --- a/plugins/processor_calyptia/calyptia_logs_from_lua.h +++ b/plugins/processor_calyptia/calyptia_logs_from_lua.h @@ -20,8 +20,8 @@ #ifndef FLB_CALYPTIA_METRICS_FROM_LUA #define FLB_CALYPTIA_METRICS_FROM_LUA -#include #include +#include int calyptia_logs_from_lua(struct flb_processor_instance *ins, lua_State *L, struct flb_mp_chunk_cobj *chunk_cobj); diff --git a/plugins/processor_calyptia/calyptia_logs_to_lua.c b/plugins/processor_calyptia/calyptia_logs_to_lua.c index 72750c7d453..6a8133df9cc 100644 --- a/plugins/processor_calyptia/calyptia_logs_to_lua.c +++ b/plugins/processor_calyptia/calyptia_logs_to_lua.c @@ -17,16 +17,6 @@ * limitations under the License. */ -#include -#include -#include -#include -#include -#include -#include - -#include - #include "calyptia_logs_to_lua.h" #include "cfl_to_lua.h" diff --git a/plugins/processor_calyptia/calyptia_logs_to_lua.h b/plugins/processor_calyptia/calyptia_logs_to_lua.h index 0bf73db268b..d11e23aeb14 100644 --- a/plugins/processor_calyptia/calyptia_logs_to_lua.h +++ b/plugins/processor_calyptia/calyptia_logs_to_lua.h @@ -20,8 +20,8 @@ #ifndef FLB_CALYPTIA_METRICS_TO_LUA_H #define FLB_CALYPTIA_METRICS_TO_LUA_H -#include #include +#include int calyptia_logs_to_lua(lua_State *L, struct flb_mp_chunk_cobj *chunk_cobj); diff --git a/plugins/processor_calyptia/calyptia_metrics_from_lua.c b/plugins/processor_calyptia/calyptia_metrics_from_lua.c index 97425e875fc..9ffe32fa122 100644 --- a/plugins/processor_calyptia/calyptia_metrics_from_lua.c +++ b/plugins/processor_calyptia/calyptia_metrics_from_lua.c @@ -17,15 +17,6 @@ * limitations under the License. */ -#include - -#include -#include -#include -#include -#include -#include -#include #include #include "calyptia_metrics_from_lua.h" diff --git a/plugins/processor_calyptia/calyptia_metrics_from_lua.h b/plugins/processor_calyptia/calyptia_metrics_from_lua.h index 6b04987c420..9252e4e8b0e 100644 --- a/plugins/processor_calyptia/calyptia_metrics_from_lua.h +++ b/plugins/processor_calyptia/calyptia_metrics_from_lua.h @@ -20,9 +20,9 @@ #ifndef FLB_CALYPTIA_METRICS_FROM_LUA #define FLB_CALYPTIA_METRICS_FROM_LUA -#include #include #include +#include int calyptia_metrics_from_lua(struct flb_processor_instance *ins, lua_State *L, struct cmt *cmt); diff --git a/plugins/processor_calyptia/calyptia_metrics_to_lua.c b/plugins/processor_calyptia/calyptia_metrics_to_lua.c index 423912bd120..e3d4f6ce4cf 100644 --- a/plugins/processor_calyptia/calyptia_metrics_to_lua.c +++ b/plugins/processor_calyptia/calyptia_metrics_to_lua.c @@ -17,7 +17,10 @@ * limitations under the License. */ -#include +#include + +#include + #include #include #include @@ -26,8 +29,6 @@ #include #include -#include - #include "calyptia_metrics_to_lua.h" #include "cfl_to_lua.h" diff --git a/plugins/processor_calyptia/calyptia_metrics_to_lua.h b/plugins/processor_calyptia/calyptia_metrics_to_lua.h index 9bf62ea7d55..6dec06d5acd 100644 --- a/plugins/processor_calyptia/calyptia_metrics_to_lua.h +++ b/plugins/processor_calyptia/calyptia_metrics_to_lua.h @@ -20,9 +20,8 @@ #ifndef FLB_CALYPTIA_METRICS_TO_LUA_H #define FLB_CALYPTIA_METRICS_TO_LUA_H -#include #include - +#include int calyptia_metrics_to_lua(lua_State *L, struct cmt *cmt); diff --git a/plugins/processor_calyptia/calyptia_traces.c b/plugins/processor_calyptia/calyptia_traces.c index 836a4278675..eb443eb8c64 100644 --- a/plugins/processor_calyptia/calyptia_traces.c +++ b/plugins/processor_calyptia/calyptia_traces.c @@ -25,6 +25,8 @@ #include "calyptia_traces_to_lua.h" #include "calyptia_traces_from_lua.h" +#include + static void drop_traces(struct ctrace *ctx) { struct cfl_list *head; diff --git a/plugins/processor_calyptia/calyptia_traces_from_lua.c b/plugins/processor_calyptia/calyptia_traces_from_lua.c index 6f584e9b145..fed19f42ba5 100644 --- a/plugins/processor_calyptia/calyptia_traces_from_lua.c +++ b/plugins/processor_calyptia/calyptia_traces_from_lua.c @@ -17,7 +17,6 @@ * limitations under the License. */ -#include #include #include diff --git a/plugins/processor_calyptia/calyptia_traces_from_lua.h b/plugins/processor_calyptia/calyptia_traces_from_lua.h index e3aaf98e275..f825e2538f5 100644 --- a/plugins/processor_calyptia/calyptia_traces_from_lua.h +++ b/plugins/processor_calyptia/calyptia_traces_from_lua.h @@ -20,8 +20,8 @@ #ifndef CALYPTIA_TRACES_FROM_LUA_H #define CALYPTIA_TRACES_FROM_LUA_H -#include #include +#include int calyptia_traces_from_lua(lua_State *L, struct ctrace *ctx); diff --git a/plugins/processor_calyptia/calyptia_traces_to_lua.c b/plugins/processor_calyptia/calyptia_traces_to_lua.c index ed23a5994d0..f17688d7a98 100644 --- a/plugins/processor_calyptia/calyptia_traces_to_lua.c +++ b/plugins/processor_calyptia/calyptia_traces_to_lua.c @@ -16,10 +16,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include #include #include +#include "calyptia_traces_to_lua.h" #include "cfl_to_lua.h" static void push_attributes(lua_State *L, struct ctrace_attributes *attr) diff --git a/plugins/processor_calyptia/calyptia_traces_to_lua.h b/plugins/processor_calyptia/calyptia_traces_to_lua.h index a4b62ea3d27..fdc908d98a1 100644 --- a/plugins/processor_calyptia/calyptia_traces_to_lua.h +++ b/plugins/processor_calyptia/calyptia_traces_to_lua.h @@ -20,8 +20,8 @@ #ifndef CALYPTIA_TRACES_TO_LUA_H #define CALYPTIA_TRACES_TO_LUA_H -#include #include +#include int calyptia_traces_to_lua(lua_State *L, struct ctrace *ctx); diff --git a/plugins/processor_calyptia/cfl_to_lua.c b/plugins/processor_calyptia/cfl_to_lua.c index 7901e2223b6..eb1afe12b6c 100644 --- a/plugins/processor_calyptia/cfl_to_lua.c +++ b/plugins/processor_calyptia/cfl_to_lua.c @@ -19,7 +19,6 @@ #include "cfl_to_lua.h" - void push_string(lua_State *L, const char *str, size_t size) { lua_pushlstring(L, str, size); diff --git a/plugins/processor_calyptia/cfl_to_lua.h b/plugins/processor_calyptia/cfl_to_lua.h index 03729a30bba..851e36aeaea 100644 --- a/plugins/processor_calyptia/cfl_to_lua.h +++ b/plugins/processor_calyptia/cfl_to_lua.h @@ -20,8 +20,8 @@ #ifndef FLB_CALYPTIA_PROCESSOR_CFL_TO_LUA_H #define FLB_CALYPTIA_PROCESSOR_CFL_TO_LUA_H -#include #include +#include void push_variant(lua_State *L, struct cfl_variant *variant); void push_string(lua_State *L, const char *str, size_t size); From 050794aed982715f7f4dcd30ca45bc2a33380978 Mon Sep 17 00:00:00 2001 From: Thiago Padilha Date: Tue, 10 Sep 2024 13:57:52 -0300 Subject: [PATCH 10/11] Fix macos build Signed-off-by: Thiago Padilha --- plugins/processor_calyptia/calyptia_metrics_from_lua.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/processor_calyptia/calyptia_metrics_from_lua.c b/plugins/processor_calyptia/calyptia_metrics_from_lua.c index 9ffe32fa122..98e04f2ba29 100644 --- a/plugins/processor_calyptia/calyptia_metrics_from_lua.c +++ b/plugins/processor_calyptia/calyptia_metrics_from_lua.c @@ -17,6 +17,9 @@ * limitations under the License. */ +#include +#include +#include #include #include "calyptia_metrics_from_lua.h" From 7544fe91b950562721d7bf62fff5eb2cefb6756c Mon Sep 17 00:00:00 2001 From: Thiago Padilha Date: Tue, 10 Sep 2024 14:33:36 -0300 Subject: [PATCH 11/11] processor_calyptia: add test script Signed-off-by: Thiago Padilha --- tests/test-processor-calyptia.sh | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100755 tests/test-processor-calyptia.sh diff --git a/tests/test-processor-calyptia.sh b/tests/test-processor-calyptia.sh new file mode 100755 index 00000000000..bd01b6e6077 --- /dev/null +++ b/tests/test-processor-calyptia.sh @@ -0,0 +1,8 @@ +#!/bin/sh -e + +for d in examples/processor_calyptia/*; do + valgrind --log-file=${d##*/}-valgrind.log --leak-check=full --show-leak-kinds=all --track-origins=yes ./build/bin/fluent-bit -c ${d}/fluent-bit.yaml & +done + +wait +# wait around 10 seconds, kill with ctrl+c then use `cat *-valgrind.log` to see the results