diff --git a/include/fluent-bit/flb_conditionals.h b/include/fluent-bit/flb_conditionals.h index ea9314f55bb..5cc864b3154 100644 --- a/include/fluent-bit/flb_conditionals.h +++ b/include/fluent-bit/flb_conditionals.h @@ -28,10 +28,20 @@ #include #include +struct flb_condition_rule; + +typedef struct cfl_variant *(*flb_condition_get_variant_fn)(struct flb_condition_rule *rule, + void *ctx); + /* Context types enum */ enum record_context_type { - RECORD_CONTEXT_BODY = 0, - RECORD_CONTEXT_METADATA = 1 + RECORD_CONTEXT_BODY = 0, + RECORD_CONTEXT_METADATA = 1, + RECORD_CONTEXT_GROUP_METADATA, + RECORD_CONTEXT_GROUP_ATTRIBUTES, + RECORD_CONTEXT_OTEL_RESOURCE_ATTRIBUTES, + RECORD_CONTEXT_OTEL_SCOPE_ATTRIBUTES, + RECORD_CONTEXT_OTEL_SCOPE_METADATA }; struct flb_condition; @@ -88,6 +98,9 @@ int flb_condition_add_rule(struct flb_condition *cond, void flb_condition_destroy(struct flb_condition *cond); /* Evaluation function */ +int flb_condition_evaluate_ex(struct flb_condition *cond, + void *ctx, + flb_condition_get_variant_fn get_variant); int flb_condition_evaluate(struct flb_condition *cond, struct flb_mp_chunk_record *record); diff --git a/include/fluent-bit/flb_log_event.h b/include/fluent-bit/flb_log_event.h index 2c1d99acc31..89374dd9fd2 100644 --- a/include/fluent-bit/flb_log_event.h +++ b/include/fluent-bit/flb_log_event.h @@ -33,6 +33,17 @@ #define FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V1 FLB_LOG_EVENT_FORMAT_FORWARD #define FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2 4 +/* + * Log event type identification via timestamp value: + * - Non-negative timestamps (>= 0): Normal log records with actual timestamps + * - -1 (FLB_LOG_EVENT_GROUP_START): Group marker indicating start of a log group + * - -2 (FLB_LOG_EVENT_GROUP_END): Group marker indicating end of a log group + * - Other negative values: Invalid/corrupted data (will be skipped by decoder) + * + * NOTE: Negative timestamps are RESERVED for group markers. Only -1 and -2 are valid. + * Any other negative timestamp is considered invalid and will be skipped during decoding. + * Encoders must respect this contract and only use -1/-2 for group markers. + */ #define FLB_LOG_EVENT_NORMAL (int32_t) 0 #define FLB_LOG_EVENT_GROUP_START (int32_t) -1 #define FLB_LOG_EVENT_GROUP_END (int32_t) -2 diff --git a/include/fluent-bit/flb_log_event_decoder.h b/include/fluent-bit/flb_log_event_decoder.h index 7fdbaa619b9..6198b67bc51 100644 --- a/include/fluent-bit/flb_log_event_decoder.h +++ b/include/fluent-bit/flb_log_event_decoder.h @@ -62,6 +62,7 @@ struct flb_log_event_decoder { size_t length; int last_result; int read_groups; + unsigned int recursion_depth; /* Safety guard for recursion limit */ }; void flb_log_event_decoder_reset(struct flb_log_event_decoder *context, diff --git a/include/fluent-bit/flb_router.h b/include/fluent-bit/flb_router.h index 8b34e0af160..c0d181e670e 100644 --- a/include/fluent-bit/flb_router.h +++ b/include/fluent-bit/flb_router.h @@ -33,6 +33,7 @@ struct flb_mp_chunk_cobj; struct flb_log_event_encoder; struct flb_log_event_decoder; +struct flb_mp_chunk_record; struct flb_router_chunk_context { struct flb_mp_chunk_cobj *chunk_cobj; @@ -88,6 +89,7 @@ struct flb_route_condition_rule { flb_sds_t value; flb_sds_t *values; size_t values_count; + enum record_context_type context; struct cfl_list _head; }; @@ -102,6 +104,7 @@ struct flb_route_condition { struct flb_route_output { flb_sds_t name; flb_sds_t fallback; + struct flb_output_instance *ins; struct cfl_list _head; }; @@ -155,6 +158,8 @@ int flb_router_chunk_context_prepare_logs(struct flb_router_chunk_context *conte int flb_route_condition_eval(struct flb_event_chunk *chunk, struct flb_router_chunk_context *context, struct flb_route *route); +int flb_router_condition_evaluate_record(struct flb_route *route, + struct flb_mp_chunk_record *record); int flb_condition_eval_logs(struct flb_event_chunk *chunk, struct flb_router_chunk_context *context, struct flb_route *route); diff --git a/plugins/processor_content_modifier/cm_logs.c b/plugins/processor_content_modifier/cm_logs.c index 49e6e2fe08d..6af448cda4d 100644 --- a/plugins/processor_content_modifier/cm_logs.c +++ b/plugins/processor_content_modifier/cm_logs.c @@ -316,6 +316,12 @@ int cm_logs_process(struct flb_processor_instance *ins, continue; } + if (record_type == FLB_LOG_EVENT_GROUP_START && + (ctx->context_type == CM_CONTEXT_LOG_METADATA || + ctx->context_type == CM_CONTEXT_LOG_BODY)) { + continue; + } + /* retrieve the target cfl object */ if (ctx->context_type == CM_CONTEXT_LOG_METADATA) { obj = record->cobj_metadata; diff --git a/src/flb_conditionals.c b/src/flb_conditionals.c index 05f943be17e..8f481984cec 100644 --- a/src/flb_conditionals.c +++ b/src/flb_conditionals.c @@ -25,15 +25,16 @@ #include #include -/* Function to get the record variant based on context */ -static inline struct cfl_variant *get_record_variant(struct flb_mp_chunk_record *record, - enum record_context_type context_type) +static struct cfl_variant *default_get_record_variant(struct flb_condition_rule *rule, + void *ctx) { - if (!record) { + struct flb_mp_chunk_record *record = (struct flb_mp_chunk_record *) ctx; + + if (!record || !rule) { return NULL; } - switch (context_type) { + switch (rule->context) { case RECORD_CONTEXT_METADATA: if (record->cobj_metadata) { return record->cobj_metadata->variant; @@ -45,6 +46,9 @@ static inline struct cfl_variant *get_record_variant(struct flb_mp_chunk_record return record->cobj_record->variant; } break; + + default: + break; } return NULL; @@ -356,8 +360,9 @@ static int evaluate_rule(struct flb_condition_rule *rule, return result; } -int flb_condition_evaluate(struct flb_condition *cond, - struct flb_mp_chunk_record *record) +int flb_condition_evaluate_ex(struct flb_condition *cond, + void *ctx, + flb_condition_get_variant_fn get_variant) { struct mk_list *head; struct flb_condition_rule *rule; @@ -366,11 +371,16 @@ int flb_condition_evaluate(struct flb_condition *cond, int any_rule_evaluated = FLB_FALSE; int any_rule_matched = FLB_FALSE; - if (!cond || !record) { - flb_trace("[condition] NULL condition or record, returning TRUE"); + if (!cond) { + flb_trace("[condition] NULL condition, returning TRUE"); return FLB_TRUE; } + if (!get_variant) { + flb_trace("[condition] missing variant provider, returning FALSE"); + return FLB_FALSE; + } + flb_trace("[condition] evaluating condition with %d rules", mk_list_size(&cond->rules)); if (mk_list_size(&cond->rules) == 0) { @@ -382,8 +392,7 @@ int flb_condition_evaluate(struct flb_condition *cond, rule = mk_list_entry(head, struct flb_condition_rule, _head); flb_trace("[condition] processing rule with op=%d", rule->op); - /* Get the variant for this rule's context */ - record_variant = get_record_variant(record, rule->context); + record_variant = get_variant(rule, ctx); any_rule_evaluated = FLB_TRUE; if (!record_variant) { flb_trace("[condition] no record variant found for context %d", rule->context); @@ -427,3 +436,17 @@ int flb_condition_evaluate(struct flb_condition *cond, flb_trace("[condition] final evaluation result: TRUE"); return FLB_TRUE; } + +int flb_condition_evaluate(struct flb_condition *cond, + struct flb_mp_chunk_record *record) +{ + if (!cond) { + return FLB_TRUE; + } + + if (!record) { + return FLB_FALSE; + } + + return flb_condition_evaluate_ex(cond, record, default_get_record_variant); +} diff --git a/src/flb_input_log.c b/src/flb_input_log.c index ae06d079379..4eb281a7497 100644 --- a/src/flb_input_log.c +++ b/src/flb_input_log.c @@ -99,7 +99,6 @@ static int route_payload_apply_outputs(struct flb_input_instance *ins, struct flb_route_payload *payload) { int ret; - int routes_found = 0; size_t out_size = 0; size_t chunk_size_sz = 0; ssize_t chunk_size; @@ -171,15 +170,20 @@ static int route_payload_apply_outputs(struct flb_input_instance *ins, } memset(chunk->routes_mask, 0, sizeof(flb_route_mask_element) * ins->config->route_mask_size); + cfl_list_foreach(head, &ins->routes_direct) { route_path = cfl_list_entry(head, struct flb_router_path, _head); - if (route_path->route != payload->route || !route_path->ins) { + if (!route_path->route || !route_path->ins) { + continue; + } + + if (route_path->route != payload->route) { continue; } + flb_routes_mask_set_bit(chunk->routes_mask, route_path->ins->id, ins->config); - routes_found++; } if (flb_routes_mask_is_empty(chunk->routes_mask, ins->config) == FLB_TRUE) { @@ -318,25 +322,29 @@ static int encode_chunk_record(struct flb_log_event_encoder *encoder, return 0; } -static int build_payload_for_route(struct flb_route_payload *payload, +static int build_payload_for_route(struct flb_input_instance *ins, + struct flb_route_payload *payload, struct flb_mp_chunk_record **records, size_t record_count, uint8_t *matched_non_default) { - size_t i; + int i; + int j; int ret; int condition_result; int matched; - struct flb_condition *compiled; + int32_t record_type; struct flb_log_event_encoder *encoder; + struct flb_mp_chunk_record *group_end = NULL; + struct flb_mp_chunk_record *group_start_record = NULL; + uint8_t *matched_by_route = NULL; if (!payload || !records || record_count == 0 || !matched_non_default) { return -1; } - - compiled = flb_router_route_get_condition(payload->route); - if (!compiled) { + /* Check if route has a condition (flb_router_condition_evaluate_record handles NULL conditions) */ + if (!payload->route->condition) { return 0; } @@ -345,38 +353,128 @@ static int build_payload_for_route(struct flb_route_payload *payload, return -1; } + /* Track which records match THIS specific route */ + matched_by_route = flb_calloc(record_count, sizeof(uint8_t)); + if (!matched_by_route) { + flb_errno(); + flb_log_event_encoder_destroy(encoder); + return -1; + } + matched = 0; + /* First pass: evaluate conditions and mark matching records */ for (i = 0; i < record_count; i++) { - condition_result = flb_condition_evaluate(compiled, records[i]); + if (flb_log_event_decoder_get_record_type(&records[i]->event, &record_type) == 0) { + if (record_type == FLB_LOG_EVENT_GROUP_START) { + continue; + } + else if (record_type == FLB_LOG_EVENT_GROUP_END) { + group_end = records[i]; + continue; + } + } + + condition_result = flb_router_condition_evaluate_record(payload->route, records[i]); if (condition_result != FLB_TRUE) { continue; } - ret = encode_chunk_record(encoder, records[i]); + matched_by_route[i] = 1; + matched_non_default[i] = 1; + matched++; + } + + /* If no matches, return early */ + if (matched == 0) { + flb_free(matched_by_route); + flb_log_event_encoder_destroy(encoder); + return 0; + } + + /* Second pass: encode records in order, preserving group structure */ + for (i = 0; i < record_count; i++) { + if (flb_log_event_decoder_get_record_type(&records[i]->event, &record_type) != 0) { + continue; + } + + if (record_type == FLB_LOG_EVENT_GROUP_START) { + group_start_record = records[i]; + group_end = NULL; + continue; + } + else if (record_type == FLB_LOG_EVENT_GROUP_END) { + if (group_end != NULL && + group_start_record != NULL && + records[i]->cobj_group_metadata == group_start_record->cobj_group_metadata) { + ret = encode_chunk_record(encoder, group_end); + if (ret != 0) { + flb_free(matched_by_route); + flb_log_event_encoder_destroy(encoder); + return -1; + } + group_end = NULL; + } + group_start_record = NULL; + continue; + } + else if (record_type == FLB_LOG_EVENT_NORMAL && matched_by_route[i]) { + if (group_start_record != NULL && + records[i]->cobj_group_metadata == group_start_record->cobj_group_metadata) { + if (group_end == NULL) { + ret = encode_chunk_record(encoder, group_start_record); + if (ret != 0) { + flb_free(matched_by_route); + flb_log_event_encoder_destroy(encoder); + return -1; + } + for (j = i + 1; j < record_count; j++) { + if (flb_log_event_decoder_get_record_type(&records[j]->event, &record_type) == 0 && + record_type == FLB_LOG_EVENT_GROUP_END && + records[j]->cobj_group_metadata == group_start_record->cobj_group_metadata) { + group_end = records[j]; + break; + } + } + } + } + + ret = encode_chunk_record(encoder, records[i]); + if (ret != 0) { + flb_free(matched_by_route); + flb_log_event_encoder_destroy(encoder); + return -1; + } + } + } + + if (group_end != NULL) { + ret = encode_chunk_record(encoder, group_end); if (ret != 0) { + flb_free(matched_by_route); flb_log_event_encoder_destroy(encoder); return -1; } - - matched_non_default[i] = 1; - matched++; } - if (matched == 0) { + flb_free(matched_by_route); + + /* Ensure output_buffer and output_length are up to date */ + if (encoder->buffer.size == 0) { flb_log_event_encoder_destroy(encoder); return 0; } - payload->data = flb_malloc(encoder->output_length); + payload->size = encoder->buffer.size; + payload->data = flb_malloc(payload->size); if (!payload->data) { flb_log_event_encoder_destroy(encoder); flb_errno(); return -1; } - memcpy(payload->data, encoder->output_buffer, encoder->output_length); - payload->size = encoder->output_length; + /* Copy the buffer data - msgpack_sbuffer uses flat memory, no zones */ + memcpy(payload->data, encoder->buffer.data, payload->size); payload->total_records = matched; flb_log_event_encoder_destroy(encoder); @@ -384,16 +482,22 @@ static int build_payload_for_route(struct flb_route_payload *payload, return 0; } -static int build_payload_for_default_route(struct flb_route_payload *payload, +static int build_payload_for_default_route(struct flb_input_instance *ins, + struct flb_route_payload *payload, struct flb_mp_chunk_record **records, size_t record_count, uint8_t *matched_non_default) { - size_t i; + int i; + int j; int matched; int ret; - struct flb_condition *compiled; + int condition_result; + int32_t record_type; struct flb_log_event_encoder *encoder; + struct flb_mp_chunk_record *group_end = NULL; + struct flb_mp_chunk_record *group_start_record = NULL; + int *matched_by_default = NULL; if (!payload || !records || !matched_non_default) { return -1; @@ -404,42 +508,146 @@ static int build_payload_for_default_route(struct flb_route_payload *payload, return -1; } - compiled = flb_router_route_get_condition(payload->route); matched = 0; + /* First pass: evaluate conditions */ for (i = 0; i < record_count; i++) { + if (flb_log_event_decoder_get_record_type(&records[i]->event, &record_type) == 0) { + if (record_type == FLB_LOG_EVENT_GROUP_START) { + continue; + } + else if (record_type == FLB_LOG_EVENT_GROUP_END) { + group_end = records[i]; + continue; + } + } + if (matched_non_default[i]) { continue; } - if (compiled && - flb_condition_evaluate(compiled, records[i]) != FLB_TRUE) { + condition_result = flb_router_condition_evaluate_record(payload->route, records[i]); + if (condition_result != FLB_TRUE) { continue; } - ret = encode_chunk_record(encoder, records[i]); + matched++; + } + + /* If no matches, return early - no need to create payload */ + if (matched == 0) { + flb_log_event_encoder_destroy(encoder); + return 0; + } + + matched_by_default = flb_calloc(record_count, sizeof(int)); + if (!matched_by_default) { + flb_errno(); + flb_log_event_encoder_destroy(encoder); + return -1; + } + + /* Mark matching records */ + for (i = 0; i < record_count; i++) { + if (flb_log_event_decoder_get_record_type(&records[i]->event, &record_type) == 0 && + record_type == FLB_LOG_EVENT_NORMAL) { + if (!matched_non_default[i]) { + if (payload->route->condition) { + condition_result = flb_router_condition_evaluate_record(payload->route, records[i]); + if (condition_result == FLB_TRUE) { + matched_by_default[i] = 1; + } + } + else { + matched_by_default[i] = 1; + } + } + } + } + + /* Second pass: encode records in order, preserving group structure */ + for (i = 0; i < record_count; i++) { + if (flb_log_event_decoder_get_record_type(&records[i]->event, &record_type) != 0) { + continue; + } + + if (record_type == FLB_LOG_EVENT_GROUP_START) { + group_start_record = records[i]; + group_end = NULL; + continue; + } + else if (record_type == FLB_LOG_EVENT_GROUP_END) { + if (group_end != NULL && + group_start_record != NULL && + records[i]->cobj_group_metadata == group_start_record->cobj_group_metadata) { + ret = encode_chunk_record(encoder, group_end); + if (ret != 0) { + flb_free(matched_by_default); + flb_log_event_encoder_destroy(encoder); + return -1; + } + group_end = NULL; + } + group_start_record = NULL; + continue; + } + else if (record_type == FLB_LOG_EVENT_NORMAL && matched_by_default[i]) { + if (group_start_record != NULL && + records[i]->cobj_group_metadata == group_start_record->cobj_group_metadata) { + if (group_end == NULL) { + ret = encode_chunk_record(encoder, group_start_record); + if (ret != 0) { + flb_free(matched_by_default); + flb_log_event_encoder_destroy(encoder); + return -1; + } + for (j = i + 1; j < record_count; j++) { + if (flb_log_event_decoder_get_record_type(&records[j]->event, &record_type) == 0 && + record_type == FLB_LOG_EVENT_GROUP_END && + records[j]->cobj_group_metadata == group_start_record->cobj_group_metadata) { + group_end = records[j]; + break; + } + } + } + } + + ret = encode_chunk_record(encoder, records[i]); + if (ret != 0) { + flb_free(matched_by_default); + flb_log_event_encoder_destroy(encoder); + return -1; + } + } + } + + if (group_end != NULL) { + ret = encode_chunk_record(encoder, group_end); if (ret != 0) { + flb_free(matched_by_default); flb_log_event_encoder_destroy(encoder); return -1; } - - matched++; } - if (matched == 0) { + flb_free(matched_by_default); + + /* Ensure output_buffer and output_length are up to date */ + if (encoder->buffer.size == 0) { flb_log_event_encoder_destroy(encoder); return 0; } - payload->data = flb_malloc(encoder->output_length); + payload->size = encoder->buffer.size; + payload->data = flb_malloc(payload->size); if (!payload->data) { flb_log_event_encoder_destroy(encoder); flb_errno(); return -1; } - memcpy(payload->data, encoder->output_buffer, encoder->output_length); - payload->size = encoder->output_length; + /* Copy the buffer data - msgpack_sbuffer uses flat memory, no zones */ + memcpy(payload->data, encoder->buffer.data, payload->size); payload->total_records = matched; flb_log_event_encoder_destroy(encoder); @@ -673,11 +881,15 @@ static int split_and_append_route_payloads(struct flb_input_instance *ins, flb_router_chunk_context_destroy(&context); route_payload_list_destroy(&payloads); flb_event_chunk_destroy(chunk); - return handled ? 1 : 0; + if (handled) { + return 1; + } + else { + return 0; + } } - records_array = flb_calloc(record_count, - sizeof(struct flb_mp_chunk_record *)); + records_array = flb_calloc(record_count, sizeof(struct flb_mp_chunk_record *)); if (!records_array) { flb_errno(); flb_router_chunk_context_destroy(&context); @@ -687,14 +899,9 @@ static int split_and_append_route_payloads(struct flb_input_instance *ins, } matched_non_default = flb_calloc(record_count, sizeof(uint8_t)); - if (!records_array || !matched_non_default) { + if (!matched_non_default) { flb_errno(); - if (records_array) { - flb_free(records_array); - } - if (matched_non_default) { - flb_free(matched_non_default); - } + flb_free(records_array); flb_router_chunk_context_destroy(&context); route_payload_list_destroy(&payloads); flb_event_chunk_destroy(chunk); @@ -712,7 +919,8 @@ static int split_and_append_route_payloads(struct flb_input_instance *ins, continue; } - ret = build_payload_for_route(payload, + ret = build_payload_for_route(ins, + payload, records_array, record_count, matched_non_default); @@ -732,7 +940,8 @@ static int split_and_append_route_payloads(struct flb_input_instance *ins, continue; } - ret = build_payload_for_default_route(payload, + ret = build_payload_for_default_route(ins, + payload, records_array, record_count, matched_non_default); @@ -751,7 +960,6 @@ static int split_and_append_route_payloads(struct flb_input_instance *ins, cfl_list_foreach_safe(head, tmp, &payloads) { payload = cfl_list_entry(head, struct flb_route_payload, _head); - if (payload->total_records <= 0 || !payload->data) { route_payload_destroy(payload); } @@ -761,6 +969,11 @@ static int split_and_append_route_payloads(struct flb_input_instance *ins, cfl_list_foreach(head, &payloads) { payload = cfl_list_entry(head, struct flb_route_payload, _head); + /* Skip payloads with no data or no records */ + if (payload->total_records <= 0 || !payload->data || payload->size == 0) { + continue; + } + ret = flb_input_chunk_append_raw(ins, FLB_INPUT_LOGS, payload->total_records, @@ -807,7 +1020,14 @@ static int split_and_append_route_payloads(struct flb_input_instance *ins, route_payload_list_destroy(&payloads); flb_event_chunk_destroy(chunk); - return handled ? (appended > 0 ? appended : 1) : 0; + if (handled) { + if (appended > 0) { + return appended; + } + return 1; + } + + return 0; } static int input_log_append(struct flb_input_instance *ins, diff --git a/src/flb_lib.c b/src/flb_lib.c index 30d4d99dd40..29bc200630a 100644 --- a/src/flb_lib.c +++ b/src/flb_lib.c @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -34,9 +35,13 @@ #include #include #include +#include #include #include +#include +#include +#include #ifdef FLB_HAVE_MTRACE #include @@ -675,17 +680,81 @@ int flb_service_set(flb_ctx_t *ctx, ...) /* Load a configuration file that may be used by the input or output plugin */ int flb_lib_config_file(struct flb_lib_ctx *ctx, const char *path) { - if (access(path, R_OK) != 0) { + struct flb_cf *cf; + int ret; + char tmp[PATH_MAX + 1]; + char *cfg = NULL; + char *end; + char *real_path; + struct stat st; + + /* Check if file exists and resolve path */ + ret = stat(path, &st); + if (ret == -1 && errno == ENOENT) { + /* Try to resolve the real path (if exists) */ + if (path[0] == '/') { + fprintf(stderr, "Error: configuration file not found: %s\n", path); + return -1; + } + + if (ctx->config->conf_path) { + snprintf(tmp, PATH_MAX, "%s%s", ctx->config->conf_path, path); + cfg = tmp; + } + else { + cfg = (char *) path; + } + } + else { + cfg = (char *) path; + } + + if (access(cfg, R_OK) != 0) { perror("access"); + fprintf(stderr, "Error: cannot read configuration file: %s\n", cfg); return -1; } - ctx->config->file = mk_rconf_open(path); - if (!ctx->config->file) { - fprintf(stderr, "Error reading configuration file: %s\n", path); + /* Use modern config format API that supports both .conf and .yaml/.yml */ + cf = flb_cf_create_from_file(NULL, cfg); + if (!cf) { + fprintf(stderr, "Error reading configuration file: %s\n", cfg); return -1; } + /* Set configuration root path */ + if (cfg) { + real_path = realpath(cfg, NULL); + if (real_path) { + end = strrchr(real_path, FLB_DIRCHAR); + if (end) { + end++; + *end = '\0'; + if (ctx->config->conf_path) { + flb_free(ctx->config->conf_path); + } + ctx->config->conf_path = flb_strdup(real_path); + } + free(real_path); + } + } + + /* Load the configuration format into the config */ + ret = flb_config_load_config_format(ctx->config, cf); + if (ret != 0) { + flb_cf_destroy(cf); + fprintf(stderr, "Error loading configuration from file: %s\n", cfg); + return -1; + } + + /* Destroy old cf_main if it exists (created by flb_config_init) */ + if (ctx->config->cf_main) { + flb_cf_destroy(ctx->config->cf_main); + } + + /* Store the config format object */ + ctx->config->cf_main = cf; + return 0; } @@ -803,7 +872,7 @@ int flb_lib_push(flb_ctx_t *ctx, int ffd, const void *data, size_t len) /* Emulate some data from the response */ int flb_lib_response(flb_ctx_t *ctx, int ffd, int status, const void *data, size_t len) { - int ret; + int ret = -1; struct flb_output_instance *o_ins; if (ctx->status == FLB_LIB_NONE || ctx->status == FLB_LIB_ERROR) { @@ -816,7 +885,7 @@ int flb_lib_response(flb_ctx_t *ctx, int ffd, int status, const void *data, size return -1; } - /* If input's test_formatter is registered, priorize to run it. */ + /* If output's test_response callback is registered, prioritize to run it. */ if (o_ins->test_response.callback != NULL) { ret = flb_output_run_response(ctx, o_ins, status, data, len); } @@ -965,8 +1034,9 @@ int flb_stop(flb_ctx_t *ctx) return 0; } - if (ctx->config->file) { - mk_rconf_free(ctx->config->file); + if (ctx->config->cf_main) { + flb_cf_destroy(ctx->config->cf_main); + ctx->config->cf_main = NULL; } flb_debug("[lib] sending STOP signal to the engine"); diff --git a/src/flb_log_event_decoder.c b/src/flb_log_event_decoder.c index f500d17aa6d..f47111a987e 100644 --- a/src/flb_log_event_decoder.c +++ b/src/flb_log_event_decoder.c @@ -20,6 +20,9 @@ #include #include #include +#include + +#define FLB_LOG_EVENT_DECODER_MAX_RECURSION_DEPTH 1000 /* Safety limit for recursion */ static int create_empty_map(struct flb_log_event_decoder *context) { msgpack_packer packer; @@ -74,6 +77,7 @@ void flb_log_event_decoder_reset(struct flb_log_event_decoder *context, context->last_result = FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA; context->current_group_metadata = NULL; context->current_group_attributes = NULL; + context->recursion_depth = 0; /* Reset recursion counter */ msgpack_unpacked_destroy(&context->unpacked_group_record); msgpack_unpacked_init(&context->unpacked_group_record); @@ -150,11 +154,6 @@ void flb_log_event_decoder_destroy(struct flb_log_event_decoder *context) if (context != NULL) { if (context->initialized) { - if (context->unpacked_group_record.zone == - context->unpacked_event.zone) { - msgpack_unpacked_init(&context->unpacked_event); - } - msgpack_unpacked_destroy(&context->unpacked_group_record); msgpack_unpacked_destroy(&context->unpacked_empty_map); msgpack_unpacked_destroy(&context->unpacked_event); @@ -314,6 +313,7 @@ int flb_log_event_decoder_next(struct flb_log_event_decoder *context, int result; int record_type; size_t previous_offset; + int32_t invalid_timestamp; if (context == NULL) { return FLB_EVENT_DECODER_ERROR_INVALID_CONTEXT; @@ -331,11 +331,6 @@ int flb_log_event_decoder_next(struct flb_log_event_decoder *context, return context->last_result; } - if (context->unpacked_group_record.zone == - context->unpacked_event.zone) { - msgpack_unpacked_init(&context->unpacked_event); - } - previous_offset = context->offset; result = msgpack_unpack_next(&context->unpacked_event, context->buffer, @@ -357,33 +352,80 @@ int flb_log_event_decoder_next(struct flb_log_event_decoder *context, &context->unpacked_event.data); if (context->last_result == FLB_EVENT_DECODER_SUCCESS) { + /* Check recursion depth limit to prevent stack overflow */ + if (context->recursion_depth >= FLB_LOG_EVENT_DECODER_MAX_RECURSION_DEPTH) { + flb_warn("[decoder] Maximum recursion depth (%d) reached, possible corruption or excessive group markers", + FLB_LOG_EVENT_DECODER_MAX_RECURSION_DEPTH); + context->last_result = FLB_EVENT_DECODER_ERROR_DESERIALIZATION_FAILURE; + return context->last_result; + } + /* get log event type */ ret = flb_log_event_decoder_get_record_type(event, &record_type); if (ret != 0) { - context->current_group_metadata = NULL; - context->current_group_attributes = NULL; - - context->last_result = FLB_EVENT_DECODER_ERROR_DESERIALIZATION_FAILURE; - return context->last_result; + /* Invalid group marker (negative timestamp but not -1 or -2). + * Log the invalid marker for debugging, but preserve group state + * to avoid losing valid group metadata if corruption occurs mid-group. + * Skip the record and continue processing. + */ + invalid_timestamp = (int32_t) event->timestamp.tm.tv_sec; + flb_debug("[decoder] Invalid group marker timestamp (%d), skipping record. " + "Group state preserved.", invalid_timestamp); + + /* Increment recursion depth before recursive call */ + context->recursion_depth++; + memset(event, 0, sizeof(struct flb_log_event)); + ret = flb_log_event_decoder_next(context, event); + context->recursion_depth--; /* Restore after return */ + return ret; } /* Meta records such as the group opener and closer are identified by negative - * timestamp values. In these cases we track the current group metadata and - * attributes in order to transparently provide them through the log_event - * structure but we also want to allow the client code raw access to such - * records which is why the read_groups decoder context property is used - * to determine the behavior. + * timestamp values (-1 for GROUP_START, -2 for GROUP_END). Only these two + * negative values are valid; any other negative timestamp is considered + * invalid and is skipped (see handling above). + * + * We track the current group metadata and attributes in order to transparently + * provide them through the log_event structure, but we also want to allow the + * client code raw access to such records, which is why the read_groups decoder + * context property is used to determine the behavior. */ if (record_type != FLB_LOG_EVENT_NORMAL) { msgpack_unpacked_destroy(&context->unpacked_group_record); if (record_type == FLB_LOG_EVENT_GROUP_START) { - memcpy(&context->unpacked_group_record, - &context->unpacked_event, - sizeof(msgpack_unpacked)); - - context->current_group_metadata = event->metadata; - context->current_group_attributes = event->body; + /* + * Transfer zone ownership from unpacked_event to unpacked_group_record + * instead of using memcpy. This prevents double-free issues when + * both structures would otherwise reference the same zone. + */ + context->unpacked_group_record.zone = msgpack_unpacked_release_zone(&context->unpacked_event); + context->unpacked_group_record.data = context->unpacked_event.data; + + /* + * Extract pointers from the transferred data structure to ensure they + * remain valid. The pointers must come from unpacked_group_record.data + * since that's where the zone (and the data) now reside. + */ + if (context->unpacked_group_record.data.type == MSGPACK_OBJECT_ARRAY && + context->unpacked_group_record.data.via.array.size == 2) { + msgpack_object *header = &context->unpacked_group_record.data.via.array.ptr[0]; + msgpack_object *root_body = &context->unpacked_group_record.data.via.array.ptr[1]; + + if (header->type == MSGPACK_OBJECT_ARRAY && + header->via.array.size == 2) { + context->current_group_metadata = &header->via.array.ptr[1]; + } + else { + context->current_group_metadata = context->empty_map; + } + context->current_group_attributes = root_body; + } + else { + /* Fallback to using event pointers if structure is unexpected */ + context->current_group_metadata = event->metadata; + context->current_group_attributes = event->body; + } } else { context->current_group_metadata = NULL; @@ -391,9 +433,17 @@ int flb_log_event_decoder_next(struct flb_log_event_decoder *context, } if (context->read_groups != FLB_TRUE) { + /* + * Skip group markers by recursively calling to get next record. + * msgpack_unpack_next will properly destroy and reinitialize + * unpacked_event, so no explicit cleanup needed here. + * Increment recursion depth before recursive call. + */ + context->recursion_depth++; memset(event, 0, sizeof(struct flb_log_event)); - - return flb_log_event_decoder_next(context, event); + ret = flb_log_event_decoder_next(context, event); + context->recursion_depth--; /* Restore after return */ + return ret; } } else { diff --git a/src/flb_router_condition.c b/src/flb_router_condition.c index d24fcc67ba1..212b89c60e2 100644 --- a/src/flb_router_condition.c +++ b/src/flb_router_condition.c @@ -24,12 +24,131 @@ #include #include #include +#include #define FLB_ROUTE_CONDITION_COMPILED_SUCCESS 1 #define FLB_ROUTE_CONDITION_COMPILED_FAILURE -1 static struct flb_condition *route_condition_get_compiled(struct flb_route_condition *condition); +static inline struct cfl_variant *get_object_variant(struct cfl_object *object) +{ + if (!object) { + return NULL; + } + + return object->variant; +} + +static inline struct cfl_variant *get_body_variant(struct flb_mp_chunk_record *record) +{ + if (!record || !record->cobj_record) { + return NULL; + } + + return record->cobj_record->variant; +} + +static struct cfl_variant *get_otel_container_variant(struct flb_mp_chunk_record *record, + const char *key, + int use_group_attributes) +{ + struct cfl_variant *source; + struct cfl_variant *container; + + /* For OTLP, resource/scope attributes are in group_attributes, not body */ + if (use_group_attributes && record->cobj_group_attributes && record->cobj_group_attributes->variant) { + source = record->cobj_group_attributes->variant; + } + else { + source = get_body_variant(record); + } + + if (!source || source->type != CFL_VARIANT_KVLIST) { + return NULL; + } + + container = cfl_kvlist_fetch(source->data.as_kvlist, key); + if (!container || container->type != CFL_VARIANT_KVLIST) { + return NULL; + } + + return container; +} + +static struct cfl_variant *get_otel_attributes_variant(struct flb_mp_chunk_record *record, + enum record_context_type context_type) +{ + struct cfl_variant *container; + const char *container_key = NULL; + + if (context_type == RECORD_CONTEXT_OTEL_RESOURCE_ATTRIBUTES) { + container_key = "resource"; + } + else if (context_type == RECORD_CONTEXT_OTEL_SCOPE_ATTRIBUTES) { + container_key = "scope"; + } + else { + return NULL; + } + + /* For OTLP resource/scope attributes, look in group_attributes first */ + container = get_otel_container_variant(record, container_key, 1); + if (!container) { + return NULL; + } + + container = cfl_kvlist_fetch(container->data.as_kvlist, "attributes"); + if (!container || container->type != CFL_VARIANT_KVLIST) { + return NULL; + } + + return container; +} + +static struct cfl_variant *get_otel_scope_metadata_variant(struct flb_mp_chunk_record *record) +{ + struct cfl_variant *scope; + + /* For OTLP scope metadata, also check group_attributes first */ + scope = get_otel_container_variant(record, "scope", 1); + if (!scope || scope->type != CFL_VARIANT_KVLIST) { + return NULL; + } + + return scope; +} + +static struct cfl_variant *route_logs_get_variant(struct flb_condition_rule *rule, + void *ctx) +{ + struct flb_mp_chunk_record *record = (struct flb_mp_chunk_record *) ctx; + + if (!rule || !record) { + return NULL; + } + + switch (rule->context) { + case RECORD_CONTEXT_METADATA: + return get_object_variant(record->cobj_metadata); + case RECORD_CONTEXT_BODY: + return get_body_variant(record); + case RECORD_CONTEXT_GROUP_METADATA: + return get_object_variant(record->cobj_group_metadata); + case RECORD_CONTEXT_GROUP_ATTRIBUTES: + return get_object_variant(record->cobj_group_attributes); + case RECORD_CONTEXT_OTEL_RESOURCE_ATTRIBUTES: + case RECORD_CONTEXT_OTEL_SCOPE_ATTRIBUTES: + return get_otel_attributes_variant(record, rule->context); + case RECORD_CONTEXT_OTEL_SCOPE_METADATA: + return get_otel_scope_metadata_variant(record); + default: + break; + } + + return NULL; +} + int flb_router_chunk_context_init(struct flb_router_chunk_context *context) { if (!context) { @@ -185,7 +304,7 @@ int flb_condition_eval_logs(struct flb_event_chunk *chunk, cfl_list_foreach(head, &context->chunk_cobj->records) { record = cfl_list_entry(head, struct flb_mp_chunk_record, _head); - if (flb_condition_evaluate(compiled, record) == FLB_TRUE) { + if (flb_condition_evaluate_ex(compiled, record, route_logs_get_variant) == FLB_TRUE) { result = FLB_TRUE; break; } @@ -291,6 +410,31 @@ struct flb_condition *flb_router_route_get_condition(struct flb_route *route) return route_condition_get_compiled(route->condition); } +int flb_router_condition_evaluate_record(struct flb_route *route, + struct flb_mp_chunk_record *record) +{ + struct flb_condition *compiled; + + if (!route || !record) { + return FLB_FALSE; + } + + if (!route->condition) { + return FLB_TRUE; + } + + compiled = flb_router_route_get_condition(route); + if (!compiled) { + if (route->condition->is_default) { + return FLB_TRUE; + } + + return FLB_FALSE; + } + + return flb_condition_evaluate_ex(compiled, record, route_logs_get_variant); +} + static int parse_rule_operator(const flb_sds_t op_str, enum flb_rule_operator *out) { @@ -391,7 +535,7 @@ static struct flb_condition *route_condition_compile(struct flb_route_condition return NULL; } ret = flb_condition_add_rule(compiled, rule->field, op, - rule->value, 1, RECORD_CONTEXT_BODY); + rule->value, 1, rule->context); break; case FLB_RULE_OP_GT: case FLB_RULE_OP_LT: @@ -406,7 +550,7 @@ static struct flb_condition *route_condition_compile(struct flb_route_condition return NULL; } ret = flb_condition_add_rule(compiled, rule->field, op, - &numeric_value, 1, RECORD_CONTEXT_BODY); + &numeric_value, 1, rule->context); break; case FLB_RULE_OP_IN: case FLB_RULE_OP_NOT_IN: @@ -417,7 +561,7 @@ static struct flb_condition *route_condition_compile(struct flb_route_condition ret = flb_condition_add_rule(compiled, rule->field, op, rule->values, (int) rule->values_count, - RECORD_CONTEXT_BODY); + rule->context); break; default: flb_condition_destroy(compiled); diff --git a/src/flb_router_config.c b/src/flb_router_config.c index da2da77ba6f..9c21a481161 100644 --- a/src/flb_router_config.c +++ b/src/flb_router_config.c @@ -515,6 +515,64 @@ static int parse_processors(struct cfl_variant *variant, return 0; } +static int parse_condition_rule_context(const char *value, + enum record_context_type *out_context) +{ + if (!out_context) { + return -1; + } + + if (!value) { + *out_context = RECORD_CONTEXT_BODY; + return 0; + } + + if (strcasecmp(value, "metadata") == 0 || + strcasecmp(value, "record_metadata") == 0 || + strcasecmp(value, "attributes") == 0) { + *out_context = RECORD_CONTEXT_METADATA; + return 0; + } + + if (strcasecmp(value, "body") == 0 || + strcasecmp(value, "record") == 0 || + strcasecmp(value, "message") == 0 || + strcasecmp(value, "record_body") == 0) { + *out_context = RECORD_CONTEXT_BODY; + return 0; + } + + if (strcasecmp(value, "group_metadata") == 0) { + *out_context = RECORD_CONTEXT_GROUP_METADATA; + return 0; + } + + if (strcasecmp(value, "group_attributes") == 0 || + strcasecmp(value, "group_body") == 0) { + *out_context = RECORD_CONTEXT_GROUP_ATTRIBUTES; + return 0; + } + + if (strcasecmp(value, "otel_resource_attributes") == 0) { + *out_context = RECORD_CONTEXT_OTEL_RESOURCE_ATTRIBUTES; + return 0; + } + + if (strcasecmp(value, "otel_scope_attributes") == 0) { + *out_context = RECORD_CONTEXT_OTEL_SCOPE_ATTRIBUTES; + return 0; + } + + if (strcasecmp(value, "otel_scope_name") == 0 || + strcasecmp(value, "otel_scope_version") == 0 || + strcasecmp(value, "otel_scope_metadata") == 0) { + *out_context = RECORD_CONTEXT_OTEL_SCOPE_METADATA; + return 0; + } + + return -1; +} + static struct flb_route_condition_rule *parse_condition_rule(struct cfl_variant *variant) { struct flb_route_condition_rule *rule; @@ -522,6 +580,7 @@ static struct flb_route_condition_rule *parse_condition_rule(struct cfl_variant struct cfl_variant *field_var; struct cfl_variant *op_var; struct cfl_variant *value_var; + struct cfl_variant *context_var; if (!variant || variant->type != CFL_VARIANT_KVLIST) { return NULL; @@ -547,6 +606,7 @@ static struct flb_route_condition_rule *parse_condition_rule(struct cfl_variant cfl_list_init(&rule->_head); rule->values = NULL; rule->values_count = 0; + rule->context = RECORD_CONTEXT_BODY; rule->field = copy_from_cfl_sds(field_var->data.as_string); if (!rule->field) { @@ -618,6 +678,32 @@ static struct flb_route_condition_rule *parse_condition_rule(struct cfl_variant } } + context_var = cfl_kvlist_fetch(kvlist, "context"); + if (context_var) { + if (context_var->type != CFL_VARIANT_STRING || + parse_condition_rule_context(context_var->data.as_string, &rule->context) != 0) { + size_t j; + + if (rule->values) { + for (j = 0; j < rule->values_count; j++) { + if (rule->values[j]) { + flb_sds_destroy(rule->values[j]); + } + } + flb_free(rule->values); + } + + if (rule->value) { + flb_sds_destroy(rule->value); + } + + flb_sds_destroy(rule->op); + flb_sds_destroy(rule->field); + flb_free(rule); + return NULL; + } + } + return rule; } @@ -1310,6 +1396,8 @@ int flb_router_apply_config(struct flb_config *config) continue; } + route_output->ins = output_ins; + if (input_has_direct_route(input_ins, output_ins)) { continue; } diff --git a/src/flb_task.c b/src/flb_task.c index a829b73abf3..5dcf6fdd334 100644 --- a/src/flb_task.c +++ b/src/flb_task.c @@ -440,6 +440,15 @@ struct flb_task *flb_task_create(uint64_t ref_id, o_ins = route_path->ins; + /* For conditional routing, also check the route mask */ + if (task_ic->routes_mask) { + if (flb_routes_mask_get_bit(task_ic->routes_mask, + o_ins->id, + o_ins->config) == 0) { + continue; + } + } + route = flb_calloc(1, sizeof(struct flb_task_route)); if (!route) { flb_errno(); diff --git a/tests/internal/conditional_routing.c b/tests/internal/conditional_routing.c new file mode 100644 index 00000000000..430580e6bcf --- /dev/null +++ b/tests/internal/conditional_routing.c @@ -0,0 +1,983 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "flb_tests_internal.h" + +/* Test data structures */ +struct test_log_record { + const char *level; + const char *service; + const char *message; + const char *expected_route; +}; + +static const struct test_log_record test_records[] = { + {"info", "web-server", "Application started successfully", "info_logs"}, + {"error", "database", "Database connection failed", "error_logs"}, + {"undef", "logger", "Unknown log level detected", "default_logs"}, + {"info", "auth", "User authentication successful", "info_logs"}, + {"error", "file-service", "File not found", "error_logs"}, + {"undef", "parser", "Invalid log format", "default_logs"}, + {"info", "cache", "Cache updated successfully", "info_logs"}, + {"error", "memory-manager", "Memory allocation failed", "error_logs"}, + {"undef", "event-processor", "Unrecognized event type", "default_logs"}, + {"info", "test", "Test log entry", "info_logs"} +}; + +static const size_t test_records_count = sizeof(test_records) / sizeof(test_records[0]); + +/* Helper function to create a test log chunk */ +static int create_test_log_chunk(const char *level, + const char *service, + const char *message, + struct flb_log_event_encoder *encoder, + struct flb_event_chunk *chunk) +{ + int ret; + + if (!level || !service || !message || !encoder || !chunk) { + return -1; + } + + ret = flb_log_event_encoder_init(encoder, FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + return -1; + } + + ret = flb_log_event_encoder_begin_record(encoder); + TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_set_current_timestamp(encoder); + TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_body_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE("level", 5), + FLB_LOG_EVENT_CSTRING_VALUE(level), + FLB_LOG_EVENT_STRING_VALUE("service", 7), + FLB_LOG_EVENT_CSTRING_VALUE(service), + FLB_LOG_EVENT_STRING_VALUE("message", 7), + FLB_LOG_EVENT_CSTRING_VALUE(message)); + TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_commit_record(encoder); + TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + memset(chunk, 0, sizeof(*chunk)); + chunk->type = FLB_EVENT_TYPE_LOGS; + chunk->data = encoder->output_buffer; + chunk->size = encoder->output_length; + chunk->total_events = 1; + + return 0; +} + +/* Test conditional routing configuration parsing */ +void test_conditional_routing_config_parse() +{ + struct cfl_list routes; + struct cfl_variant *inputs; + struct flb_cf *cf; + struct flb_input_routes *input_routes; + struct flb_route *route; + struct cfl_list *head; + struct cfl_list *route_head; + int ret; + int seen_info = 0; + int seen_error = 0; + int seen_default = 0; + + cfl_list_init(&routes); + + /* Create test configuration */ + inputs = create_conditional_routing_inputs(); + TEST_CHECK(inputs != NULL); + if (!inputs) { + return; + } + + cf = cf_from_inputs_variant(inputs); + TEST_CHECK(cf != NULL); + if (!cf) { + cfl_variant_destroy(inputs); + return; + } + + ret = flb_router_config_parse(cf, &routes, NULL); + TEST_CHECK(ret == 0); + if (ret == 0) { + TEST_CHECK(cfl_list_size(&routes) == 1); + head = routes.next; + input_routes = cfl_list_entry(head, struct flb_input_routes, _head); + TEST_CHECK(strcmp(input_routes->input_name, "tail") == 0); + TEST_CHECK(cfl_list_size(&input_routes->routes) == 3); + + cfl_list_foreach(route_head, &input_routes->routes) { + route = cfl_list_entry(route_head, struct flb_route, _head); + if (strcmp(route->name, "info_logs") == 0) { + seen_info = 1; + TEST_CHECK(route->per_record_routing == FLB_TRUE); + TEST_CHECK(route->condition != NULL); + TEST_CHECK(route->condition->is_default == FLB_FALSE); + } + else if (strcmp(route->name, "error_logs") == 0) { + seen_error = 1; + TEST_CHECK(route->per_record_routing == FLB_TRUE); + TEST_CHECK(route->condition != NULL); + TEST_CHECK(route->condition->is_default == FLB_FALSE); + } + else if (strcmp(route->name, "default_logs") == 0) { + seen_default = 1; + TEST_CHECK(route->per_record_routing == FLB_TRUE); + TEST_CHECK(route->condition != NULL); + TEST_CHECK(route->condition->is_default == FLB_TRUE); + } + } + + TEST_CHECK(seen_info == 1); + TEST_CHECK(seen_error == 1); + TEST_CHECK(seen_default == 1); + + flb_router_routes_destroy(&routes); + } + + flb_cf_destroy(cf); + cfl_variant_destroy(inputs); +} + +/* Test condition evaluation for individual records */ +void test_conditional_routing_condition_eval() +{ + struct flb_route route; + struct flb_route_condition *condition; + struct flb_route_condition_rule *rule; + struct flb_log_event_encoder encoder; + struct flb_event_chunk chunk; + struct flb_router_chunk_context context; + int ret; + size_t i; + + memset(&route, 0, sizeof(route)); + cfl_list_init(&route.outputs); + cfl_list_init(&route.processors); + + /* Create condition: level == "info" */ + condition = flb_calloc(1, sizeof(struct flb_route_condition)); + TEST_CHECK(condition != NULL); + if (!condition) { + return; + } + + cfl_list_init(&condition->rules); + condition->op = FLB_COND_OP_AND; + condition->compiled_status = 0; + condition->compiled = NULL; + condition->is_default = FLB_FALSE; + + rule = flb_calloc(1, sizeof(struct flb_route_condition_rule)); + TEST_CHECK(rule != NULL); + if (!rule) { + flb_free(condition); + return; + } + + cfl_list_init(&rule->_head); + rule->field = flb_sds_create("$level"); + rule->op = flb_sds_create("eq"); + rule->value = flb_sds_create("info"); + TEST_CHECK(rule->field != NULL && rule->op != NULL && rule->value != NULL); + + cfl_list_add(&rule->_head, &condition->rules); + route.condition = condition; + route.signals = FLB_ROUTER_SIGNAL_LOGS; + route.per_record_routing = FLB_TRUE; + + flb_router_chunk_context_init(&context); + + /* Test each record */ + for (i = 0; i < test_records_count; i++) { + const struct test_log_record *record = &test_records[i]; + int expected_result = (strcmp(record->level, "info") == 0) ? FLB_TRUE : FLB_FALSE; + + ret = create_test_log_chunk(record->level, record->service, record->message, + &encoder, &chunk); + TEST_CHECK(ret == 0); + if (ret == 0) { + int result = flb_condition_eval_logs(&chunk, &context, &route); + TEST_CHECK(result == expected_result); + if (result != expected_result) { + fprintf(stderr, "Condition evaluation failed for record %zu: level=%s, expected=%d, got=%d\n", + i, record->level, expected_result, result); + } + } + flb_router_chunk_context_reset(&context); + flb_log_event_encoder_destroy(&encoder); + } + + flb_router_chunk_context_destroy(&context); + flb_free(condition); + flb_sds_destroy(rule->field); + flb_sds_destroy(rule->op); + flb_sds_destroy(rule->value); + flb_free(rule); +} + +/* Test per-record routing functionality */ +void test_conditional_routing_per_record() +{ + struct flb_config config; + struct flb_input_instance input; + struct flb_output_instance output1, output2, output3; + struct flb_input_plugin input_plugin; + struct flb_output_plugin output_plugin; + struct flb_input_routes input_routes; + struct flb_route route1, route2, route3; + struct flb_route_output route_output1, route_output2, route_output3; + struct flb_router_path *path; + struct flb_log_event_encoder encoder; + struct flb_event_chunk chunk; + int ret; + size_t i; + + /* Setup test instances */ + setup_conditional_routing_instances(&config, &input, &input_plugin, + &output1, &output2, &output3, &output_plugin); + + /* Setup routes */ + setup_conditional_routes(&input_routes, &route1, &route2, &route3, + &route_output1, &route_output2, &route_output3, + &output1, &output2, &output3); + + /* Apply configuration */ + ret = flb_router_apply_config(&config); + TEST_CHECK(ret == 0); + TEST_CHECK(cfl_list_size(&input.routes_direct) == 3); + + /* Test per-record routing for each test record */ + for (i = 0; i < test_records_count; i++) { + const struct test_log_record *record = &test_records[i]; + + ret = create_test_log_chunk(record->level, record->service, record->message, + &encoder, &chunk); + TEST_CHECK(ret == 0); + if (ret == 0) { + /* Test that the record routes to the expected output */ + int routed_correctly = test_record_routing(&input, &chunk, record->expected_route); + TEST_CHECK(routed_correctly == 1); + if (!routed_correctly) { + fprintf(stderr, "Record %zu did not route correctly: level=%s, expected=%s\n", + i, record->level, record->expected_route); + } + } + flb_log_event_encoder_destroy(&encoder); + } + + /* Cleanup */ + flb_router_exit(&config); + cleanup_conditional_routing_instances(&input, &output1, &output2, &output3, + &input_routes, &route1, &route2, &route3, + &route_output1, &route_output2, &route_output3); +} + +/* Test default route handling */ +void test_conditional_routing_default_route() +{ + struct flb_config config; + struct flb_input_instance input; + struct flb_output_instance output1, output2, output3; + struct flb_input_plugin input_plugin; + struct flb_output_plugin output_plugin; + struct flb_input_routes input_routes; + struct flb_route route1, route2, route3; + struct flb_route_output route_output1, route_output2, route_output3; + struct flb_log_event_encoder encoder; + struct flb_event_chunk chunk; + int ret; + size_t i; + int default_route_count = 0; + + /* Setup test instances */ + setup_conditional_routing_instances(&config, &input, &input_plugin, + &output1, &output2, &output3, &output_plugin); + + /* Setup routes */ + setup_conditional_routes(&input_routes, &route1, &route2, &route3, + &route_output1, &route_output2, &route_output3, + &output1, &output2, &output3); + + /* Apply configuration */ + ret = flb_router_apply_config(&config); + TEST_CHECK(ret == 0); + + /* Test that records with "undef" level go to default route */ + for (i = 0; i < test_records_count; i++) { + const struct test_log_record *record = &test_records[i]; + + if (strcmp(record->level, "undef") == 0) { + ret = create_test_log_chunk(record->level, record->service, record->message, + &encoder, &chunk); + TEST_CHECK(ret == 0); + if (ret == 0) { + int routed_to_default = test_record_routing(&input, &chunk, "default_logs"); + TEST_CHECK(routed_to_default == 1); + if (routed_to_default) { + default_route_count++; + } + } + flb_log_event_encoder_destroy(&encoder); + } + } + + /* Verify that all "undef" records went to default route */ + TEST_CHECK(default_route_count == 3); /* Should have 3 "undef" records */ + + /* Cleanup */ + flb_router_exit(&config); + cleanup_conditional_routing_instances(&input, &output1, &output2, &output3, + &input_routes, &route1, &route2, &route3, + &route_output1, &route_output2, &route_output3); +} + +/* Test route mask functionality */ +void test_conditional_routing_route_mask() +{ + struct flb_config config; + struct flb_input_instance input; + struct flb_output_instance output1, output2, output3; + struct flb_input_plugin input_plugin; + struct flb_output_plugin output_plugin; + struct flb_input_routes input_routes; + struct flb_route route1, route2, route3; + struct flb_route_output route_output1, route_output2, route_output3; + struct flb_input_chunk *chunk; + flb_route_mask_element *routes_mask; + int ret; + size_t i; + + /* Setup test instances */ + setup_conditional_routing_instances(&config, &input, &input_plugin, + &output1, &output2, &output3, &output_plugin); + + /* Setup routes */ + setup_conditional_routes(&input_routes, &route1, &route2, &route3, + &route_output1, &route_output2, &route_output3, + &output1, &output2, &output3); + + /* Apply configuration */ + ret = flb_router_apply_config(&config); + TEST_CHECK(ret == 0); + + /* Test route mask for info records */ + for (i = 0; i < test_records_count; i++) { + const struct test_log_record *record = &test_records[i]; + + if (strcmp(record->level, "info") == 0) { + /* Create a test chunk */ + chunk = flb_input_chunk_create(&input, "test_tag", 8, NULL, 0); + TEST_CHECK(chunk != NULL); + if (chunk) { + /* Set route mask for info output only */ + routes_mask = chunk->routes_mask; + flb_routes_mask_set_bit(routes_mask, output1.id, &config); + + /* Verify route mask is set correctly */ + TEST_CHECK(flb_routes_mask_get_bit(routes_mask, output1.id, &config) == 1); + TEST_CHECK(flb_routes_mask_get_bit(routes_mask, output2.id, &config) == 0); + TEST_CHECK(flb_routes_mask_get_bit(routes_mask, output3.id, &config) == 0); + + flb_input_chunk_destroy(chunk); + } + } + } + + /* Cleanup */ + flb_router_exit(&config); + cleanup_conditional_routing_instances(&input, &output1, &output2, &output3, + &input_routes, &route1, &route2, &route3, + &route_output1, &route_output2, &route_output3); +} + +/* Test no duplicate routing */ +void test_conditional_routing_no_duplicates() +{ + struct flb_config config; + struct flb_input_instance input; + struct flb_output_instance output1, output2, output3; + struct flb_input_plugin input_plugin; + struct flb_output_plugin output_plugin; + struct flb_input_routes input_routes; + struct flb_route route1, route2, route3; + struct flb_route_output route_output1, route_output2, route_output3; + struct flb_log_event_encoder encoder; + struct flb_event_chunk chunk; + int ret; + size_t i; + int total_routed = 0; + int info_routed = 0; + int error_routed = 0; + int default_routed = 0; + + /* Setup test instances */ + setup_conditional_routing_instances(&config, &input, &input_plugin, + &output1, &output2, &output3, &output_plugin); + + /* Setup routes */ + setup_conditional_routes(&input_routes, &route1, &route2, &route3, + &route_output1, &route_output2, &route_output3, + &output1, &output2, &output3); + + /* Apply configuration */ + ret = flb_router_apply_config(&config); + TEST_CHECK(ret == 0); + + /* Test all records and count routing */ + for (i = 0; i < test_records_count; i++) { + const struct test_log_record *record = &test_records[i]; + + ret = create_test_log_chunk(record->level, record->service, record->message, + &encoder, &chunk); + TEST_CHECK(ret == 0); + if (ret == 0) { + int routed = test_record_routing(&input, &chunk, record->expected_route); + TEST_CHECK(routed == 1); + if (routed == 1) { + total_routed++; + if (strcmp(record->expected_route, "info_logs") == 0) { + info_routed++; + } else if (strcmp(record->expected_route, "error_logs") == 0) { + error_routed++; + } else if (strcmp(record->expected_route, "default_logs") == 0) { + default_routed++; + } + } + } + flb_log_event_encoder_destroy(&encoder); + } + + /* Verify no duplicates - each record should be routed exactly once */ + TEST_CHECK(total_routed == test_records_count); + TEST_CHECK(info_routed == 4); /* 4 info records */ + TEST_CHECK(error_routed == 3); /* 3 error records */ + TEST_CHECK(default_routed == 3); /* 3 default records */ + + /* Cleanup */ + flb_router_exit(&config); + cleanup_conditional_routing_instances(&input, &output1, &output2, &output3, + &input_routes, &route1, &route2, &route3, + &route_output1, &route_output2, &route_output3); +} + +/* Helper functions for test setup */ + +static struct cfl_variant *create_conditional_routing_inputs() +{ + struct cfl_array *inputs; + struct cfl_kvlist *input; + struct cfl_kvlist *routes; + struct cfl_array *log_routes; + struct cfl_kvlist *route; + struct cfl_kvlist *condition; + struct cfl_array *rules; + struct cfl_kvlist *rule_kv; + struct cfl_variant *rule_variant; + struct cfl_array *outputs; + struct cfl_kvlist *to; + struct cfl_variant *inputs_variant; + + inputs = cfl_array_create(1); + TEST_CHECK(inputs != NULL); + if (!inputs) { + return NULL; + } + + input = cfl_kvlist_create(); + TEST_CHECK(input != NULL); + if (!input) { + cfl_array_destroy(inputs); + return NULL; + } + + TEST_CHECK(cfl_kvlist_insert_string(input, "name", "tail") == 0); + + routes = cfl_kvlist_create(); + TEST_CHECK(routes != NULL); + log_routes = cfl_array_create(3); + TEST_CHECK(log_routes != NULL); + + /* info_logs route */ + route = cfl_kvlist_create(); + TEST_CHECK(route != NULL); + TEST_CHECK(cfl_kvlist_insert_string(route, "name", "info_logs") == 0); + TEST_CHECK(cfl_kvlist_insert_bool(route, "per_record_routing", 1) == 0); + + condition = cfl_kvlist_create(); + TEST_CHECK(condition != NULL); + rules = cfl_array_create(1); + TEST_CHECK(rules != NULL); + + rule_kv = cfl_kvlist_create(); + TEST_CHECK(rule_kv != NULL); + TEST_CHECK(cfl_kvlist_insert_string(rule_kv, "field", "$level") == 0); + TEST_CHECK(cfl_kvlist_insert_string(rule_kv, "op", "eq") == 0); + TEST_CHECK(cfl_kvlist_insert_string(rule_kv, "value", "info") == 0); + rule_variant = cfl_variant_create_from_kvlist(rule_kv); + TEST_CHECK(rule_variant != NULL); + TEST_CHECK(cfl_array_append(rules, rule_variant) == 0); + + TEST_CHECK(cfl_kvlist_insert_array(condition, "rules", rules) == 0); + TEST_CHECK(cfl_kvlist_insert_kvlist(route, "condition", condition) == 0); + + outputs = cfl_array_create(1); + TEST_CHECK(outputs != NULL); + TEST_CHECK(cfl_array_append_string(outputs, "info_destination") == 0); + to = cfl_kvlist_create(); + TEST_CHECK(to != NULL); + TEST_CHECK(cfl_kvlist_insert_array(to, "outputs", outputs) == 0); + TEST_CHECK(cfl_kvlist_insert_kvlist(route, "to", to) == 0); + + TEST_CHECK(cfl_array_append(log_routes, cfl_variant_create_from_kvlist(route)) == 0); + + /* error_logs route */ + route = cfl_kvlist_create(); + TEST_CHECK(route != NULL); + TEST_CHECK(cfl_kvlist_insert_string(route, "name", "error_logs") == 0); + TEST_CHECK(cfl_kvlist_insert_bool(route, "per_record_routing", 1) == 0); + + condition = cfl_kvlist_create(); + TEST_CHECK(condition != NULL); + rules = cfl_array_create(1); + TEST_CHECK(rules != NULL); + + rule_kv = cfl_kvlist_create(); + TEST_CHECK(rule_kv != NULL); + TEST_CHECK(cfl_kvlist_insert_string(rule_kv, "field", "$level") == 0); + TEST_CHECK(cfl_kvlist_insert_string(rule_kv, "op", "eq") == 0); + TEST_CHECK(cfl_kvlist_insert_string(rule_kv, "value", "error") == 0); + rule_variant = cfl_variant_create_from_kvlist(rule_kv); + TEST_CHECK(rule_variant != NULL); + TEST_CHECK(cfl_array_append(rules, rule_variant) == 0); + + TEST_CHECK(cfl_kvlist_insert_array(condition, "rules", rules) == 0); + TEST_CHECK(cfl_kvlist_insert_kvlist(route, "condition", condition) == 0); + + outputs = cfl_array_create(1); + TEST_CHECK(outputs != NULL); + TEST_CHECK(cfl_array_append_string(outputs, "error_destination") == 0); + to = cfl_kvlist_create(); + TEST_CHECK(to != NULL); + TEST_CHECK(cfl_kvlist_insert_array(to, "outputs", outputs) == 0); + TEST_CHECK(cfl_kvlist_insert_kvlist(route, "to", to) == 0); + + TEST_CHECK(cfl_array_append(log_routes, cfl_variant_create_from_kvlist(route)) == 0); + + /* default_logs route */ + route = cfl_kvlist_create(); + TEST_CHECK(route != NULL); + TEST_CHECK(cfl_kvlist_insert_string(route, "name", "default_logs") == 0); + TEST_CHECK(cfl_kvlist_insert_bool(route, "per_record_routing", 1) == 0); + + condition = cfl_kvlist_create(); + TEST_CHECK(condition != NULL); + TEST_CHECK(cfl_kvlist_insert_bool(condition, "default", 1) == 0); + TEST_CHECK(cfl_kvlist_insert_kvlist(route, "condition", condition) == 0); + + outputs = cfl_array_create(1); + TEST_CHECK(outputs != NULL); + TEST_CHECK(cfl_array_append_string(outputs, "default_destination") == 0); + to = cfl_kvlist_create(); + TEST_CHECK(to != NULL); + TEST_CHECK(cfl_kvlist_insert_array(to, "outputs", outputs) == 0); + TEST_CHECK(cfl_kvlist_insert_kvlist(route, "to", to) == 0); + + TEST_CHECK(cfl_array_append(log_routes, cfl_variant_create_from_kvlist(route)) == 0); + + TEST_CHECK(cfl_kvlist_insert_array(routes, "logs", log_routes) == 0); + TEST_CHECK(cfl_kvlist_insert_kvlist(input, "routes", routes) == 0); + TEST_CHECK(cfl_array_append(inputs, cfl_variant_create_from_kvlist(input)) == 0); + + inputs_variant = cfl_variant_create_from_array(inputs); + TEST_CHECK(inputs_variant != NULL); + + return inputs_variant; +} + +static struct flb_cf *cf_from_inputs_variant(struct cfl_variant *inputs) +{ + struct flb_cf *cf; + struct cfl_array *array; + size_t idx; + + if (!inputs || inputs->type != CFL_VARIANT_ARRAY) { + return NULL; + } + + cf = flb_cf_create(); + if (!cf) { + return NULL; + } + + array = inputs->data.as_array; + for (idx = 0; idx < cfl_array_size(array); idx++) { + struct cfl_variant *entry; + struct cfl_kvlist *copy; + struct flb_cf_section *section; + + entry = cfl_array_fetch_by_index(array, idx); + if (!entry || entry->type != CFL_VARIANT_KVLIST) { + flb_cf_destroy(cf); + return NULL; + } + + copy = clone_kvlist(entry->data.as_kvlist); + if (!copy) { + flb_cf_destroy(cf); + return NULL; + } + + section = flb_cf_section_create(cf, "input", 5); + if (!section) { + cfl_kvlist_destroy(copy); + flb_cf_destroy(cf); + return NULL; + } + + cfl_kvlist_destroy(section->properties); + section->properties = copy; + } + + return cf; +} + +static struct cfl_kvlist *clone_kvlist(struct cfl_kvlist *kvlist) +{ + struct cfl_kvlist *copy; + struct cfl_list *head; + struct cfl_kvpair *pair; + struct cfl_variant *value_copy; + + if (!kvlist) { + return NULL; + } + + copy = cfl_kvlist_create(); + if (!copy) { + return NULL; + } + + cfl_list_foreach(head, &kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + value_copy = clone_variant(pair->val); + if (!value_copy) { + cfl_kvlist_destroy(copy); + return NULL; + } + + if (cfl_kvlist_insert_s(copy, + pair->key, + cfl_sds_len(pair->key), + value_copy) != 0) { + cfl_variant_destroy(value_copy); + cfl_kvlist_destroy(copy); + return NULL; + } + } + + return copy; +} + +static struct cfl_variant *clone_variant(struct cfl_variant *var) +{ + struct cfl_array *array_copy; + struct cfl_kvlist *kvlist_copy; + int referenced; + + array_copy = NULL; + kvlist_copy = NULL; + referenced = CFL_FALSE; + + if (!var) { + return NULL; + } + + switch (var->type) { + case CFL_VARIANT_STRING: + referenced = (var->referenced == CFL_TRUE) ? CFL_TRUE : CFL_FALSE; + return cfl_variant_create_from_string_s(var->data.as_string, + cfl_sds_len(var->data.as_string), + referenced); + case CFL_VARIANT_BOOL: + return cfl_variant_create_from_bool(var->data.as_bool); + case CFL_VARIANT_ARRAY: + array_copy = clone_array(var->data.as_array); + if (!array_copy) { + return NULL; + } + return cfl_variant_create_from_array(array_copy); + case CFL_VARIANT_KVLIST: + kvlist_copy = clone_kvlist(var->data.as_kvlist); + if (!kvlist_copy) { + return NULL; + } + return cfl_variant_create_from_kvlist(kvlist_copy); + default: + break; + } + + return NULL; +} + +static struct cfl_array *clone_array(struct cfl_array *array) +{ + struct cfl_array *copy; + struct cfl_variant *entry; + struct cfl_variant *entry_copy; + size_t idx; + + if (!array) { + return NULL; + } + + copy = cfl_array_create(cfl_array_size(array)); + if (!copy) { + return NULL; + } + + for (idx = 0; idx < cfl_array_size(array); idx++) { + entry = cfl_array_fetch_by_index(array, idx); + entry_copy = clone_variant(entry); + if (!entry_copy) { + cfl_array_destroy(copy); + return NULL; + } + + if (cfl_array_append(copy, entry_copy) != 0) { + cfl_variant_destroy(entry_copy); + cfl_array_destroy(copy); + return NULL; + } + } + + return copy; +} + +static void setup_conditional_routing_instances(struct flb_config *config, + struct flb_input_instance *input, + struct flb_input_plugin *input_plugin, + struct flb_output_instance *output1, + struct flb_output_instance *output2, + struct flb_output_instance *output3, + struct flb_output_plugin *output_plugin) +{ + memset(config, 0, sizeof(struct flb_config)); + mk_list_init(&config->inputs); + mk_list_init(&config->outputs); + cfl_list_init(&config->input_routes); + + memset(input, 0, sizeof(struct flb_input_instance)); + mk_list_init(&input->_head); + cfl_list_init(&input->routes_direct); + cfl_list_init(&input->routes); + mk_list_init(&input->tasks); + mk_list_init(&input->chunks); + mk_list_init(&input->collectors); + snprintf(input->name, sizeof(input->name), "tail.0"); + input->alias = flb_sds_create("test_input"); + input_plugin->name = "tail"; + input->p = input_plugin; + mk_list_add(&input->_head, &config->inputs); + + memset(output1, 0, sizeof(struct flb_output_instance)); + mk_list_init(&output1->_head); + mk_list_init(&output1->properties); + mk_list_init(&output1->net_properties); + snprintf(output1->name, sizeof(output1->name), "stdout.0"); + output1->alias = flb_sds_create("info_destination"); + output1->event_type = FLB_OUTPUT_LOGS; + output1->id = 1; + output_plugin->name = "stdout"; + output1->p = output_plugin; + mk_list_add(&output1->_head, &config->outputs); + + memset(output2, 0, sizeof(struct flb_output_instance)); + mk_list_init(&output2->_head); + mk_list_init(&output2->properties); + mk_list_init(&output2->net_properties); + snprintf(output2->name, sizeof(output2->name), "stdout.1"); + output2->alias = flb_sds_create("error_destination"); + output2->event_type = FLB_OUTPUT_LOGS; + output2->id = 2; + output2->p = output_plugin; + mk_list_add(&output2->_head, &config->outputs); + + memset(output3, 0, sizeof(struct flb_output_instance)); + mk_list_init(&output3->_head); + mk_list_init(&output3->properties); + mk_list_init(&output3->net_properties); + snprintf(output3->name, sizeof(output3->name), "stdout.2"); + output3->alias = flb_sds_create("default_destination"); + output3->event_type = FLB_OUTPUT_LOGS; + output3->id = 3; + output3->p = output_plugin; + mk_list_add(&output3->_head, &config->outputs); +} + +static void setup_conditional_routes(struct flb_input_routes *input_routes, + struct flb_route *route1, + struct flb_route *route2, + struct flb_route *route3, + struct flb_route_output *route_output1, + struct flb_route_output *route_output2, + struct flb_route_output *route_output3, + struct flb_output_instance *output1, + struct flb_output_instance *output2, + struct flb_output_instance *output3) +{ + memset(input_routes, 0, sizeof(struct flb_input_routes)); + cfl_list_init(&input_routes->_head); + cfl_list_init(&input_routes->routes); + input_routes->input_name = flb_sds_create("tail"); + + /* Route 1: info_logs */ + memset(route1, 0, sizeof(struct flb_route)); + cfl_list_init(&route1->_head); + cfl_list_init(&route1->outputs); + route1->name = flb_sds_create("info_logs"); + route1->signals = FLB_ROUTER_SIGNAL_LOGS; + route1->per_record_routing = FLB_TRUE; + cfl_list_add(&route1->_head, &input_routes->routes); + + memset(route_output1, 0, sizeof(struct flb_route_output)); + cfl_list_init(&route_output1->_head); + route_output1->name = flb_sds_create("info_destination"); + cfl_list_add(&route_output1->_head, &route1->outputs); + + /* Route 2: error_logs */ + memset(route2, 0, sizeof(struct flb_route)); + cfl_list_init(&route2->_head); + cfl_list_init(&route2->outputs); + route2->name = flb_sds_create("error_logs"); + route2->signals = FLB_ROUTER_SIGNAL_LOGS; + route2->per_record_routing = FLB_TRUE; + cfl_list_add(&route2->_head, &input_routes->routes); + + memset(route_output2, 0, sizeof(struct flb_route_output)); + cfl_list_init(&route_output2->_head); + route_output2->name = flb_sds_create("error_destination"); + cfl_list_add(&route_output2->_head, &route2->outputs); + + /* Route 3: default_logs */ + memset(route3, 0, sizeof(struct flb_route)); + cfl_list_init(&route3->_head); + cfl_list_init(&route3->outputs); + route3->name = flb_sds_create("default_logs"); + route3->signals = FLB_ROUTER_SIGNAL_LOGS; + route3->per_record_routing = FLB_TRUE; + cfl_list_add(&route3->_head, &input_routes->routes); + + memset(route_output3, 0, sizeof(struct flb_route_output)); + cfl_list_init(&route_output3->_head); + route_output3->name = flb_sds_create("default_destination"); + cfl_list_add(&route_output3->_head, &route3->outputs); +} + +static int test_record_routing(struct flb_input_instance *input, + struct flb_event_chunk *chunk, + const char *expected_route) +{ + struct mk_list *head; + struct flb_router_path *path; + struct flb_router_chunk_context context; + int found = 0; + + flb_router_chunk_context_init(&context); + + cfl_list_foreach(head, &input->routes_direct) { + path = cfl_list_entry(head, struct flb_router_path, _head); + + if (path->route && strcmp(path->route->name, expected_route) == 0) { + if (flb_router_path_should_route(chunk, &context, path) == FLB_TRUE) { + found = 1; + break; + } + } + } + + flb_router_chunk_context_destroy(&context); + return found; +} + +static void cleanup_conditional_routing_instances(struct flb_input_instance *input, + struct flb_output_instance *output1, + struct flb_output_instance *output2, + struct flb_output_instance *output3, + struct flb_input_routes *input_routes, + struct flb_route *route1, + struct flb_route *route2, + struct flb_route *route3, + struct flb_route_output *route_output1, + struct flb_route_output *route_output2, + struct flb_route_output *route_output3) +{ + flb_sds_destroy(input->alias); + flb_sds_destroy(output1->alias); + flb_sds_destroy(output2->alias); + flb_sds_destroy(output3->alias); + flb_sds_destroy(input_routes->input_name); + flb_sds_destroy(route1->name); + flb_sds_destroy(route2->name); + flb_sds_destroy(route3->name); + flb_sds_destroy(route_output1->name); + flb_sds_destroy(route_output2->name); + flb_sds_destroy(route_output3->name); +} + +TEST_LIST = { + { "config_parse", test_conditional_routing_config_parse }, + { "condition_eval", test_conditional_routing_condition_eval }, + { "per_record", test_conditional_routing_per_record }, + { "default_route", test_conditional_routing_default_route }, + { "route_mask", test_conditional_routing_route_mask }, + { "no_duplicates", test_conditional_routing_no_duplicates }, + { 0 } +}; diff --git a/tests/internal/conditionals.c b/tests/internal/conditionals.c index 7ce49021fdc..c567ee7f9dd 100644 --- a/tests/internal/conditionals.c +++ b/tests/internal/conditionals.c @@ -364,7 +364,7 @@ void test_condition_equals() cond = flb_condition_create(FLB_COND_OP_AND); TEST_CHECK(cond != NULL); - TEST_CHECK(flb_condition_add_rule(cond, "$level", FLB_RULE_OP_EQ, + TEST_CHECK(flb_condition_add_rule(cond, "$level", FLB_RULE_OP_EQ, "error", 0, RECORD_CONTEXT_BODY) == FLB_TRUE); result = flb_condition_evaluate(cond, &record_data->chunk); @@ -445,7 +445,7 @@ void test_condition_not_equals() cond = flb_condition_create(FLB_COND_OP_AND); TEST_CHECK(cond != NULL); - TEST_CHECK(flb_condition_add_rule(cond, "$level", FLB_RULE_OP_NEQ, + TEST_CHECK(flb_condition_add_rule(cond, "$level", FLB_RULE_OP_NEQ, "error", 0, RECORD_CONTEXT_BODY) == FLB_TRUE); result = flb_condition_evaluate(cond, &record_data->chunk); @@ -734,7 +734,7 @@ void test_condition_invalid_expressions() result = flb_condition_evaluate(NULL, &record_data->chunk); TEST_CHECK(result == FLB_TRUE); - /* Test NULL record */ + /* Test NULL record with AND condition */ cond = flb_condition_create(FLB_COND_OP_AND); TEST_CHECK(cond != NULL); @@ -742,7 +742,30 @@ void test_condition_invalid_expressions() "error", 0, RECORD_CONTEXT_BODY) == FLB_TRUE); result = flb_condition_evaluate(cond, NULL); - TEST_CHECK(result == FLB_TRUE); + TEST_CHECK(result == FLB_FALSE); /* NULL record should fail condition */ + + flb_condition_destroy(cond); + + /* Test NULL record with OR condition */ + cond = flb_condition_create(FLB_COND_OP_OR); + TEST_CHECK(cond != NULL); + + TEST_CHECK(flb_condition_add_rule(cond, "$level", FLB_RULE_OP_EQ, + "error", 0, RECORD_CONTEXT_BODY) == FLB_TRUE); + TEST_CHECK(flb_condition_add_rule(cond, "$level", FLB_RULE_OP_EQ, + "warn", 0, RECORD_CONTEXT_BODY) == FLB_TRUE); + + result = flb_condition_evaluate(cond, NULL); + TEST_CHECK(result == FLB_FALSE); /* NULL record should fail condition */ + + flb_condition_destroy(cond); + + /* Test NULL record with empty condition */ + cond = flb_condition_create(FLB_COND_OP_AND); + TEST_CHECK(cond != NULL); + + result = flb_condition_evaluate(cond, NULL); + TEST_CHECK(result == FLB_FALSE); /* NULL record should fail even with empty condition */ flb_condition_destroy(cond); diff --git a/tests/internal/data/config_format/yaml/routing/context.yaml b/tests/internal/data/config_format/yaml/routing/context.yaml new file mode 100644 index 00000000000..73709a25b8c --- /dev/null +++ b/tests/internal/data/config_format/yaml/routing/context.yaml @@ -0,0 +1,27 @@ +pipeline: + inputs: + - name: dummy + routes: + logs: + - name: ctx_route + condition: + rules: + - context: metadata + field: "$['meta_key']" + op: eq + value: "meta" + - context: group_attributes + field: "$['group_attr']" + op: eq + value: "attr" + - context: otel_resource_attributes + field: "$['service.name']" + op: eq + value: "backend" + to: + outputs: + - name: ctx_out + outputs: + - name: null + alias: ctx_out + match: "*" diff --git a/tests/internal/log_event_decoder.c b/tests/internal/log_event_decoder.c index 8661e4fb1b3..531fce41d86 100644 --- a/tests/internal/log_event_decoder.c +++ b/tests/internal/log_event_decoder.c @@ -22,8 +22,10 @@ #include #include #include +#include #include #include +#include #include "flb_tests_internal.h" @@ -270,6 +272,1154 @@ void decoder_next() msgpack_sbuffer_destroy(&sbuf); } +static void pack_group_marker(msgpack_packer *pck, int32_t marker_type) +{ + struct flb_time tm; + + /* Set negative timestamp to indicate group marker */ + flb_time_set(&tm, marker_type, 0); + + msgpack_pack_array(pck, 2); /* Root array: [header, body] */ + msgpack_pack_array(pck, 2); /* Header array: [timestamp, metadata] */ + pack_event_time(pck, &tm); /* Group marker timestamp */ + msgpack_pack_map(pck, 1); /* Metadata: group info */ + msgpack_pack_str(pck, 5); + msgpack_pack_str_body(pck, "group", 5); + msgpack_pack_str(pck, 6); + msgpack_pack_str_body(pck, "marker", 6); + msgpack_pack_map(pck, 1); /* Body: group attributes */ + msgpack_pack_str(pck, 3); + msgpack_pack_str_body(pck, "tag", 3); + msgpack_pack_str(pck, 4); + msgpack_pack_str_body(pck, "test", 4); +} + +void decoder_skip_groups() +{ + struct flb_log_event_decoder dec; + struct flb_log_event event; + int ret; + struct flb_time tm1, tm2, tm3; + msgpack_sbuffer sbuf; + msgpack_packer pck; + char *json = NULL; + int record_count = 0; + int32_t decoded_record_type; + + /* Create timestamps for normal log records */ + flb_time_set(&tm1, 1000, 100); + flb_time_set(&tm2, 2000, 200); + flb_time_set(&tm3, 3000, 300); + + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + /* Pack: GROUP_START, normal log1, normal log2, GROUP_END, normal log3 */ + + /* GROUP_START marker */ + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + + /* Normal log 1 */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + /* Normal log 2 */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm2); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "2", 1); + + /* GROUP_END marker */ + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + + /* Normal log 3 */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm3); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "3", 1); + + /* Initialize decoder with read_groups = false */ + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + if (!TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS)) { + TEST_MSG("flb_log_event_decoder_init failed. ret=%s", + flb_log_event_decoder_get_error_description(ret)); + msgpack_sbuffer_destroy(&sbuf); + return; + } + + ret = flb_log_event_decoder_read_groups(&dec, FLB_FALSE); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("flb_log_event_decoder_read_groups failed"); + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + return; + } + + /* Decode records and verify group markers are skipped */ + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + /* Verify we never get a zeroed event (both sec and nsec should not be 0) */ + if (!TEST_CHECK(!(event.timestamp.tm.tv_sec == 0 && event.timestamp.tm.tv_nsec == 0))) { + TEST_MSG("Received zeroed event - group marker was not skipped properly"); + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + return; + } + + /* Get record type */ + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("flb_log_event_decoder_get_record_type failed"); + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + return; + } + + /* Verify we never receive group markers when read_groups is false */ + if (!TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL)) { + TEST_MSG("Received group marker (type=%d) when read_groups=false", + decoded_record_type); + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + return; + } + + record_count++; + + /* Verify expected timestamps are returned in order */ + if (record_count == 1) { + if (!TEST_CHECK(flb_time_equal(&tm1, &event.timestamp))) { + TEST_MSG("First record timestamp mismatch"); + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + return; + } + } + else if (record_count == 2) { + if (!TEST_CHECK(flb_time_equal(&tm2, &event.timestamp))) { + TEST_MSG("Second record timestamp mismatch"); + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + return; + } + } + else if (record_count == 3) { + if (!TEST_CHECK(flb_time_equal(&tm3, &event.timestamp))) { + TEST_MSG("Third record timestamp mismatch"); + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + return; + } + } + + /* Verify body is valid */ + json = flb_msgpack_to_json_str(4096, event.body, FLB_TRUE); + if (TEST_CHECK(json != NULL)) { + char expected_log[16]; + snprintf(expected_log, sizeof(expected_log), "\"log\":\"%d\"", record_count); + if (!TEST_CHECK(strstr(json, expected_log) != NULL)) { + TEST_MSG("Expected %s in body, got json=%s", expected_log, json); + } + flb_free(json); + json = NULL; + } + } + + /* Verify we got exactly 3 normal records (group markers should be skipped) */ + if (!TEST_CHECK(record_count == 3)) { + TEST_MSG("Expected 3 normal records, got %d. Group markers were not skipped properly.", + record_count); + } + + /* Verify we reached end of data, not an error */ + if (!TEST_CHECK(ret == FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA || + ret == FLB_EVENT_DECODER_SUCCESS)) { + TEST_MSG("Unexpected decoder result: %s", + flb_log_event_decoder_get_error_description(ret)); + } + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); +} + +void decoder_skip_groups_corrupted() +{ + struct flb_log_event_decoder dec; + struct flb_log_event event; + int ret; + struct flb_time tm1, tm2; + msgpack_sbuffer sbuf; + msgpack_packer pck; + int record_count = 0; + int32_t decoded_record_type; + + flb_time_set(&tm1, 1000, 100); + flb_time_set(&tm2, 2000, 200); + + /* Test Case 1: Unmatched GROUP_START (no GROUP_END) */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + + /* Normal log */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + /* Another GROUP_START without END */ + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + + /* Another normal log */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm2); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "2", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_FALSE); + TEST_CHECK(ret == 0); + + record_count = 0; + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + record_count++; + } + + /* Should get 2 normal records, skipping unmatched GROUP_START markers */ + TEST_CHECK(record_count == 2); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + + /* Test Case 2: Unmatched GROUP_END (no GROUP_START) */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + + /* Normal log */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_FALSE); + TEST_CHECK(ret == 0); + + record_count = 0; + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + record_count++; + } + + /* Should get 1 normal record, skipping unmatched GROUP_END */ + TEST_CHECK(record_count == 1); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + + /* Test Case 3: Multiple consecutive GROUP_START */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + + /* Normal log */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_FALSE); + TEST_CHECK(ret == 0); + + record_count = 0; + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + record_count++; + } + + /* Should get 1 normal record, skipping all GROUP_START markers */ + TEST_CHECK(record_count == 1); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + + /* Test Case 4: Multiple consecutive GROUP_END */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + /* Normal log */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_FALSE); + TEST_CHECK(ret == 0); + + record_count = 0; + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + record_count++; + } + + /* Should get 1 normal record, skipping all GROUP_END markers */ + TEST_CHECK(record_count == 1); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + + /* Test Case 5: Mixed invalid states - GROUP_END, GROUP_START, GROUP_END, normal log */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + + /* Normal log */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_FALSE); + TEST_CHECK(ret == 0); + + record_count = 0; + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + /* Verify we never get a zeroed event */ + TEST_CHECK(!(event.timestamp.tm.tv_sec == 0 && event.timestamp.tm.tv_nsec == 0)); + + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + record_count++; + } + + /* Should get 1 normal record, skipping all invalid group markers */ + TEST_CHECK(record_count == 1); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + + /* Test Case 6: Only group markers, no normal logs */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_FALSE); + TEST_CHECK(ret == 0); + + record_count = 0; + ret = flb_log_event_decoder_next(&dec, &event); + + /* Should get INSUFFICIENT_DATA since all records are group markers */ + TEST_CHECK(ret == FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA); + TEST_CHECK(record_count == 0); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); +} + +void decoder_read_groups() +{ + struct flb_log_event_decoder dec; + struct flb_log_event event; + int ret; + struct flb_time tm1, tm2; + msgpack_sbuffer sbuf; + msgpack_packer pck; + int record_count = 0; + int32_t decoded_record_type; + int group_start_count = 0; + int group_end_count = 0; + int normal_count = 0; + + flb_time_set(&tm1, 1000, 100); + flb_time_set(&tm2, 2000, 200); + + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + /* Pack: GROUP_START, normal log1, normal log2, GROUP_END, normal log3 */ + + /* GROUP_START marker */ + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + + /* Normal log 1 */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + /* Normal log 2 */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm2); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "2", 1); + + /* GROUP_END marker */ + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + + /* Initialize decoder with read_groups = true */ + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_TRUE); + TEST_CHECK(ret == 0); + + /* Decode records and verify group markers ARE returned */ + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + record_count++; + + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + + if (decoded_record_type == FLB_LOG_EVENT_GROUP_START) { + group_start_count++; + /* Verify GROUP_START has negative timestamp */ + TEST_CHECK(event.timestamp.tm.tv_sec == FLB_LOG_EVENT_GROUP_START); + } + else if (decoded_record_type == FLB_LOG_EVENT_GROUP_END) { + group_end_count++; + /* Verify GROUP_END has negative timestamp */ + TEST_CHECK(event.timestamp.tm.tv_sec == FLB_LOG_EVENT_GROUP_END); + } + else if (decoded_record_type == FLB_LOG_EVENT_NORMAL) { + normal_count++; + /* Normal logs should have group metadata/attributes from active group */ + if (record_count > 1 && record_count < 4) { + /* Logs 1 and 2 should have group metadata from GROUP_START */ + TEST_CHECK(event.group_metadata != NULL || event.group_attributes != NULL); + } + } + } + + /* When read_groups=true, we should get: + * 1 GROUP_START + 2 normal logs + 1 GROUP_END = 4 records total + */ + TEST_CHECK(record_count == 4); + TEST_CHECK(group_start_count == 1); + TEST_CHECK(group_end_count == 1); + TEST_CHECK(normal_count == 2); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); +} + +void decoder_read_groups_corrupted() +{ + struct flb_log_event_decoder dec; + struct flb_log_event event; + int ret; + struct flb_time tm1; + msgpack_sbuffer sbuf; + msgpack_packer pck; + int record_count = 0; + int32_t decoded_record_type; + int group_start_count = 0; + int group_end_count = 0; + int normal_count = 0; + + flb_time_set(&tm1, 1000, 100); + + /* Test Case 1: Unmatched GROUP_START - should still return it */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + + /* Normal log */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_TRUE); + TEST_CHECK(ret == 0); + + record_count = 0; + group_start_count = 0; + normal_count = 0; + + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + record_count++; + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + + if (decoded_record_type == FLB_LOG_EVENT_GROUP_START) { + group_start_count++; + } + else if (decoded_record_type == FLB_LOG_EVENT_NORMAL) { + normal_count++; + } + } + + /* Should get 1 GROUP_START + 1 normal log */ + TEST_CHECK(record_count == 2); + TEST_CHECK(group_start_count == 1); + TEST_CHECK(normal_count == 1); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + + /* Test Case 2: Unmatched GROUP_END - should still return it */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + + /* Normal log */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_TRUE); + TEST_CHECK(ret == 0); + + record_count = 0; + group_end_count = 0; + normal_count = 0; + + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + record_count++; + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + + if (decoded_record_type == FLB_LOG_EVENT_GROUP_END) { + group_end_count++; + } + else if (decoded_record_type == FLB_LOG_EVENT_NORMAL) { + normal_count++; + } + } + + /* Should get 1 GROUP_END + 1 normal log */ + TEST_CHECK(record_count == 2); + TEST_CHECK(group_end_count == 1); + TEST_CHECK(normal_count == 1); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + + /* Test Case 3: Multiple consecutive GROUP_START - all should be returned */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + + /* Normal log */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_TRUE); + TEST_CHECK(ret == 0); + + record_count = 0; + group_start_count = 0; + normal_count = 0; + + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + record_count++; + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + + if (decoded_record_type == FLB_LOG_EVENT_GROUP_START) { + group_start_count++; + } + else if (decoded_record_type == FLB_LOG_EVENT_NORMAL) { + normal_count++; + } + } + + /* Should get 3 GROUP_START + 1 normal log */ + TEST_CHECK(record_count == 4); + TEST_CHECK(group_start_count == 3); + TEST_CHECK(normal_count == 1); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + + /* Test Case 4: Mixed invalid states - all markers should be returned */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + + /* Normal log */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_TRUE); + TEST_CHECK(ret == 0); + + record_count = 0; + group_start_count = 0; + group_end_count = 0; + normal_count = 0; + + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + record_count++; + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + + if (decoded_record_type == FLB_LOG_EVENT_GROUP_START) { + group_start_count++; + } + else if (decoded_record_type == FLB_LOG_EVENT_GROUP_END) { + group_end_count++; + } + else if (decoded_record_type == FLB_LOG_EVENT_NORMAL) { + normal_count++; + } + } + + /* Should get 2 GROUP_END + 1 GROUP_START + 1 normal log */ + TEST_CHECK(record_count == 4); + TEST_CHECK(group_start_count == 1); + TEST_CHECK(group_end_count == 2); + TEST_CHECK(normal_count == 1); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); +} + +void decoder_corrupted_group_timestamps() +{ + struct flb_log_event_decoder dec; + struct flb_log_event event; + int ret; + struct flb_time tm1; + struct flb_time corrupted_tm; + msgpack_sbuffer sbuf; + msgpack_packer pck; + msgpack_sbuffer sbuf2; + msgpack_packer pck2; + int32_t decoded_record_type; + + flb_time_set(&tm1, 1000, 100); + + /* Test Case 1: Invalid negative timestamp (not -1 or -2) - should skip */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + /* Create a record with corrupted group timestamp (-3) */ + flb_time_set(&corrupted_tm, -3, 0); /* Invalid group marker timestamp */ + + msgpack_pack_array(&pck, 2); /* Root array: [header, body] */ + msgpack_pack_array(&pck, 2); /* Header array: [timestamp, metadata] */ + pack_event_time(&pck, &corrupted_tm); /* Invalid group marker timestamp */ + msgpack_pack_map(&pck, 0); /* Empty metadata */ + msgpack_pack_map(&pck, 0); /* Empty body */ + + /* Normal log after corrupted marker */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_FALSE); + TEST_CHECK(ret == 0); + + /* When read_groups=false, corrupted group marker should be skipped */ + ret = flb_log_event_decoder_next(&dec, &event); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + TEST_CHECK(flb_time_equal(&tm1, &event.timestamp)); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + + /* Test Case 2: Invalid negative timestamp with read_groups=true - should also skip */ + msgpack_sbuffer_init(&sbuf2); + msgpack_packer_init(&pck2, &sbuf2, msgpack_sbuffer_write); + + flb_time_set(&corrupted_tm, -10, 0); /* Another invalid group marker timestamp */ + + msgpack_pack_array(&pck2, 2); + msgpack_pack_array(&pck2, 2); + pack_event_time(&pck2, &corrupted_tm); + msgpack_pack_map(&pck2, 0); + msgpack_pack_map(&pck2, 0); + + /* Normal log after corrupted marker */ + msgpack_pack_array(&pck2, 2); + msgpack_pack_array(&pck2, 2); + pack_event_time(&pck2, &tm1); + msgpack_pack_map(&pck2, 0); + msgpack_pack_map(&pck2, 1); + msgpack_pack_str(&pck2, 3); + msgpack_pack_str_body(&pck2, "log", 3); + msgpack_pack_str(&pck2, 1); + msgpack_pack_str_body(&pck2, "1", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf2.data, sbuf2.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_TRUE); + TEST_CHECK(ret == 0); + + /* When read_groups=true, corrupted group marker should also be skipped */ + ret = flb_log_event_decoder_next(&dec, &event); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + TEST_CHECK(flb_time_equal(&tm1, &event.timestamp)); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf2); + + /* Test Case 3: Very negative timestamp - should skip */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + flb_time_set(&corrupted_tm, -1000, 0); /* Very negative but invalid */ + + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &corrupted_tm); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 0); + + /* Normal log after corrupted marker */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_FALSE); + TEST_CHECK(ret == 0); + + /* Corrupted marker should be skipped, normal log should be returned */ + ret = flb_log_event_decoder_next(&dec, &event); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + TEST_CHECK(flb_time_equal(&tm1, &event.timestamp)); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); +} + +void decoder_invalid_marker_preserves_group_state() +{ + struct flb_log_event_decoder dec; + struct flb_log_event event; + int ret; + struct flb_time tm1; + struct flb_time tm2; + struct flb_time corrupted_tm; + msgpack_sbuffer sbuf; + msgpack_packer pck; + int32_t decoded_record_type; + int record_count = 0; + + flb_time_set(&tm1, 1000, 100); + flb_time_set(&tm2, 2000, 200); + + /* Test: GROUP_START → normal_log1 → [corrupted -3 marker] → normal_log2 + * Expected: normal_log2 should STILL have group metadata (state preserved) */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + /* GROUP_START with metadata */ + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + + /* Normal log 1 - should have group metadata */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + /* Corrupted marker (-3) - should NOT clear group state */ + flb_time_set(&corrupted_tm, -3, 0); + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &corrupted_tm); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 0); + + /* Normal log 2 - should STILL have group metadata (state preserved) */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm2); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "2", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_FALSE); + TEST_CHECK(ret == 0); + + /* Read normal log 1 - should have group metadata */ + ret = flb_log_event_decoder_next(&dec, &event); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + TEST_CHECK(flb_time_equal(&tm1, &event.timestamp)); + TEST_CHECK(event.group_metadata != NULL || event.group_attributes != NULL); + record_count++; + + /* Read normal log 2 - should STILL have group metadata (state preserved) */ + ret = flb_log_event_decoder_next(&dec, &event); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + TEST_CHECK(flb_time_equal(&tm2, &event.timestamp)); + /* CRITICAL: Group state should be preserved despite invalid marker */ + TEST_CHECK(event.group_metadata != NULL || event.group_attributes != NULL); + record_count++; + + TEST_CHECK(record_count == 2); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); +} + +void decoder_group_end_start_sequence() +{ + struct flb_log_event_decoder dec; + struct flb_log_event event; + int ret; + struct flb_time tm1; + msgpack_sbuffer sbuf; + msgpack_packer pck; + int record_count = 0; + int32_t decoded_record_type; + int group_start_count = 0; + int group_end_count = 0; + int normal_count = 0; + + flb_time_set(&tm1, 1000, 100); + + /* Test Case: GROUP_END (unmatched) → GROUP_START → normal log */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + /* GROUP_END without preceding GROUP_START */ + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + + /* GROUP_START */ + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + + /* Normal log */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + /* Test with read_groups = false */ + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_FALSE); + TEST_CHECK(ret == 0); + + record_count = 0; + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + record_count++; + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + + /* Verify we got the normal log */ + TEST_CHECK(flb_time_equal(&tm1, &event.timestamp)); + } + + /* Should get 1 normal log, skipping both GROUP_END and GROUP_START */ + TEST_CHECK(record_count == 1); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + + /* Test with read_groups = true */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_TRUE); + TEST_CHECK(ret == 0); + + record_count = 0; + group_start_count = 0; + group_end_count = 0; + normal_count = 0; + + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + record_count++; + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + + if (decoded_record_type == FLB_LOG_EVENT_GROUP_START) { + group_start_count++; + } + else if (decoded_record_type == FLB_LOG_EVENT_GROUP_END) { + group_end_count++; + } + else if (decoded_record_type == FLB_LOG_EVENT_NORMAL) { + normal_count++; + /* The log should have group metadata from GROUP_START (not GROUP_END) */ + if (record_count == 3) { + /* After GROUP_END (clears state) and GROUP_START (sets state), log should have group data */ + TEST_CHECK(event.group_metadata != NULL || event.group_attributes != NULL); + } + } + } + + /* Should get: GROUP_END, GROUP_START, normal log */ + TEST_CHECK(record_count == 3); + TEST_CHECK(group_start_count == 1); + TEST_CHECK(group_end_count == 1); + TEST_CHECK(normal_count == 1); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + + /* Test Case 2: GROUP_START → GROUP_END → GROUP_START → log */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_TRUE); + TEST_CHECK(ret == 0); + + record_count = 0; + group_start_count = 0; + group_end_count = 0; + normal_count = 0; + + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + record_count++; + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + + if (decoded_record_type == FLB_LOG_EVENT_GROUP_START) { + group_start_count++; + } + else if (decoded_record_type == FLB_LOG_EVENT_GROUP_END) { + group_end_count++; + } + else if (decoded_record_type == FLB_LOG_EVENT_NORMAL) { + normal_count++; + /* Log should have metadata from the last GROUP_START */ + TEST_CHECK(event.group_metadata != NULL || event.group_attributes != NULL); + } + } + + /* Should get: GROUP_START, GROUP_END, GROUP_START, normal log */ + TEST_CHECK(record_count == 4); + TEST_CHECK(group_start_count == 2); + TEST_CHECK(group_end_count == 1); + TEST_CHECK(normal_count == 1); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); +} + TEST_LIST = { @@ -278,5 +1428,12 @@ TEST_LIST = { { "decode_timestamp", decode_timestamp }, { "decode_object", decode_object }, { "decoder_next", decoder_next }, + { "decoder_skip_groups", decoder_skip_groups }, + { "decoder_skip_groups_corrupted", decoder_skip_groups_corrupted }, + { "decoder_read_groups", decoder_read_groups }, + { "decoder_read_groups_corrupted", decoder_read_groups_corrupted }, + { "decoder_corrupted_group_timestamps", decoder_corrupted_group_timestamps }, + { "decoder_invalid_marker_preserves_group_state", decoder_invalid_marker_preserves_group_state }, + { "decoder_group_end_start_sequence", decoder_group_end_start_sequence }, { 0 } }; diff --git a/tests/internal/router_config.c b/tests/internal/router_config.c index 9f4c888b3d1..8b2067c89b0 100644 --- a/tests/internal/router_config.c +++ b/tests/internal/router_config.c @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include #include @@ -16,6 +18,7 @@ #include #include #include +#include #include "flb_tests_internal.h" @@ -85,6 +88,285 @@ static int build_log_chunk(const char *level, return 0; } +static int build_log_chunk_with_metadata(const char *metadata_key, + const char *metadata_value, + const char *body_key, + const char *body_value, + struct flb_log_event_encoder *encoder, + struct flb_event_chunk *chunk) +{ + int ret; + + if (!encoder || !chunk) { + return -1; + } + + ret = flb_log_event_encoder_init(encoder, FLB_LOG_EVENT_FORMAT_DEFAULT); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + return -1; + } + + ret = flb_log_event_encoder_begin_record(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_set_current_timestamp(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + if (metadata_key && metadata_value) { + ret = flb_log_event_encoder_append_metadata_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE(metadata_key, strlen(metadata_key)), + FLB_LOG_EVENT_CSTRING_VALUE(metadata_value)); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + } + + if (body_key && body_value) { + ret = flb_log_event_encoder_append_body_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE(body_key, strlen(body_key)), + FLB_LOG_EVENT_CSTRING_VALUE(body_value)); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + } + + ret = flb_log_event_encoder_commit_record(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + memset(chunk, 0, sizeof(*chunk)); + chunk->type = FLB_EVENT_TYPE_LOGS; + chunk->data = encoder->output_buffer; + chunk->size = encoder->output_length; + chunk->total_events = 1; + + return 0; +} + +static int build_log_group_chunk(const char *group_metadata_key, + const char *group_metadata_value, + const char *group_attribute_key, + const char *group_attribute_value, + const char *record_body_key, + const char *record_body_value, + struct flb_log_event_encoder *encoder, + struct flb_event_chunk *chunk) +{ + int ret; + + if (!encoder || !chunk) { + return -1; + } + + ret = flb_log_event_encoder_init(encoder, FLB_LOG_EVENT_FORMAT_DEFAULT); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + return -1; + } + + ret = flb_log_event_encoder_group_init(encoder); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_metadata_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE(group_metadata_key, strlen(group_metadata_key)), + FLB_LOG_EVENT_CSTRING_VALUE(group_metadata_value)); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_body_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE(group_attribute_key, strlen(group_attribute_key)), + FLB_LOG_EVENT_CSTRING_VALUE(group_attribute_value)); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_group_header_end(encoder); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_begin_record(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_set_current_timestamp(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_body_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE(record_body_key, strlen(record_body_key)), + FLB_LOG_EVENT_CSTRING_VALUE(record_body_value)); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_commit_record(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_group_end(encoder); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + memset(chunk, 0, sizeof(*chunk)); + chunk->type = FLB_EVENT_TYPE_LOGS; + chunk->data = encoder->output_buffer; + chunk->size = encoder->output_length; + chunk->total_events = 1; + + return 0; +} + +static int build_log_chunk_with_otel(const char *service_name, + const char *scope_name, + const char *scope_version, + const char *scope_attribute_key, + const char *scope_attribute_value, + struct flb_log_event_encoder *encoder, + struct flb_event_chunk *chunk) +{ + char *otlp_json = NULL; + int ret; + int error_status = 0; + size_t json_len; + const char *attr_key = scope_attribute_key; + + if (!encoder || !chunk) { + return -1; + } + + /* Extract attribute key from "scope.attr" format if needed */ + if (strncmp(scope_attribute_key, "scope.", 6) == 0) { + attr_key = scope_attribute_key + 6; /* Skip "scope." prefix */ + } + + /* Build OTLP JSON format with nested structure for service.name */ + /* Note: We create a nested service object to match $service['name'] accessor */ + json_len = snprintf(NULL, 0, + "{" + "\"resourceLogs\":[" + "{" + "\"resource\":{" + "\"attributes\":[" + "{\"key\":\"service\",\"value\":{\"kvlistValue\":{\"values\":[{\"key\":\"name\",\"value\":{\"stringValue\":\"%s\"}}]}}}" + "]" + "}," + "\"scopeLogs\":[" + "{" + "\"scope\":{" + "\"name\":\"%s\"," + "\"version\":\"%s\"," + "\"attributes\":[" + "{\"key\":\"scope\",\"value\":{\"kvlistValue\":{\"values\":[{\"key\":\"%s\",\"value\":{\"stringValue\":\"%s\"}}]}}}" + "]" + "}," + "\"logRecords\":[" + "{" + "\"timeUnixNano\":\"1728172800000000000\"," + "\"severityNumber\":9," + "\"severityText\":\"INFO\"," + "\"body\":{\"stringValue\":\"test log\"}" + "}" + "]" + "}" + "]" + "}" + "]" + "}", + service_name, scope_name, scope_version, attr_key, scope_attribute_value); + + otlp_json = flb_malloc(json_len + 1); + if (!otlp_json) { + return -1; + } + + snprintf(otlp_json, json_len + 1, + "{" + "\"resourceLogs\":[" + "{" + "\"resource\":{" + "\"attributes\":[" + "{\"key\":\"service\",\"value\":{\"kvlistValue\":{\"values\":[{\"key\":\"name\",\"value\":{\"stringValue\":\"%s\"}}]}}}" + "]" + "}," + "\"scopeLogs\":[" + "{" + "\"scope\":{" + "\"name\":\"%s\"," + "\"version\":\"%s\"," + "\"attributes\":[" + "{\"key\":\"scope\",\"value\":{\"kvlistValue\":{\"values\":[{\"key\":\"%s\",\"value\":{\"stringValue\":\"%s\"}}]}}}" + "]" + "}," + "\"logRecords\":[" + "{" + "\"timeUnixNano\":\"1728172800000000000\"," + "\"severityNumber\":9," + "\"severityText\":\"INFO\"," + "\"body\":{\"stringValue\":\"test log\"}" + "}" + "]" + "}" + "]" + "}" + "]" + "}", + service_name, scope_name, scope_version, attr_key, scope_attribute_value); + + /* Initialize encoder buffer (needed for msgpack_sbuffer_write) */ + memset(encoder, 0, sizeof(*encoder)); + msgpack_sbuffer_init(&encoder->buffer); + + /* Convert OTLP JSON to msgpack using the actual converter */ + ret = flb_opentelemetry_logs_json_to_msgpack(encoder, otlp_json, json_len, NULL, &error_status); + flb_free(otlp_json); + + if (ret != 0) { + msgpack_sbuffer_destroy(&encoder->buffer); + return -1; + } + + /* Set up the chunk from the encoder output */ + memset(chunk, 0, sizeof(*chunk)); + chunk->type = FLB_EVENT_TYPE_LOGS; + chunk->data = encoder->output_buffer; + chunk->size = encoder->output_length; + /* Count actual events in the buffer */ + chunk->total_events = flb_mp_count(encoder->output_buffer, encoder->output_length); + + return 0; +} + static void free_route_condition(struct flb_route_condition *condition) { struct cfl_list *tmp; @@ -128,6 +410,7 @@ static void free_route_condition(struct flb_route_condition *condition) flb_free(condition); } + static struct cfl_array *clone_array(struct cfl_array *array) { struct cfl_array *copy; @@ -757,6 +1040,58 @@ void test_router_config_parse_file_metrics() flb_cf_destroy(cf); } +void test_router_config_parse_file_contexts() +{ + struct cfl_list routes; + struct flb_cf *cf; + struct flb_input_routes *input_routes; + struct flb_route *route; + struct flb_route_condition_rule *rule; + struct cfl_list *head; + enum record_context_type expected[3] = { + RECORD_CONTEXT_METADATA, + RECORD_CONTEXT_GROUP_ATTRIBUTES, + RECORD_CONTEXT_OTEL_RESOURCE_ATTRIBUTES + }; + size_t idx; + int ret; + + cf = load_cf_from_yaml(FLB_ROUTER_TEST_FILE("context.yaml")); + TEST_CHECK(cf != NULL); + if (!cf) { + return; + } + + cfl_list_init(&routes); + + ret = flb_router_config_parse(cf, &routes, NULL); + TEST_CHECK(ret == 0); + if (ret != 0) { + flb_cf_destroy(cf); + return; + } + + input_routes = cfl_list_entry(routes.next, struct flb_input_routes, _head); + TEST_CHECK(strcmp(input_routes->input_name, "dummy") == 0); + + route = cfl_list_entry(input_routes->routes.next, struct flb_route, _head); + TEST_CHECK(route->condition != NULL); + + idx = 0; + cfl_list_foreach(head, &route->condition->rules) { + rule = cfl_list_entry(head, struct flb_route_condition_rule, _head); + TEST_CHECK(idx < sizeof(expected) / sizeof(expected[0])); + if (idx < sizeof(expected) / sizeof(expected[0])) { + TEST_CHECK(rule->context == expected[idx]); + } + idx++; + } + TEST_CHECK(idx == sizeof(expected) / sizeof(expected[0])); + + flb_router_routes_destroy(&routes); + flb_cf_destroy(cf); +} + static void setup_test_instances(struct flb_config *config, struct flb_input_instance *input, struct flb_input_plugin *input_plugin, @@ -951,6 +1286,490 @@ void test_router_route_default_precedence() flb_cf_destroy(cf); } +static void test_router_condition_eval_logs_metadata_context() +{ + struct flb_route route; + struct flb_route_condition *condition; + struct flb_route_condition_rule *rule; + struct flb_log_event_encoder encoder; + struct flb_event_chunk chunk; + struct flb_router_chunk_context context; + int ret; + + memset(&route, 0, sizeof(route)); + cfl_list_init(&route.outputs); + cfl_list_init(&route.processors); + + condition = flb_calloc(1, sizeof(struct flb_route_condition)); + TEST_CHECK(condition != NULL); + if (!condition) { + return; + } + + cfl_list_init(&condition->rules); + condition->op = FLB_COND_OP_AND; + condition->compiled_status = 0; + condition->compiled = NULL; + condition->is_default = FLB_FALSE; + + rule = flb_calloc(1, sizeof(struct flb_route_condition_rule)); + TEST_CHECK(rule != NULL); + if (!rule) { + free_route_condition(condition); + return; + } + + cfl_list_init(&rule->_head); + rule->field = flb_sds_create("$source"); + rule->op = flb_sds_create("eq"); + rule->value = flb_sds_create("app"); + rule->context = RECORD_CONTEXT_METADATA; + TEST_CHECK(rule->field != NULL && rule->op != NULL && rule->value != NULL); + if (!rule->field || !rule->op || !rule->value) { + if (rule->field) { + flb_sds_destroy(rule->field); + } + if (rule->op) { + flb_sds_destroy(rule->op); + } + if (rule->value) { + flb_sds_destroy(rule->value); + } + flb_free(rule); + free_route_condition(condition); + return; + } + + cfl_list_add(&rule->_head, &condition->rules); + + route.condition = condition; + route.signals = FLB_ROUTER_SIGNAL_LOGS; + + flb_router_chunk_context_init(&context); + + ret = build_log_chunk_with_metadata("source", "app", "level", "info", + &encoder, &chunk); + TEST_CHECK(ret == 0); + if (ret == 0) { + TEST_CHECK(flb_condition_eval_logs(&chunk, &context, &route) == FLB_TRUE); + } + flb_router_chunk_context_reset(&context); + flb_log_event_encoder_destroy(&encoder); + + ret = build_log_chunk_with_metadata("source", "other", "level", "info", + &encoder, &chunk); + TEST_CHECK(ret == 0); + if (ret == 0) { + TEST_CHECK(flb_condition_eval_logs(&chunk, &context, &route) == FLB_FALSE); + } + flb_router_chunk_context_reset(&context); + flb_log_event_encoder_destroy(&encoder); + + flb_router_chunk_context_destroy(&context); + free_route_condition(condition); +} + +static void test_router_condition_eval_logs_group_context() +{ + struct flb_route route; + struct flb_route_condition *condition; + struct flb_route_condition_rule *rule; + struct flb_log_event_encoder encoder; + struct flb_event_chunk chunk; + struct flb_router_chunk_context context; + int ret; + + memset(&route, 0, sizeof(route)); + cfl_list_init(&route.outputs); + cfl_list_init(&route.processors); + + condition = flb_calloc(1, sizeof(struct flb_route_condition)); + TEST_CHECK(condition != NULL); + if (!condition) { + return; + } + + cfl_list_init(&condition->rules); + condition->op = FLB_COND_OP_AND; + condition->compiled_status = 0; + condition->compiled = NULL; + condition->is_default = FLB_FALSE; + + rule = flb_calloc(1, sizeof(struct flb_route_condition_rule)); + TEST_CHECK(rule != NULL); + if (!rule) { + free_route_condition(condition); + return; + } + + cfl_list_init(&rule->_head); + rule->field = flb_sds_create("$tenant"); + rule->op = flb_sds_create("eq"); + rule->value = flb_sds_create("acme"); + rule->context = RECORD_CONTEXT_GROUP_METADATA; + TEST_CHECK(rule->field != NULL && rule->op != NULL && rule->value != NULL); + if (!rule->field || !rule->op || !rule->value) { + if (rule->field) { + flb_sds_destroy(rule->field); + } + if (rule->op) { + flb_sds_destroy(rule->op); + } + if (rule->value) { + flb_sds_destroy(rule->value); + } + flb_free(rule); + free_route_condition(condition); + return; + } + + cfl_list_add(&rule->_head, &condition->rules); + + route.condition = condition; + route.signals = FLB_ROUTER_SIGNAL_LOGS; + + flb_router_chunk_context_init(&context); + + ret = build_log_group_chunk("tenant", "acme", "service", "frontend", + "message", "hello", &encoder, &chunk); + TEST_CHECK(ret == 0); + if (ret == 0) { + TEST_CHECK(flb_condition_eval_logs(&chunk, &context, &route) == FLB_TRUE); + } + flb_router_chunk_context_reset(&context); + flb_log_event_encoder_destroy(&encoder); + + ret = build_log_group_chunk("tenant", "other", "service", "frontend", + "message", "hello", &encoder, &chunk); + TEST_CHECK(ret == 0); + if (ret == 0) { + TEST_CHECK(flb_condition_eval_logs(&chunk, &context, &route) == FLB_FALSE); + } + flb_router_chunk_context_reset(&context); + flb_log_event_encoder_destroy(&encoder); + + flb_router_chunk_context_destroy(&context); + free_route_condition(condition); + + /* Group attributes context */ + memset(&route, 0, sizeof(route)); + cfl_list_init(&route.outputs); + cfl_list_init(&route.processors); + + condition = flb_calloc(1, sizeof(struct flb_route_condition)); + TEST_CHECK(condition != NULL); + if (!condition) { + return; + } + + cfl_list_init(&condition->rules); + condition->op = FLB_COND_OP_AND; + condition->compiled_status = 0; + condition->compiled = NULL; + condition->is_default = FLB_FALSE; + + rule = flb_calloc(1, sizeof(struct flb_route_condition_rule)); + TEST_CHECK(rule != NULL); + if (!rule) { + free_route_condition(condition); + return; + } + + cfl_list_init(&rule->_head); + rule->field = flb_sds_create("$service"); + rule->op = flb_sds_create("eq"); + rule->value = flb_sds_create("frontend"); + rule->context = RECORD_CONTEXT_GROUP_ATTRIBUTES; + TEST_CHECK(rule->field != NULL && rule->op != NULL && rule->value != NULL); + if (!rule->field || !rule->op || !rule->value) { + if (rule->field) { + flb_sds_destroy(rule->field); + } + if (rule->op) { + flb_sds_destroy(rule->op); + } + if (rule->value) { + flb_sds_destroy(rule->value); + } + flb_free(rule); + free_route_condition(condition); + return; + } + + cfl_list_add(&rule->_head, &condition->rules); + + route.condition = condition; + route.signals = FLB_ROUTER_SIGNAL_LOGS; + + flb_router_chunk_context_init(&context); + + ret = build_log_group_chunk("tenant", "acme", "service", "frontend", + "message", "hello", &encoder, &chunk); + TEST_CHECK(ret == 0); + if (ret == 0) { + TEST_CHECK(flb_condition_eval_logs(&chunk, &context, &route) == FLB_TRUE); + } + flb_router_chunk_context_reset(&context); + flb_log_event_encoder_destroy(&encoder); + + ret = build_log_group_chunk("tenant", "acme", "service", "backend", + "message", "hello", &encoder, &chunk); + TEST_CHECK(ret == 0); + if (ret == 0) { + TEST_CHECK(flb_condition_eval_logs(&chunk, &context, &route) == FLB_FALSE); + } + flb_router_chunk_context_reset(&context); + flb_log_event_encoder_destroy(&encoder); + + flb_router_chunk_context_destroy(&context); + free_route_condition(condition); +} + +static void test_router_condition_eval_logs_otel_contexts() +{ + struct flb_route route; + struct flb_route_condition *condition; + struct flb_route_condition_rule *rule; + struct flb_log_event_encoder encoder; + struct flb_event_chunk chunk; + struct flb_router_chunk_context context; + int ret; + + flb_router_chunk_context_init(&context); + + /* Resource attributes context */ + memset(&route, 0, sizeof(route)); + cfl_list_init(&route.outputs); + cfl_list_init(&route.processors); + + condition = flb_calloc(1, sizeof(struct flb_route_condition)); + TEST_CHECK(condition != NULL); + if (!condition) { + flb_router_chunk_context_destroy(&context); + return; + } + + cfl_list_init(&condition->rules); + condition->op = FLB_COND_OP_AND; + condition->compiled_status = 0; + condition->compiled = NULL; + condition->is_default = FLB_FALSE; + + rule = flb_calloc(1, sizeof(struct flb_route_condition_rule)); + TEST_CHECK(rule != NULL); + if (!rule) { + free_route_condition(condition); + flb_router_chunk_context_destroy(&context); + return; + } + + cfl_list_init(&rule->_head); + rule->field = flb_sds_create("$service['name']"); + rule->op = flb_sds_create("eq"); + rule->value = flb_sds_create("backend"); + rule->context = RECORD_CONTEXT_OTEL_RESOURCE_ATTRIBUTES; + TEST_CHECK(rule->field != NULL && rule->op != NULL && rule->value != NULL); + if (!rule->field || !rule->op || !rule->value) { + if (rule->field) { + flb_sds_destroy(rule->field); + } + if (rule->op) { + flb_sds_destroy(rule->op); + } + if (rule->value) { + flb_sds_destroy(rule->value); + } + flb_free(rule); + free_route_condition(condition); + flb_router_chunk_context_destroy(&context); + return; + } + + cfl_list_add(&rule->_head, &condition->rules); + + route.condition = condition; + route.signals = FLB_ROUTER_SIGNAL_LOGS; + + ret = build_log_chunk_with_otel("backend", "demo", "1.0.0", + "scope.attr", "enabled", + &encoder, &chunk); + TEST_CHECK(ret == 0); + if (ret == 0) { + TEST_CHECK(flb_condition_eval_logs(&chunk, &context, &route) == FLB_TRUE); + } + flb_router_chunk_context_reset(&context); + msgpack_sbuffer_destroy(&encoder.buffer); + flb_log_event_encoder_destroy(&encoder); + + ret = build_log_chunk_with_otel("api", "demo", "1.0.0", + "scope.attr", "enabled", + &encoder, &chunk); + TEST_CHECK(ret == 0); + if (ret == 0) { + TEST_CHECK(flb_condition_eval_logs(&chunk, &context, &route) == FLB_FALSE); + } + flb_router_chunk_context_reset(&context); + msgpack_sbuffer_destroy(&encoder.buffer); + flb_log_event_encoder_destroy(&encoder); + + free_route_condition(condition); + + /* Scope metadata context */ + memset(&route, 0, sizeof(route)); + cfl_list_init(&route.outputs); + cfl_list_init(&route.processors); + + condition = flb_calloc(1, sizeof(struct flb_route_condition)); + TEST_CHECK(condition != NULL); + if (!condition) { + flb_router_chunk_context_destroy(&context); + return; + } + + cfl_list_init(&condition->rules); + condition->op = FLB_COND_OP_AND; + condition->compiled_status = 0; + condition->compiled = NULL; + condition->is_default = FLB_FALSE; + + rule = flb_calloc(1, sizeof(struct flb_route_condition_rule)); + TEST_CHECK(rule != NULL); + if (!rule) { + free_route_condition(condition); + flb_router_chunk_context_destroy(&context); + return; + } + + cfl_list_init(&rule->_head); + rule->field = flb_sds_create("$name"); + rule->op = flb_sds_create("eq"); + rule->value = flb_sds_create("demo"); + rule->context = RECORD_CONTEXT_OTEL_SCOPE_METADATA; + TEST_CHECK(rule->field != NULL && rule->op != NULL && rule->value != NULL); + if (!rule->field || !rule->op || !rule->value) { + if (rule->field) { + flb_sds_destroy(rule->field); + } + if (rule->op) { + flb_sds_destroy(rule->op); + } + if (rule->value) { + flb_sds_destroy(rule->value); + } + flb_free(rule); + free_route_condition(condition); + flb_router_chunk_context_destroy(&context); + return; + } + + cfl_list_add(&rule->_head, &condition->rules); + + route.condition = condition; + route.signals = FLB_ROUTER_SIGNAL_LOGS; + + ret = build_log_chunk_with_otel("backend", "demo", "1.0.0", + "scope.attr", "enabled", + &encoder, &chunk); + TEST_CHECK(ret == 0); + if (ret == 0) { + TEST_CHECK(flb_condition_eval_logs(&chunk, &context, &route) == FLB_TRUE); + } + flb_router_chunk_context_reset(&context); + msgpack_sbuffer_destroy(&encoder.buffer); + flb_log_event_encoder_destroy(&encoder); + + ret = build_log_chunk_with_otel("backend", "other", "1.0.0", + "scope.attr", "enabled", + &encoder, &chunk); + TEST_CHECK(ret == 0); + if (ret == 0) { + TEST_CHECK(flb_condition_eval_logs(&chunk, &context, &route) == FLB_FALSE); + } + flb_router_chunk_context_reset(&context); + msgpack_sbuffer_destroy(&encoder.buffer); + flb_log_event_encoder_destroy(&encoder); + + free_route_condition(condition); + + /* Scope attributes context */ + memset(&route, 0, sizeof(route)); + cfl_list_init(&route.outputs); + cfl_list_init(&route.processors); + + condition = flb_calloc(1, sizeof(struct flb_route_condition)); + TEST_CHECK(condition != NULL); + if (!condition) { + flb_router_chunk_context_destroy(&context); + return; + } + + cfl_list_init(&condition->rules); + condition->op = FLB_COND_OP_AND; + condition->compiled_status = 0; + condition->compiled = NULL; + condition->is_default = FLB_FALSE; + + rule = flb_calloc(1, sizeof(struct flb_route_condition_rule)); + TEST_CHECK(rule != NULL); + if (!rule) { + free_route_condition(condition); + flb_router_chunk_context_destroy(&context); + return; + } + + cfl_list_init(&rule->_head); + rule->field = flb_sds_create("$scope['attr']"); + rule->op = flb_sds_create("eq"); + rule->value = flb_sds_create("enabled"); + rule->context = RECORD_CONTEXT_OTEL_SCOPE_ATTRIBUTES; + TEST_CHECK(rule->field != NULL && rule->op != NULL && rule->value != NULL); + if (!rule->field || !rule->op || !rule->value) { + if (rule->field) { + flb_sds_destroy(rule->field); + } + if (rule->op) { + flb_sds_destroy(rule->op); + } + if (rule->value) { + flb_sds_destroy(rule->value); + } + flb_free(rule); + free_route_condition(condition); + flb_router_chunk_context_destroy(&context); + return; + } + + cfl_list_add(&rule->_head, &condition->rules); + + route.condition = condition; + route.signals = FLB_ROUTER_SIGNAL_LOGS; + + ret = build_log_chunk_with_otel("backend", "demo", "1.0.0", + "scope.attr", "enabled", + &encoder, &chunk); + TEST_CHECK(ret == 0); + if (ret == 0) { + TEST_CHECK(flb_condition_eval_logs(&chunk, &context, &route) == FLB_TRUE); + } + flb_router_chunk_context_reset(&context); + msgpack_sbuffer_destroy(&encoder.buffer); + flb_log_event_encoder_destroy(&encoder); + + ret = build_log_chunk_with_otel("backend", "demo", "1.0.0", + "scope.attr", "disabled", + &encoder, &chunk); + TEST_CHECK(ret == 0); + if (ret == 0) { + TEST_CHECK(flb_condition_eval_logs(&chunk, &context, &route) == FLB_FALSE); + } + flb_router_chunk_context_reset(&context); + msgpack_sbuffer_destroy(&encoder.buffer); + flb_log_event_encoder_destroy(&encoder); + + free_route_condition(condition); + flb_router_chunk_context_destroy(&context); +} + static void test_router_condition_eval_logs_match() { struct flb_route route; @@ -1226,9 +2045,13 @@ TEST_LIST = { { "parse_basic_file", test_router_config_parse_file_basic }, { "parse_multi_signal_file", test_router_config_parse_file_multi_signal }, { "parse_metrics_file", test_router_config_parse_file_metrics }, + { "parse_contexts_file", test_router_config_parse_file_contexts }, { "apply_config_success", test_router_apply_config_success }, { "apply_config_missing_output", test_router_apply_config_missing_output }, { "route_default_precedence", test_router_route_default_precedence }, + { "condition_eval_logs_metadata_context", test_router_condition_eval_logs_metadata_context }, + { "condition_eval_logs_group_context", test_router_condition_eval_logs_group_context }, + { "condition_eval_logs_otel_contexts", test_router_condition_eval_logs_otel_contexts }, { "condition_eval_logs_match", test_router_condition_eval_logs_match }, { "condition_eval_logs_in_operator", test_router_condition_eval_logs_in_operator }, { "path_should_route_condition", test_router_path_should_route_condition }, diff --git a/tests/runtime/CMakeLists.txt b/tests/runtime/CMakeLists.txt index 01838c68d27..dd76c16faee 100644 --- a/tests/runtime/CMakeLists.txt +++ b/tests/runtime/CMakeLists.txt @@ -61,6 +61,7 @@ if(FLB_OUT_LIB) FLB_RT_TEST(FLB_IN_FLUENTBIT_METRICS "in_fluentbit_metrics.c") FLB_RT_TEST(FLB_IN_PROMETHEUS_TEXTFILE "in_prometheus_textfile.c") FLB_RT_TEST(FLB_IN_KUBERNETES_EVENTS "in_kubernetes_events.c") + FLB_RT_TEST(FLB_IN_OPENTELEMETRY "in_opentelemetry_routing.c") if (FLB_IN_SYSTEMD) FLB_RT_TEST(FLB_IN_SYSTEMD "in_systemd.c") endif () diff --git a/tests/runtime/data/opentelemetry/routing_logs.json b/tests/runtime/data/opentelemetry/routing_logs.json new file mode 100644 index 00000000000..ce39690c608 --- /dev/null +++ b/tests/runtime/data/opentelemetry/routing_logs.json @@ -0,0 +1,203 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { + "key": "service_name", + "value": { + "stringValue": "service-a" + } + }, + { + "key": "service_version", + "value": { + "stringValue": "1.0.0" + } + }, + { + "key": "environment", + "value": { + "stringValue": "production" + } + } + ], + "droppedAttributesCount": 0 + }, + "scopeLogs": [ + { + "scope": { + "name": "scope-a", + "version": "v1.0", + "attributes": [ + { + "key": "component", + "value": { + "stringValue": "backend" + } + } + ], + "droppedAttributesCount": 0 + }, + "logRecords": [ + { + "timeUnixNano": "1640995200000000000", + "body": { + "stringValue": "record from service-a scope-a" + }, + "attributes": [ + { + "key": "level", + "value": { + "stringValue": "info" + } + } + ] + }, + { + "timeUnixNano": "1640995201000000000", + "body": { + "stringValue": "error record from service-a" + }, + "attributes": [ + { + "key": "level", + "value": { + "stringValue": "error" + } + } + ] + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "service_name", + "value": { + "stringValue": "service-b" + } + }, + { + "key": "service_version", + "value": { + "stringValue": "2.0.0" + } + }, + { + "key": "environment", + "value": { + "stringValue": "staging" + } + } + ], + "droppedAttributesCount": 0 + }, + "scopeLogs": [ + { + "scope": { + "name": "scope-b", + "version": "v2.0", + "attributes": [ + { + "key": "component", + "value": { + "stringValue": "frontend" + } + } + ], + "droppedAttributesCount": 0 + }, + "logRecords": [ + { + "timeUnixNano": "1640995202000000000", + "body": { + "stringValue": "record from service-b scope-b" + }, + "attributes": [ + { + "key": "level", + "value": { + "stringValue": "debug" + } + } + ] + } + ] + }, + { + "scope": { + "name": "scope-c", + "version": "v1.5", + "attributes": [ + { + "key": "component", + "value": { + "stringValue": "database" + } + } + ], + "droppedAttributesCount": 0 + }, + "logRecords": [ + { + "timeUnixNano": "1640995203000000000", + "body": { + "stringValue": "query log from service-b" + }, + "attributes": [ + { + "key": "operation", + "value": { + "stringValue": "SELECT" + } + } + ] + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "service_name", + "value": { + "stringValue": "service-c" + } + }, + { + "key": "service_version", + "value": { + "stringValue": "3.0.0" + } + } + ], + "droppedAttributesCount": 0 + }, + "scopeLogs": [ + { + "scope": { + "name": "scope-d", + "version": "v3.0", + "attributes": [], + "droppedAttributesCount": 0 + }, + "logRecords": [ + { + "timeUnixNano": "1640995204000000000", + "body": { + "stringValue": "unmatched record should go to default" + } + } + ] + } + ] + } + ] +} + diff --git a/tests/runtime/data/routing/otlp_comprehensive_routing_test.yaml b/tests/runtime/data/routing/otlp_comprehensive_routing_test.yaml new file mode 100644 index 00000000000..42b5e6b537b --- /dev/null +++ b/tests/runtime/data/routing/otlp_comprehensive_routing_test.yaml @@ -0,0 +1,184 @@ +pipeline: + inputs: + - name: opentelemetry + port: 4318 + routes: + logs: + # Route 1: Match by resource attribute - service_name + - name: service_a_logs + condition: + rules: + - context: otel_resource_attributes + field: "$service_name" + op: eq + value: "service-a" + to: + outputs: + - name: service_a_destination + + # Route 2: Match by resource attribute - service_version + - name: version_2_logs + condition: + rules: + - context: otel_resource_attributes + field: "$service_version" + op: eq + value: "2.0.0" + to: + outputs: + - name: version_2_destination + + # Route 3: Match by resource attribute - environment + - name: production_logs + condition: + rules: + - context: otel_resource_attributes + field: "$environment" + op: eq + value: "production" + to: + outputs: + - name: production_destination + + # Route 4: Match by scope name + - name: scope_a_logs + condition: + rules: + - context: otel_scope_metadata + field: "$name" + op: eq + value: "scope-a" + to: + outputs: + - name: scope_a_destination + + # Route 5: Match by scope version + - name: scope_v2_logs + condition: + rules: + - context: otel_scope_metadata + field: "$version" + op: eq + value: "v2.0" + to: + outputs: + - name: scope_v2_destination + + # Route 6: Match by scope attributes + - name: backend_component_logs + condition: + rules: + - context: otel_scope_attributes + field: "$component" + op: eq + value: "backend" + to: + outputs: + - name: backend_component_destination + + # Route 7: Match by record body content + - name: error_body_logs + condition: + rules: + - context: body + field: "$log" + op: regex + value: "error" + to: + outputs: + - name: error_body_destination + + # Route 8: Match by record attributes (metadata) + - name: info_level_logs + condition: + rules: + - context: metadata + field: "$otlp['attributes']['level']" + op: eq + value: "info" + to: + outputs: + - name: info_level_destination + + # Route 9: Match by record attributes in metadata + - name: select_operation_logs + condition: + rules: + - context: metadata + field: "$otlp['attributes']['operation']" + op: eq + value: "SELECT" + to: + outputs: + - name: select_operation_destination + + # Route 10: Default route for unmatched records + - name: default_logs + condition: + default: true + to: + outputs: + - name: default_destination + + outputs: + - name: file + alias: service_a_destination + match: service_a_logs + path: otlp_routing_test_output + file: service_a_logs.out + + - name: file + alias: version_2_destination + match: version_2_logs + path: otlp_routing_test_output + file: version_2_logs.out + + - name: file + alias: production_destination + match: production_logs + path: otlp_routing_test_output + file: production_logs.out + + - name: file + alias: scope_a_destination + match: scope_a_logs + path: otlp_routing_test_output + file: scope_a_logs.out + + - name: file + alias: scope_v2_destination + match: scope_v2_logs + path: otlp_routing_test_output + file: scope_v2_logs.out + + - name: file + alias: backend_component_destination + match: backend_component_logs + path: otlp_routing_test_output + file: backend_component_logs.out + + - name: file + alias: error_body_destination + match: error_body_logs + path: otlp_routing_test_output + file: error_body_logs.out + + - name: file + alias: info_level_destination + match: info_level_logs + path: otlp_routing_test_output + file: info_level_logs.out + + - name: file + alias: select_operation_destination + match: select_operation_logs + path: otlp_routing_test_output + file: select_operation_logs.out + + - name: file + alias: default_destination + match: default_logs + path: otlp_routing_test_output + file: default_logs.out + + diff --git a/tests/runtime/in_opentelemetry_routing.c b/tests/runtime/in_opentelemetry_routing.c new file mode 100644 index 00000000000..a63ddbffeb0 --- /dev/null +++ b/tests/runtime/in_opentelemetry_routing.c @@ -0,0 +1,484 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2025 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "flb_tests_runtime.h" +#include "../../plugins/in_opentelemetry/opentelemetry.h" +#include "../../plugins/in_opentelemetry/opentelemetry_logs.h" + +#define JSON_CONTENT_TYPE "application/json" +#define PORT_OTEL 4318 +#define V1_ENDPOINT_LOGS "/v1/logs" +#define MAX_ROUTES 32 + +/* Route expectation: file path and expected count */ +struct route_expectation { + const char *route_name; + const char *output_file; + int expected_count; +}; + +struct test_ctx { + flb_ctx_t *flb; + int i_ffd; + char output_dir[PATH_MAX]; +}; + +#define TEST_OUTPUT_DIR "otlp_routing_test_output" + +/* Construct absolute path to config file */ +static char *get_config_path(const char *config_file) +{ + char path[PATH_MAX]; + char *resolved; + char *real_resolved; + char cwd[PATH_MAX]; + size_t cwd_len; + size_t config_file_len; + int ret; + + /* Try FLB_TESTS_DATA_PATH/data/routing/ first */ + snprintf(path, sizeof(path), "%s/data/routing/%s", FLB_TESTS_DATA_PATH, config_file); + if (access(path, R_OK) == 0) { + real_resolved = realpath(path, NULL); + if (real_resolved) { + resolved = flb_strdup(real_resolved); + free(real_resolved); + return resolved; + } + resolved = flb_strdup(path); + return resolved; + } + + /* Try FLB_TESTS_DATA_PATH (tests/runtime directory) */ + snprintf(path, sizeof(path), "%s/%s", FLB_TESTS_DATA_PATH, config_file); + if (access(path, R_OK) == 0) { + real_resolved = realpath(path, NULL); + if (real_resolved) { + resolved = flb_strdup(real_resolved); + free(real_resolved); + return resolved; + } + resolved = flb_strdup(path); + return resolved; + } + + /* Try source root (go up from tests/runtime) */ + snprintf(path, sizeof(path), "%s/../../%s", FLB_TESTS_DATA_PATH, config_file); + if (access(path, R_OK) == 0) { + real_resolved = realpath(path, NULL); + if (real_resolved) { + resolved = flb_strdup(real_resolved); + free(real_resolved); + return resolved; + } + resolved = flb_strdup(path); + return resolved; + } + + /* Try current working directory */ + if (getcwd(cwd, sizeof(cwd)) != NULL) { + cwd_len = strlen(cwd); + config_file_len = strlen(config_file); + if (cwd_len + 1 + config_file_len < sizeof(path)) { + ret = snprintf(path, sizeof(path), "%s/%s", cwd, config_file); + if (ret > 0 && (size_t)ret < sizeof(path)) { + if (access(path, R_OK) == 0) { + real_resolved = realpath(path, NULL); + if (real_resolved) { + resolved = flb_strdup(real_resolved); + free(real_resolved); + return resolved; + } + resolved = flb_strdup(path); + return resolved; + } + } + } + } + + /* Return original path as fallback */ + return flb_strdup(config_file); +} + +/* Get opentelemetry input instance */ +static struct flb_input_instance *get_opentelemetry_instance(flb_ctx_t *flb_ctx) +{ + struct mk_list *head; + struct flb_input_instance *ins; + + mk_list_foreach(head, &flb_ctx->config->inputs) { + ins = mk_list_entry(head, struct flb_input_instance, _head); + if (ins->p && strcmp(ins->p->name, "opentelemetry") == 0) { + return ins; + } + } + return NULL; +} + +/* Directly inject JSON payload into opentelemetry plugin */ +static int inject_otlp_json(flb_ctx_t *flb_ctx, const char *json_data, size_t json_size) +{ + struct flb_input_instance *ins; + struct flb_opentelemetry *otel_ctx; + flb_sds_t content_type; + flb_sds_t tag; + int ret; + + /* Get opentelemetry input instance */ + ins = get_opentelemetry_instance(flb_ctx); + if (!ins || !ins->context) { + return -1; + } + + otel_ctx = (struct flb_opentelemetry *)ins->context; + + /* Use default tag if not set */ + if (ins->tag && ins->tag_len > 0) { + tag = flb_sds_create_len(ins->tag, ins->tag_len); + } + else { + tag = flb_sds_create("opentelemetry.0"); + } + + /* Set content type */ + content_type = flb_sds_create("application/json"); + + /* Process logs directly */ + ret = opentelemetry_process_logs(otel_ctx, content_type, tag, flb_sds_len(tag), + (void *)json_data, json_size); + + flb_sds_destroy(content_type); + flb_sds_destroy(tag); + + return ret; +} + +/* Create test context from YAML config file */ +static struct test_ctx *test_ctx_create(const char *config_file) +{ + struct test_ctx *ctx; + char *config_path; + int ret; + char cwd[PATH_MAX]; + size_t cwd_len; + size_t test_dir_len; + + ctx = flb_calloc(1, sizeof(struct test_ctx)); + if (!TEST_CHECK(ctx != NULL)) { + return NULL; + } + + ctx->flb = flb_create(); + TEST_CHECK(ctx->flb != NULL); + + /* Create output directory */ + if (getcwd(cwd, sizeof(cwd)) != NULL) { + cwd_len = strlen(cwd); + test_dir_len = strlen(TEST_OUTPUT_DIR); + if (cwd_len + 1 + test_dir_len < sizeof(ctx->output_dir)) { + ret = snprintf(ctx->output_dir, sizeof(ctx->output_dir), "%s/%s", cwd, TEST_OUTPUT_DIR); + if (ret < 0 || (size_t)ret >= sizeof(ctx->output_dir)) { + snprintf(ctx->output_dir, sizeof(ctx->output_dir), "./%s", TEST_OUTPUT_DIR); + } + } + else { + snprintf(ctx->output_dir, sizeof(ctx->output_dir), "./%s", TEST_OUTPUT_DIR); + } + } + else { + snprintf(ctx->output_dir, sizeof(ctx->output_dir), "./%s", TEST_OUTPUT_DIR); + } + + /* Create directory if it doesn't exist */ + ret = mkdir(ctx->output_dir, 0755); + if (ret != 0 && errno != EEXIST) { + flb_error("[test] Failed to create output directory: %s", ctx->output_dir); + flb_destroy(ctx->flb); + flb_free(ctx); + return NULL; + } + + + /* Resolve config file path */ + config_path = get_config_path(config_file); + TEST_CHECK(config_path != NULL); + if (!config_path) { + flb_destroy(ctx->flb); + flb_free(ctx); + return NULL; + } + + /* Load config from YAML file */ + ret = flb_lib_config_file(ctx->flb, config_path); + flb_free(config_path); + if (!TEST_CHECK(ret == 0)) { + flb_destroy(ctx->flb); + flb_free(ctx); + return NULL; + } + + return ctx; +} + +static void test_ctx_destroy(struct test_ctx *ctx) +{ + if (!ctx) { + return; + } + + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + + /* Cleanup output directory (optional - keep for debugging) */ + /* Can uncomment to clean up: + char cmd[PATH_MAX * 2]; + snprintf(cmd, sizeof(cmd), "rm -rf %s", ctx->output_dir); + system(cmd); + */ + + flb_free(ctx); +} + +/* Read file content and count JSON records */ +static int count_records_in_file(const char *filepath) +{ + FILE *fp; + char line[8192]; + int count = 0; + char *trimmed; + + fp = fopen(filepath, "r"); + if (!fp) { + return -1; + } + + while (fgets(line, sizeof(line), fp)) { + /* Trim whitespace */ + trimmed = line; + while (*trimmed == ' ' || *trimmed == '\t' || *trimmed == '\n' || *trimmed == '\r') { + trimmed++; + } + + /* Skip empty lines */ + if (*trimmed == '\0') { + continue; + } + + /* Count lines that contain a timestamp pattern (tag: [timestamp) or JSON object/array */ + /* File output in JSON format writes: "tag: [timestamp, {...}]" */ + /* File output in plain format writes: "{...}" */ + if (strstr(trimmed, ": [") != NULL || + *trimmed == '{' || + *trimmed == '[') { + count++; + } + } + + fclose(fp); + return count; +} + +/* Remove existing output files before test */ +static void cleanup_output_files(struct test_ctx *ctx, struct route_expectation *expectations, int count) +{ + int i; + char filepath[PATH_MAX]; + size_t dir_len; + size_t file_len; + int ret; + + for (i = 0; i < count; i++) { + dir_len = strlen(ctx->output_dir); + file_len = strlen(expectations[i].output_file); + if (dir_len + 1 + file_len < sizeof(filepath)) { + ret = snprintf(filepath, sizeof(filepath), "%s/%s", ctx->output_dir, expectations[i].output_file); + if (ret > 0 && (size_t)ret < sizeof(filepath)) { + unlink(filepath); + } + } + } +} + +/* Verify all expectations by reading output files */ +static int verify_expectations(struct route_expectation *expectations, int count, struct test_ctx *ctx) +{ + int i; + int all_passed = 1; + char filepath[PATH_MAX]; + int actual_count; + struct route_expectation *exp; + size_t dir_len; + size_t file_len; + int ret; + + for (i = 0; i < count; i++) { + exp = &expectations[i]; + + dir_len = strlen(ctx->output_dir); + file_len = strlen(exp->output_file); + if (dir_len + 1 + file_len < sizeof(filepath)) { + ret = snprintf(filepath, sizeof(filepath), "%s/%s", ctx->output_dir, exp->output_file); + if (ret > 0 && (size_t)ret < sizeof(filepath)) { + actual_count = count_records_in_file(filepath); + } + else { + flb_error("[test] Route '%s': output file path too long", exp->route_name); + actual_count = -1; + } + } + else { + flb_error("[test] Route '%s': output file path too long", exp->route_name); + actual_count = -1; + } + + if (actual_count < 0) { + if (dir_len + 1 + file_len < sizeof(filepath)) { + flb_error("[test] Route '%s': failed to read output file: %s", + exp->route_name, filepath); + } + all_passed = 0; + } + else if (actual_count != exp->expected_count) { + flb_error("[test] Route '%s': expected %d records, got %d (file: %s)", + exp->route_name, exp->expected_count, actual_count, filepath); + all_passed = 0; + } + else { + flb_info("[test] Route '%s': ✓ %d records (file: %s)", + exp->route_name, actual_count, filepath); + } + } + + return all_passed; +} + +/* Load JSON test data from file */ +static flb_sds_t load_json_test_data(const char *filename) +{ + char path[PATH_MAX]; + flb_sds_t content; + + /* Try FLB_TESTS_DATA_PATH first */ + snprintf(path, sizeof(path), "%s/data/opentelemetry/%s", FLB_TESTS_DATA_PATH, filename); + content = flb_file_read(path); + + /* Try relative to current directory if not found */ + if (!content) { + snprintf(path, sizeof(path), "data/opentelemetry/%s", filename); + content = flb_file_read(path); + } + + return content; +} + +/* Main test function */ +static void flb_test_otlp_routing(const char *config_file, + const char *json_file, + struct route_expectation *expectations, + int exp_count) +{ + struct test_ctx *ctx; + flb_sds_t json_content; + int ret; + + /* Load JSON test data */ + json_content = load_json_test_data(json_file); + TEST_CHECK(json_content != NULL); + if (!json_content) { + return; + } + + /* Create test context */ + ctx = test_ctx_create(config_file); + TEST_CHECK(ctx != NULL); + if (!ctx) { + flb_sds_destroy(json_content); + return; + } + + /* Clean up any existing output files */ + cleanup_output_files(ctx, expectations, exp_count); + + /* Start Fluent Bit */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Directly inject JSON payload into opentelemetry plugin */ + ret = inject_otlp_json(ctx->flb, json_content, flb_sds_len(json_content)); + TEST_CHECK(ret == 0); + + /* Wait for records to be processed and flushed to files */ + flb_time_msleep(3000); + + /* Verify expectations by reading output files */ + ret = verify_expectations(expectations, exp_count, ctx); + TEST_CHECK(ret == 1); + + /* Cleanup */ + test_ctx_destroy(ctx); + flb_sds_destroy(json_content); +} + +/* Test case: Comprehensive routing with multiple routes */ +void flb_test_otlp_comprehensive_routing() +{ + struct route_expectation expectations[] = { + {"service_a_logs", "service_a_logs.out", 2}, /* Record 1, Record 2 */ + {"version_2_logs", "version_2_logs.out", 2}, /* Record 3, Record 4 */ + {"production_logs", "production_logs.out", 2}, /* Record 1, Record 2 */ + {"scope_a_logs", "scope_a_logs.out", 2}, /* Record 1, Record 2 */ + {"scope_v2_logs", "scope_v2_logs.out", 1}, /* Record 3 */ + {"backend_component_logs", "backend_component_logs.out", 2}, /* Record 1, Record 2 */ + {"error_body_logs", "error_body_logs.out", 1}, /* Record 2 */ + {"info_level_logs", "info_level_logs.out", 1}, /* Record 1 */ + {"select_operation_logs", "select_operation_logs.out", 1}, /* Record 4 */ + {"default_logs", "default_logs.out", 1}, /* Record 5 */ + }; + + /* Config file should be in the same directory as the test */ + flb_test_otlp_routing( + "otlp_comprehensive_routing_test.yaml", + "routing_logs.json", + expectations, + sizeof(expectations) / sizeof(expectations[0]) + ); +} + +TEST_LIST = { + {"otlp_comprehensive_routing", flb_test_otlp_comprehensive_routing}, + {NULL, NULL} +}; +