diff --git a/include/fluent-bit/flb_config.h b/include/fluent-bit/flb_config.h index b6090eea537..f04769b0e8b 100644 --- a/include/fluent-bit/flb_config.h +++ b/include/fluent-bit/flb_config.h @@ -318,6 +318,8 @@ struct flb_config { struct flb_task_map *task_map; size_t task_map_size; + int json_escape_unicode; + int dry_run; }; @@ -412,4 +414,7 @@ enum conf_type { #define FLB_CONF_STR_SCHED_CAP "scheduler.cap" #define FLB_CONF_STR_SCHED_BASE "scheduler.base" +/* json escape */ +#define FLB_CONF_UNICODE_STR_JSON_ESCAPE "json.escape_unicode" + #endif diff --git a/include/fluent-bit/flb_pack.h b/include/fluent-bit/flb_pack.h index 875c20f4251..5c76842cde9 100644 --- a/include/fluent-bit/flb_pack.h +++ b/include/fluent-bit/flb_pack.h @@ -91,15 +91,17 @@ int flb_pack_json_valid(const char *json, size_t len); flb_sds_t flb_pack_msgpack_to_json_format(const char *data, uint64_t bytes, int json_format, int date_format, - flb_sds_t date_key); + flb_sds_t date_key, int escape_unicode); int flb_pack_to_json_format_type(const char *str); int flb_pack_to_json_date_type(const char *str); void flb_pack_print(const char *data, size_t bytes); int flb_msgpack_to_json(char *json_str, size_t str_len, - const msgpack_object *obj); -char* flb_msgpack_to_json_str(size_t size, const msgpack_object *obj); -flb_sds_t flb_msgpack_raw_to_json_sds(const void *in_buf, size_t in_size); + const msgpack_object *obj, + int escape_unicode); +char* flb_msgpack_to_json_str(size_t size, const msgpack_object *obj, + int escape_unicode); +flb_sds_t flb_msgpack_raw_to_json_sds(const void *in_buf, size_t in_size, int escape_unicode); int flb_pack_time_now(msgpack_packer *pck); int flb_msgpack_expand_map(char *map_data, size_t map_size, diff --git a/include/fluent-bit/flb_utils.h b/include/fluent-bit/flb_utils.h index 56f286f6738..e1dc5ecbc90 100644 --- a/include/fluent-bit/flb_utils.h +++ b/include/fluent-bit/flb_utils.h @@ -57,9 +57,9 @@ void flb_utils_bytes_to_human_readable_size(size_t bytes, char *out_buf, size_t size); int flb_utils_time_split(const char *time, int *sec, long *nsec); int flb_utils_write_str(char *buf, int *off, size_t size, - const char *str, size_t str_len); + const char *str, size_t str_len, int escape_unicode); int flb_utils_write_str_buf(const char *str, size_t str_len, - char **out, size_t *out_size); + char **out, size_t *out_size, int escape_unicode); int flb_utils_url_split(const char *in_url, char **out_protocol, char **out_host, char **out_port, char **out_uri); diff --git a/plugins/filter_expect/expect.c b/plugins/filter_expect/expect.c index 425c9017136..7722f0f6f79 100644 --- a/plugins/filter_expect/expect.c +++ b/plugins/filter_expect/expect.c @@ -272,7 +272,7 @@ static char *ra_value_type_to_str(struct flb_ra_value *val) return "UNKNOWN"; } -static int rule_apply(struct flb_expect *ctx, msgpack_object map) +static int rule_apply(struct flb_expect *ctx, msgpack_object map, struct flb_config *config) { int n = 0; char *json; @@ -292,7 +292,7 @@ static int rule_apply(struct flb_expect *ctx, msgpack_object map) continue; } - json = flb_msgpack_to_json_str(size, &map); + json = flb_msgpack_to_json_str(size, &map, config->json_escape_unicode); flb_plg_error(ctx->ins, "exception on rule #%i 'key_exists', key '%s' " "not found. Record content:\n%s", @@ -305,7 +305,7 @@ static int rule_apply(struct flb_expect *ctx, msgpack_object map) n++; continue; } - json = flb_msgpack_to_json_str(size, &map); + json = flb_msgpack_to_json_str(size, &map, config->json_escape_unicode); flb_plg_error(ctx->ins, "exception on rule #%i 'key_not_exists', key '%s' " "exists. Record content:\n%s", @@ -316,7 +316,7 @@ static int rule_apply(struct flb_expect *ctx, msgpack_object map) } else if (rule->type == FLB_EXP_KEY_VAL_NULL) { if (!val) { - json = flb_msgpack_to_json_str(size, &map); + json = flb_msgpack_to_json_str(size, &map, config->json_escape_unicode); flb_plg_error(ctx->ins, "exception on rule #%i 'key_val_is_null', " "key '%s' not found. Record content:\n%s", @@ -325,7 +325,7 @@ static int rule_apply(struct flb_expect *ctx, msgpack_object map) return FLB_FALSE; } if (val->type != FLB_RA_NULL) { - json = flb_msgpack_to_json_str(size, &map); + json = flb_msgpack_to_json_str(size, &map, config->json_escape_unicode); flb_plg_error(ctx->ins, "exception on rule #%i 'key_val_is_null', " "key '%s' contains a value type '%s'. " @@ -340,7 +340,7 @@ static int rule_apply(struct flb_expect *ctx, msgpack_object map) } else if (rule->type == FLB_EXP_KEY_VAL_NOT_NULL) { if (!val) { - json = flb_msgpack_to_json_str(size, &map); + json = flb_msgpack_to_json_str(size, &map, config->json_escape_unicode); flb_plg_error(ctx->ins, "exception on rule #%i 'key_val_is_not_null', " "key '%s' not found. Record content:\n%s", @@ -349,7 +349,7 @@ static int rule_apply(struct flb_expect *ctx, msgpack_object map) return FLB_FALSE; } if (val->type == FLB_RA_NULL) { - json = flb_msgpack_to_json_str(size, &map); + json = flb_msgpack_to_json_str(size, &map, config->json_escape_unicode); flb_plg_error(ctx->ins, "exception on rule #%i 'key_val_is_not_null', " "key '%s' contains a value type '%s'. " @@ -364,7 +364,7 @@ static int rule_apply(struct flb_expect *ctx, msgpack_object map) } else if (rule->type == FLB_EXP_KEY_VAL_EQ) { if (!val) { - json = flb_msgpack_to_json_str(size, &map); + json = flb_msgpack_to_json_str(size, &map, config->json_escape_unicode); flb_plg_error(ctx->ins, "exception on rule #%i 'key_val_is_null', " "key '%s' not found. Record content:\n%s", @@ -376,7 +376,7 @@ static int rule_apply(struct flb_expect *ctx, msgpack_object map) if (val->type == FLB_RA_STRING) { if (flb_sds_cmp(val->val.string, rule->expect, flb_sds_len(rule->expect)) != 0) { - json = flb_msgpack_to_json_str(size, &map); + json = flb_msgpack_to_json_str(size, &map, config->json_escape_unicode); flb_plg_error(ctx->ins, "exception on rule #%i 'key_val_eq', " "key value '%s' is different than " @@ -430,7 +430,7 @@ static int cb_expect_filter(const void *data, size_t bytes, while ((ret = flb_log_event_decoder_next( &log_decoder, &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - ret = rule_apply(ctx, *log_event.body); + ret = rule_apply(ctx, *log_event.body, config); if (ret == FLB_TRUE) { /* rule matches, we are good */ continue; diff --git a/plugins/filter_nightfall/nightfall_api.c b/plugins/filter_nightfall/nightfall_api.c index 91ecf7948e2..accd4471128 100644 --- a/plugins/filter_nightfall/nightfall_api.c +++ b/plugins/filter_nightfall/nightfall_api.c @@ -195,8 +195,8 @@ static flb_sds_t build_request_body(struct flb_filter_nightfall *ctx, msgpack_pack_str_with_body(&req_pk, "policyUUIDs", 11); msgpack_pack_array(&req_pk, 1); msgpack_pack_str_with_body(&req_pk, ctx->policy_id, 36); - - request_body = flb_msgpack_raw_to_json_sds(req_sbuf.data, req_sbuf.size); + + request_body = flb_msgpack_raw_to_json_sds(req_sbuf.data, req_sbuf.size, FLB_TRUE); msgpack_sbuffer_destroy(&req_sbuf); flb_sds_destroy(num_str); diff --git a/plugins/filter_wasm/filter_wasm.c b/plugins/filter_wasm/filter_wasm.c index e7951bd8c28..052e095a1ba 100644 --- a/plugins/filter_wasm/filter_wasm.c +++ b/plugins/filter_wasm/filter_wasm.c @@ -104,7 +104,7 @@ static int cb_wasm_filter(const void *data, size_t bytes, switch(ctx->event_format) { case FLB_FILTER_WASM_FMT_JSON: /* Encode as JSON from msgpack */ - buf = flb_msgpack_to_json_str(alloc_size, log_event.body); + buf = flb_msgpack_to_json_str(alloc_size, log_event.body, config->json_escape_unicode); if (buf) { /* Execute WASM program */ diff --git a/plugins/out_azure/azure.c b/plugins/out_azure/azure.c index e5eee8c3b17..a43b13594d9 100644 --- a/plugins/out_azure/azure.c +++ b/plugins/out_azure/azure.c @@ -51,7 +51,8 @@ static int cb_azure_init(struct flb_output_instance *ins, static int azure_format(const void *in_buf, size_t in_bytes, flb_sds_t tag, flb_sds_t *tag_val_out, char **out_buf, size_t *out_size, - struct flb_azure *ctx) + struct flb_azure *ctx, + struct flb_config *config) { int i; int array_size = 0; @@ -160,7 +161,8 @@ static int azure_format(const void *in_buf, size_t in_bytes, msgpack_sbuffer_destroy(&tmp_sbuf); } - record = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + record = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); if (!record) { flb_errno(); @@ -317,7 +319,8 @@ static void cb_azure_flush(struct flb_event_chunk *event_chunk, /* Convert binary logs into a JSON payload */ ret = azure_format(event_chunk->data, event_chunk->size, - event_chunk->tag, &final_log_type, &buf_data, &buf_size, ctx); + event_chunk->tag, &final_log_type, &buf_data, &buf_size, ctx, + config); /* If cannot get matching record using log_type_prefix, use log_type directly */ if (!final_log_type) { final_log_type = ctx->log_type; diff --git a/plugins/out_azure_blob/azure_blob.c b/plugins/out_azure_blob/azure_blob.c index d0ae18d503c..a650809c4d4 100644 --- a/plugins/out_azure_blob/azure_blob.c +++ b/plugins/out_azure_blob/azure_blob.c @@ -68,7 +68,8 @@ static int azure_blob_format(struct flb_config *config, out_buf = flb_pack_msgpack_to_json_format(data, bytes, FLB_PACK_JSON_FORMAT_LINES, FLB_PACK_JSON_DATE_ISO8601, - ctx->date_key); + ctx->date_key, + config->json_escape_unicode); if (!out_buf) { return -1; } diff --git a/plugins/out_azure_kusto/azure_kusto.c b/plugins/out_azure_kusto/azure_kusto.c index 04d00ca2922..6f2abd53642 100644 --- a/plugins/out_azure_kusto/azure_kusto.c +++ b/plugins/out_azure_kusto/azure_kusto.c @@ -957,7 +957,8 @@ static int cb_azure_kusto_init(struct flb_output_instance *ins, struct flb_confi */ static int azure_kusto_format(struct flb_azure_kusto *ctx, const char *tag, int tag_len, const void *data, size_t bytes, void **out_data, - size_t *out_size) + size_t *out_size, + struct flb_config *config) { int index; int records = 0; @@ -1064,7 +1065,8 @@ static int azure_kusto_format(struct flb_azure_kusto *ctx, const char *tag, int msgpack_pack_str_body(&mp_pck, "log_attribute_missing", 20); } - flb_sds_t json_record = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + flb_sds_t json_record = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); if (!json_record) { flb_plg_error(ctx->ins, "error converting msgpack to JSON"); flb_sds_destroy(out_buf); @@ -1234,7 +1236,8 @@ static void cb_azure_kusto_flush(struct flb_event_chunk *event_chunk, /* Reformat msgpack to JSON payload */ ret = azure_kusto_format(ctx, tag_name, tag_name_len, event_chunk->data, - event_chunk->size, (void **)&json, &json_size); + event_chunk->size, (void **)&json, &json_size, + config); if (ret != 0) { flb_plg_error(ctx->ins, "cannot reformat data into json"); ret = FLB_RETRY; @@ -1345,7 +1348,8 @@ static void cb_azure_kusto_flush(struct flb_event_chunk *event_chunk, /* Reformat msgpack data to JSON payload */ ret = azure_kusto_format(ctx, event_chunk->tag, tag_len, event_chunk->data, - event_chunk->size, (void **)&json, &json_size); + event_chunk->size, (void **)&json, &json_size, + config); if (ret != 0) { flb_plg_error(ctx->ins, "cannot reformat data into json"); ret = FLB_RETRY; diff --git a/plugins/out_azure_logs_ingestion/azure_logs_ingestion.c b/plugins/out_azure_logs_ingestion/azure_logs_ingestion.c index e816488cd8a..f9f11188b5f 100644 --- a/plugins/out_azure_logs_ingestion/azure_logs_ingestion.c +++ b/plugins/out_azure_logs_ingestion/azure_logs_ingestion.c @@ -54,7 +54,8 @@ static int cb_azure_logs_ingestion_init(struct flb_output_instance *ins, allocates sds string */ static int az_li_format(const void *in_buf, size_t in_bytes, char **out_buf, size_t *out_size, - struct flb_az_li *ctx) + struct flb_az_li *ctx, + struct flb_config *config) { int i; int array_size = 0; @@ -141,7 +142,8 @@ static int az_li_format(const void *in_buf, size_t in_bytes, msgpack_sbuffer_destroy(&tmp_sbuf); } - record = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + record = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); if (!record) { flb_errno(); msgpack_sbuffer_destroy(&mp_sbuf); @@ -266,7 +268,8 @@ static void cb_azure_logs_ingestion_flush(struct flb_event_chunk *event_chunk, /* Convert binary logs into a JSON payload */ ret = az_li_format(event_chunk->data, event_chunk->size, - &json_payload, &json_payload_size, ctx); + &json_payload, &json_payload_size, ctx, + config); if (ret == -1) { flb_upstream_conn_release(u_conn); FLB_OUTPUT_RETURN(FLB_ERROR); diff --git a/plugins/out_bigquery/bigquery.c b/plugins/out_bigquery/bigquery.c index b21cff5241c..c5cfeeb2b25 100644 --- a/plugins/out_bigquery/bigquery.c +++ b/plugins/out_bigquery/bigquery.c @@ -847,7 +847,8 @@ static int cb_bigquery_init(struct flb_output_instance *ins, static int bigquery_format(const void *data, size_t bytes, const char *tag, size_t tag_len, char **out_data, size_t *out_size, - struct flb_bigquery *ctx) + struct flb_bigquery *ctx, + struct flb_config *config) { int array_size = 0; flb_sds_t out_buf; @@ -937,7 +938,8 @@ static int bigquery_format(const void *data, size_t bytes, } /* Convert from msgpack to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); flb_log_event_decoder_destroy(&log_decoder); msgpack_sbuffer_destroy(&mp_sbuf); @@ -996,7 +998,7 @@ static void cb_bigquery_flush(struct flb_event_chunk *event_chunk, /* Reformat msgpack to bigquery JSON payload */ ret = bigquery_format(event_chunk->data, event_chunk->size, event_chunk->tag, flb_sds_len(event_chunk->tag), - &payload_buf, &payload_size, ctx); + &payload_buf, &payload_size, ctx, config); if (ret != 0) { flb_upstream_conn_release(u_conn); flb_sds_destroy(token); diff --git a/plugins/out_calyptia/calyptia.c b/plugins/out_calyptia/calyptia.c index d26270583af..9966828ce37 100644 --- a/plugins/out_calyptia/calyptia.c +++ b/plugins/out_calyptia/calyptia.c @@ -311,7 +311,7 @@ static flb_sds_t get_agent_metadata(struct flb_calyptia *ctx) flb_mp_map_header_end(&mh); /* convert to json */ - meta = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + meta = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); /* could be ASCII */ msgpack_sbuffer_destroy(&mp_sbuf); return meta; @@ -532,7 +532,7 @@ static int store_session_get(struct flb_calyptia *ctx, } /* decode */ - json = flb_msgpack_raw_to_json_sds(buf, size); + json = flb_msgpack_raw_to_json_sds(buf, size, FLB_TRUE); /* TODO: could be ASCII? */ flb_free(buf); if (!json) { return -1; @@ -1029,7 +1029,8 @@ static void cb_calyptia_flush(struct flb_event_chunk *event_chunk, event_chunk->size, FLB_PACK_JSON_FORMAT_STREAM, FLB_PACK_JSON_DATE_DOUBLE, - NULL); + NULL, + FLB_TRUE); /* Trace is ASCII */ if (json == NULL) { flb_upstream_conn_release(u_conn); FLB_OUTPUT_RETURN(FLB_RETRY); diff --git a/plugins/out_chronicle/chronicle.c b/plugins/out_chronicle/chronicle.c index ac43c8b997c..855e1b7fc46 100644 --- a/plugins/out_chronicle/chronicle.c +++ b/plugins/out_chronicle/chronicle.c @@ -515,7 +515,9 @@ static int cb_chronicle_init(struct flb_output_instance *ins, return 0; } -static flb_sds_t flb_pack_msgpack_extract_log_key(void *out_context, uint64_t bytes, struct flb_log_event log_event) +static flb_sds_t flb_pack_msgpack_extract_log_key(void *out_context, uint64_t bytes, + struct flb_log_event log_event, + struct flb_config *config) { int i; int map_size; @@ -591,7 +593,8 @@ static flb_sds_t flb_pack_msgpack_extract_log_key(void *out_context, uint64_t by } else { ret = flb_msgpack_to_json(val_buf + val_offset, - msgpack_size - val_offset, &val); + msgpack_size - val_offset, &val, + config->json_escape_unicode); if (ret < 0) { break; } @@ -677,7 +680,8 @@ static int chronicle_format(const void *data, size_t bytes, size_t last_offset, size_t threshold, size_t *out_offset, struct flb_log_event_decoder *log_decoder, - struct flb_chronicle *ctx) + struct flb_chronicle *ctx, + struct flb_config *config) { int len; int ret; @@ -722,7 +726,7 @@ static int chronicle_format(const void *data, size_t bytes, last_off = off; if (ctx->log_key != NULL) { - log_text = flb_pack_msgpack_extract_log_key(ctx, bytes, log_event); + log_text = flb_pack_msgpack_extract_log_key(ctx, bytes, log_event, config); if (log_text == NULL) { flb_plg_error(ctx->ins, "log_key extraction failed, skipping record"); continue; @@ -730,7 +734,8 @@ static int chronicle_format(const void *data, size_t bytes, log_text_size = flb_sds_len(log_text); } else { - json_str = flb_msgpack_to_json_str(alloc_size, log_event.body); + json_str = flb_msgpack_to_json_str(alloc_size, log_event.body, + config->json_escape_unicode); if (json_str == NULL) { flb_plg_error(ctx->ins, "Could not convert record to json string"); msgpack_sbuffer_destroy(&mp_sbuf); @@ -865,7 +870,8 @@ static int chronicle_format(const void *data, size_t bytes, } /* Convert from msgpack to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); msgpack_sbuffer_destroy(&mp_sbuf); if (!out_buf) { @@ -903,7 +909,7 @@ static int cb_chronicle_format_test(struct flb_config *config, ret = chronicle_format(data, bytes, tag, tag_len, (char **)out_data, out_size, 0, bytes, &out_offset, - &log_decoder, ctx); + &log_decoder, ctx, config); flb_log_event_decoder_destroy(&log_decoder); return ret; @@ -994,7 +1000,7 @@ static void cb_chronicle_flush(struct flb_event_chunk *event_chunk, &payload_buf, &payload_size, offset, threshold, &out_offset, &log_decoder, - ctx); + ctx, config); if (ret != 0) { flb_upstream_conn_release(u_conn); flb_sds_destroy(token); diff --git a/plugins/out_cloudwatch_logs/cloudwatch_api.c b/plugins/out_cloudwatch_logs/cloudwatch_api.c index fe24f3937d5..cf60a936e59 100644 --- a/plugins/out_cloudwatch_logs/cloudwatch_api.c +++ b/plugins/out_cloudwatch_logs/cloudwatch_api.c @@ -378,7 +378,8 @@ static int truncate_log(const struct flb_cloudwatch *ctx, const char *log_buffer * which means a send must occur */ int process_event(struct flb_cloudwatch *ctx, struct cw_flush *buf, - const msgpack_object *obj, struct flb_time *tms) + const msgpack_object *obj, struct flb_time *tms, + struct flb_config *config) { size_t written; int ret; @@ -389,8 +390,8 @@ int process_event(struct flb_cloudwatch *ctx, struct cw_flush *buf, tmp_buf_ptr = buf->tmp_buf + buf->tmp_buf_offset; ret = flb_msgpack_to_json(tmp_buf_ptr, - buf->tmp_buf_size - buf->tmp_buf_offset, - obj); + buf->tmp_buf_size - buf->tmp_buf_offset, + obj, config->json_escape_unicode); if (ret <= 0) { /* * failure to write to buffer, @@ -424,7 +425,8 @@ int process_event(struct flb_cloudwatch *ctx, struct cw_flush *buf, } offset = 0; if (!flb_utils_write_str(buf->event_buf, &offset, size, - tmp_buf_ptr, written)) { + tmp_buf_ptr, written, + config->json_escape_unicode)) { return -1; } written = offset; @@ -551,7 +553,8 @@ int send_log_events(struct flb_cloudwatch *ctx, struct cw_flush *buf) { */ int add_event(struct flb_cloudwatch *ctx, struct cw_flush *buf, struct log_stream *stream, - const msgpack_object *obj, struct flb_time *tms) + const msgpack_object *obj, struct flb_time *tms, + struct flb_config *config) { int ret; struct cw_event *event; @@ -572,7 +575,7 @@ int add_event(struct flb_cloudwatch *ctx, struct cw_flush *buf, reset_flush_buf(ctx, buf); } - ret = process_event(ctx, buf, obj, tms); + ret = process_event(ctx, buf, obj, tms, config); if (ret < 0) { return -1; } @@ -788,7 +791,7 @@ int pack_emf_payload(struct flb_cloudwatch *ctx, static int process_log_events(struct flb_cloudwatch *ctx, const char *input_plugin, struct cw_flush *buf, flb_sds_t tag, - const char *data, size_t bytes) + const char *data, size_t bytes, struct flb_config *config) { int i = 0; size_t map_size; @@ -880,7 +883,8 @@ static int process_log_events(struct flb_cloudwatch *ctx, const char *input_plug found = FLB_TRUE; val = (kv+j)->val; ret = add_event(ctx, buf, stream, &val, - &log_event.timestamp); + &log_event.timestamp, + config); if (ret < 0 ) { goto error; } @@ -957,14 +961,14 @@ static int process_log_events(struct flb_cloudwatch *ctx, const char *input_plug goto error; } ret = add_event(ctx, buf, stream, &emf_payload, - &log_event.timestamp); + &log_event.timestamp, config); msgpack_unpacked_destroy(&mp_emf_result); msgpack_sbuffer_destroy(&mp_sbuf); } else { ret = add_event(ctx, buf, stream, &map, - &log_event.timestamp); + &log_event.timestamp, config); } if (ret < 0 ) { @@ -988,7 +992,7 @@ static int process_log_events(struct flb_cloudwatch *ctx, const char *input_plug static int process_metric_events(struct flb_cloudwatch *ctx, const char *input_plugin, struct cw_flush *buf, flb_sds_t tag, - const char *data, size_t bytes) + const char *data, size_t bytes, struct flb_config *config) { int i = 0; int ret; @@ -1027,7 +1031,7 @@ static int process_metric_events(struct flb_cloudwatch *ctx, const char *input_p flb_time_get(&tm); ret = add_event(ctx, buf, stream, &map, - &tm); + &tm, config); if (ret < 0 ) { goto cmt_error; @@ -1056,7 +1060,8 @@ static int process_metric_events(struct flb_cloudwatch *ctx, const char *input_p */ int process_and_send(struct flb_cloudwatch *ctx, const char *input_plugin, struct cw_flush *buf, flb_sds_t tag, - const char *data, size_t bytes, int event_type) + const char *data, size_t bytes, int event_type, + struct flb_config *config) { int ret; int i = 0; @@ -1064,12 +1069,14 @@ int process_and_send(struct flb_cloudwatch *ctx, const char *input_plugin, if (event_type == FLB_EVENT_TYPE_LOGS) { i = process_log_events(ctx, input_plugin, buf, tag, - data, bytes); + data, bytes, + config); } else if (event_type == FLB_EVENT_TYPE_METRICS) { i = process_metric_events(ctx, input_plugin, buf, tag, - data, bytes); + data, bytes, + config); } /* send any remaining events */ ret = send_log_events(ctx, buf); diff --git a/plugins/out_cloudwatch_logs/cloudwatch_api.h b/plugins/out_cloudwatch_logs/cloudwatch_api.h index 05abfff30a1..919a23e38dc 100644 --- a/plugins/out_cloudwatch_logs/cloudwatch_api.h +++ b/plugins/out_cloudwatch_logs/cloudwatch_api.h @@ -44,7 +44,8 @@ void cw_flush_destroy(struct cw_flush *buf); int process_and_send(struct flb_cloudwatch *ctx, const char *input_plugin, struct cw_flush *buf, flb_sds_t tag, - const char *data, size_t bytes, int event_type); + const char *data, size_t bytes, int event_type, + struct flb_config *config); int create_log_stream(struct flb_cloudwatch *ctx, struct log_stream *stream, int can_retry); struct log_stream *get_log_stream(struct flb_cloudwatch *ctx, flb_sds_t tag, const msgpack_object map); diff --git a/plugins/out_cloudwatch_logs/cloudwatch_logs.c b/plugins/out_cloudwatch_logs/cloudwatch_logs.c index c5e808ae141..e790dc6ca5f 100644 --- a/plugins/out_cloudwatch_logs/cloudwatch_logs.c +++ b/plugins/out_cloudwatch_logs/cloudwatch_logs.c @@ -446,7 +446,7 @@ static void cb_cloudwatch_flush(struct flb_event_chunk *event_chunk, } event_count = process_and_send(ctx, i_ins->p->name, buf, event_chunk->tag, event_chunk->data, event_chunk->size, - event_chunk->type); + event_chunk->type, config); if (event_count < 0) { flb_plg_error(ctx->ins, "Failed to send events"); cw_flush_destroy(buf); diff --git a/plugins/out_datadog/datadog.c b/plugins/out_datadog/datadog.c index e5c6621f880..a3a3d7e8e21 100644 --- a/plugins/out_datadog/datadog.c +++ b/plugins/out_datadog/datadog.c @@ -325,7 +325,8 @@ static int datadog_format(struct flb_config *config, } /* Convert from msgpack to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); msgpack_sbuffer_destroy(&mp_sbuf); if (!out_buf) { diff --git a/plugins/out_es/es.c b/plugins/out_es/es.c index 4e653e7e349..915a03f4dd6 100644 --- a/plugins/out_es/es.c +++ b/plugins/out_es/es.c @@ -557,7 +557,8 @@ static int elasticsearch_format(struct flb_config *config, } /* Convert msgpack to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size, + config->json_escape_unicode); msgpack_sbuffer_destroy(&tmp_sbuf); if (!out_buf) { flb_log_event_decoder_destroy(&log_decoder); diff --git a/plugins/out_file/file.c b/plugins/out_file/file.c index c7db85fc2d6..4c6eb301dfc 100644 --- a/plugins/out_file/file.c +++ b/plugins/out_file/file.c @@ -328,11 +328,11 @@ static int template_output(FILE *fp, struct flb_time *tm, msgpack_object *obj, } -static int plain_output(FILE *fp, msgpack_object *obj, size_t alloc_size) +static int plain_output(FILE *fp, msgpack_object *obj, size_t alloc_size, int escape_unicode) { char *buf; - buf = flb_msgpack_to_json_str(alloc_size, obj); + buf = flb_msgpack_to_json_str(alloc_size, obj, escape_unicode); if (buf) { fprintf(fp, "%s" NEWLINE, buf); @@ -616,7 +616,8 @@ static void cb_file_flush(struct flb_event_chunk *event_chunk, switch (ctx->format){ case FLB_OUT_FILE_FMT_JSON: - buf = flb_msgpack_to_json_str(alloc_size, log_event.body); + buf = flb_msgpack_to_json_str(alloc_size, log_event.body, + config->json_escape_unicode); if (buf) { fprintf(fp, "%s: [%"PRIu64".%09lu, %s]" NEWLINE, event_chunk->tag, @@ -648,7 +649,7 @@ static void cb_file_flush(struct flb_event_chunk *event_chunk, log_event.body, ctx); break; case FLB_OUT_FILE_FMT_PLAIN: - plain_output(fp, log_event.body, alloc_size); + plain_output(fp, log_event.body, alloc_size, config->json_escape_unicode); break; case FLB_OUT_FILE_FMT_TEMPLATE: diff --git a/plugins/out_http/http.c b/plugins/out_http/http.c index 869cd494449..d0c8f8d83c4 100644 --- a/plugins/out_http/http.c +++ b/plugins/out_http/http.c @@ -427,7 +427,8 @@ static int compose_payload_gelf(struct flb_out_http *ctx, static int compose_payload(struct flb_out_http *ctx, const void *in_body, size_t in_size, - void **out_body, size_t *out_size) + void **out_body, size_t *out_size, + struct flb_config *config) { flb_sds_t encoded; @@ -442,7 +443,8 @@ static int compose_payload(struct flb_out_http *ctx, in_size, ctx->out_format, ctx->json_date_format, - ctx->date_key); + ctx->date_key, + config->json_escape_unicode); if (encoded == NULL) { flb_plg_error(ctx->ins, "failed to convert json"); return FLB_ERROR; @@ -629,7 +631,7 @@ static void cb_http_flush(struct flb_event_chunk *event_chunk, } else { ret = compose_payload(ctx, event_chunk->data, event_chunk->size, - &out_body, &out_size); + &out_body, &out_size, config); if (ret != FLB_OK) { FLB_OUTPUT_RETURN(ret); } @@ -792,7 +794,7 @@ static int cb_http_format_test(struct flb_config *config, struct flb_out_http *ctx = plugin_context; int ret; - ret = compose_payload(ctx, data, bytes, out_data, out_size); + ret = compose_payload(ctx, data, bytes, out_data, out_size, config); if (ret != FLB_OK) { flb_error("ret=%d", ret); return -1; diff --git a/plugins/out_influxdb/influxdb.c b/plugins/out_influxdb/influxdb.c index 671dd5c16a6..2c191354b31 100644 --- a/plugins/out_influxdb/influxdb.c +++ b/plugins/out_influxdb/influxdb.c @@ -216,7 +216,8 @@ static int influxdb_format(struct flb_config *config, /* is this a string ? */ if (quote == FLB_TRUE) { ret = flb_utils_write_str_buf(val, val_len, - &str, &str_size); + &str, &str_size, + config->json_escape_unicode); if (ret == -1) { flb_errno(); goto error; diff --git a/plugins/out_kafka/kafka.c b/plugins/out_kafka/kafka.c index 09006bc80d8..dadd4725f74 100644 --- a/plugins/out_kafka/kafka.c +++ b/plugins/out_kafka/kafka.c @@ -284,7 +284,8 @@ int produce_message(struct flb_time *tm, msgpack_object *map, } if (ctx->format == FLB_KAFKA_FMT_JSON) { - s = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + s = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); if (!s) { flb_plg_error(ctx->ins, "error encoding to JSON"); msgpack_sbuffer_destroy(&mp_sbuf); diff --git a/plugins/out_kafka_rest/kafka.c b/plugins/out_kafka_rest/kafka.c index a9656a69a17..ef13d079fbc 100644 --- a/plugins/out_kafka_rest/kafka.c +++ b/plugins/out_kafka_rest/kafka.c @@ -94,7 +94,8 @@ static struct flb_config_map config_map[] = { static flb_sds_t kafka_rest_format(const void *data, size_t bytes, const char *tag, int tag_len, size_t *out_size, - struct flb_kafka_rest *ctx) + struct flb_kafka_rest *ctx, + struct flb_config *config) { int i; int len; @@ -211,7 +212,8 @@ static flb_sds_t kafka_rest_format(const void *data, size_t bytes, flb_log_event_decoder_destroy(&log_decoder); /* Convert to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); msgpack_sbuffer_destroy(&mp_sbuf); if (!out_buf) { return NULL; @@ -268,7 +270,7 @@ static void cb_kafka_flush(struct flb_event_chunk *event_chunk, /* Convert format */ js = kafka_rest_format(event_chunk->data, event_chunk->size, event_chunk->tag, flb_sds_len(event_chunk->tag), - &js_size, ctx); + &js_size, ctx, config); if (!js) { flb_upstream_conn_release(u_conn); FLB_OUTPUT_RETURN(FLB_ERROR); diff --git a/plugins/out_kinesis_firehose/firehose.c b/plugins/out_kinesis_firehose/firehose.c index d91c6f00a8a..cb473e1a417 100644 --- a/plugins/out_kinesis_firehose/firehose.c +++ b/plugins/out_kinesis_firehose/firehose.c @@ -339,7 +339,7 @@ static void cb_firehose_flush(struct flb_event_chunk *event_chunk, } ret = process_and_send_records(ctx, buf, - event_chunk->data, event_chunk->size); + event_chunk->data, event_chunk->size, config); if (ret < 0) { flb_plg_error(ctx->ins, "Failed to send records"); flush_destroy(buf); diff --git a/plugins/out_kinesis_firehose/firehose_api.c b/plugins/out_kinesis_firehose/firehose_api.c index d12e4299f5d..a87f2008d9f 100644 --- a/plugins/out_kinesis_firehose/firehose_api.c +++ b/plugins/out_kinesis_firehose/firehose_api.c @@ -152,7 +152,8 @@ static int end_put_payload(struct flb_firehose *ctx, struct flush *buf, * which means a send must occur */ static int process_event(struct flb_firehose *ctx, struct flush *buf, - const msgpack_object *obj, struct flb_time *tms) + const msgpack_object *obj, struct flb_time *tms, + struct flb_config *config) { size_t written = 0; int ret; @@ -170,8 +171,8 @@ static int process_event(struct flb_firehose *ctx, struct flush *buf, tmp_buf_ptr = buf->tmp_buf + buf->tmp_buf_offset; ret = flb_msgpack_to_json(tmp_buf_ptr, - buf->tmp_buf_size - buf->tmp_buf_offset, - obj); + buf->tmp_buf_size - buf->tmp_buf_offset, + obj, config->json_escape_unicode); if (ret <= 0) { /* * negative value means failure to write to buffer, @@ -431,7 +432,8 @@ static int send_log_events(struct flb_firehose *ctx, struct flush *buf) { * Processes the msgpack object, sends the current batch if needed */ static int add_event(struct flb_firehose *ctx, struct flush *buf, - const msgpack_object *obj, struct flb_time *tms) + const msgpack_object *obj, struct flb_time *tms, + struct flb_config *config) { int ret; struct firehose_event *event; @@ -445,7 +447,7 @@ static int add_event(struct flb_firehose *ctx, struct flush *buf, retry_add_event: retry_add = FLB_FALSE; - ret = process_event(ctx, buf, obj, tms); + ret = process_event(ctx, buf, obj, tms, config); if (ret < 0) { return -1; } @@ -510,7 +512,8 @@ static int add_event(struct flb_firehose *ctx, struct flush *buf, * return value is the number of events processed (number sent is stored in buf) */ int process_and_send_records(struct flb_firehose *ctx, struct flush *buf, - const char *data, size_t bytes) + const char *data, size_t bytes, + struct flb_config *config) { // size_t off = 0; int i = 0; @@ -573,7 +576,7 @@ int process_and_send_records(struct flb_firehose *ctx, struct flush *buf, if (strncmp(ctx->log_key, key_str, key_str_size) == 0) { found = FLB_TRUE; val = (kv+j)->val; - ret = add_event(ctx, buf, &val, &log_event.timestamp); + ret = add_event(ctx, buf, &val, &log_event.timestamp, config); if (ret < 0 ) { goto error; } @@ -591,7 +594,7 @@ int process_and_send_records(struct flb_firehose *ctx, struct flush *buf, continue; } - ret = add_event(ctx, buf, &map, &log_event.timestamp); + ret = add_event(ctx, buf, &map, &log_event.timestamp, config); if (ret < 0 ) { goto error; } diff --git a/plugins/out_kinesis_firehose/firehose_api.h b/plugins/out_kinesis_firehose/firehose_api.h index 9954d2c9eb3..fde963a563d 100644 --- a/plugins/out_kinesis_firehose/firehose_api.h +++ b/plugins/out_kinesis_firehose/firehose_api.h @@ -37,7 +37,8 @@ void flush_destroy(struct flush *buf); int process_and_send_records(struct flb_firehose *ctx, struct flush *buf, - const char *data, size_t bytes); + const char *data, size_t bytes, + struct flb_config *config); int put_record_batch(struct flb_firehose *ctx, struct flush *buf, size_t payload_size, int num_records); diff --git a/plugins/out_kinesis_streams/kinesis.c b/plugins/out_kinesis_streams/kinesis.c index a225f6007f7..666e9f9882d 100644 --- a/plugins/out_kinesis_streams/kinesis.c +++ b/plugins/out_kinesis_streams/kinesis.c @@ -362,7 +362,8 @@ static void cb_kinesis_flush(struct flb_event_chunk *event_chunk, ret = process_and_send_to_kinesis(ctx, buf, event_chunk->data, - event_chunk->size); + event_chunk->size, + config); if (ret < 0) { flb_plg_error(ctx->ins, "Failed to send records to kinesis"); kinesis_flush_destroy(buf); diff --git a/plugins/out_kinesis_streams/kinesis_api.c b/plugins/out_kinesis_streams/kinesis_api.c index ffd1a82e927..3f6d0f939d8 100644 --- a/plugins/out_kinesis_streams/kinesis_api.c +++ b/plugins/out_kinesis_streams/kinesis_api.c @@ -213,7 +213,8 @@ static int end_put_payload(struct flb_kinesis *ctx, struct flush *buf, * which means a send must occur */ static int process_event(struct flb_kinesis *ctx, struct flush *buf, - const msgpack_object *obj, struct flb_time *tms) + const msgpack_object *obj, struct flb_time *tms, + struct flb_config *config) { size_t written = 0; int ret; @@ -230,8 +231,8 @@ static int process_event(struct flb_kinesis *ctx, struct flush *buf, tmp_buf_ptr = buf->tmp_buf + buf->tmp_buf_offset; ret = flb_msgpack_to_json(tmp_buf_ptr, - buf->tmp_buf_size - buf->tmp_buf_offset, - obj); + buf->tmp_buf_size - buf->tmp_buf_offset, + obj, config->json_escape_unicode); if (ret <= 0) { /* * negative value means failure to write to buffer, @@ -466,7 +467,8 @@ static int send_log_events(struct flb_kinesis *ctx, struct flush *buf) { * Processes the msgpack object, sends the current batch if needed */ static int add_event(struct flb_kinesis *ctx, struct flush *buf, - const msgpack_object *obj, struct flb_time *tms) + const msgpack_object *obj, struct flb_time *tms, + struct flb_config *config) { int ret; struct kinesis_event *event; @@ -480,7 +482,7 @@ static int add_event(struct flb_kinesis *ctx, struct flush *buf, retry_add_event: retry_add = FLB_FALSE; - ret = process_event(ctx, buf, obj, tms); + ret = process_event(ctx, buf, obj, tms, config); if (ret < 0) { return -1; } @@ -545,7 +547,8 @@ static int add_event(struct flb_kinesis *ctx, struct flush *buf, * return value is the number of events processed (number sent is stored in buf) */ int process_and_send_to_kinesis(struct flb_kinesis *ctx, struct flush *buf, - const char *data, size_t bytes) + const char *data, size_t bytes, + struct flb_config *config) { int i = 0; size_t map_size; @@ -602,7 +605,7 @@ int process_and_send_to_kinesis(struct flb_kinesis *ctx, struct flush *buf, if (strncmp(ctx->log_key, key_str, key_str_size) == 0) { found = FLB_TRUE; val = (kv+j)->val; - ret = add_event(ctx, buf, &val, &log_event.timestamp); + ret = add_event(ctx, buf, &val, &log_event.timestamp, config); if (ret < 0 ) { goto error; } @@ -620,7 +623,7 @@ int process_and_send_to_kinesis(struct flb_kinesis *ctx, struct flush *buf, continue; } - ret = add_event(ctx, buf, &map, &log_event.timestamp); + ret = add_event(ctx, buf, &map, &log_event.timestamp, config); if (ret < 0 ) { goto error; } diff --git a/plugins/out_kinesis_streams/kinesis_api.h b/plugins/out_kinesis_streams/kinesis_api.h index 2ff8e08e9a4..bbc30efa96a 100644 --- a/plugins/out_kinesis_streams/kinesis_api.h +++ b/plugins/out_kinesis_streams/kinesis_api.h @@ -36,7 +36,8 @@ void kinesis_flush_destroy(struct flush *buf); int process_and_send_to_kinesis(struct flb_kinesis *ctx, struct flush *buf, - const char *data, size_t bytes); + const char *data, size_t bytes, + struct flb_config *config); int put_records(struct flb_kinesis *ctx, struct flush *buf, size_t payload_size, int num_records); diff --git a/plugins/out_lib/out_lib.c b/plugins/out_lib/out_lib.c index a654fba31e3..6cac599184e 100644 --- a/plugins/out_lib/out_lib.c +++ b/plugins/out_lib/out_lib.c @@ -167,7 +167,8 @@ static void out_lib_flush(struct flb_event_chunk *event_chunk, #ifdef FLB_HAVE_METRICS if (event_chunk->type == FLB_EVENT_TYPE_METRICS) { alloc_size = (off - last_off) + 4096; - buf = flb_msgpack_to_json_str(alloc_size, &result.data); + buf = flb_msgpack_to_json_str(alloc_size, &result.data, + config->json_escape_unicode); if (buf == NULL) { msgpack_unpacked_destroy(&result); FLB_OUTPUT_RETURN(FLB_ERROR); @@ -181,7 +182,8 @@ static void out_lib_flush(struct flb_event_chunk *event_chunk, alloc_size = (off - last_off) + 128; flb_time_pop_from_msgpack(&tm, &result, &obj); - buf = flb_msgpack_to_json_str(alloc_size, obj); + buf = flb_msgpack_to_json_str(alloc_size, obj, + config->json_escape_unicode); if (!buf) { msgpack_unpacked_destroy(&result); FLB_OUTPUT_RETURN(FLB_ERROR); diff --git a/plugins/out_logdna/logdna.c b/plugins/out_logdna/logdna.c index bc834c6017d..f2b987e55ce 100644 --- a/plugins/out_logdna/logdna.c +++ b/plugins/out_logdna/logdna.c @@ -131,7 +131,8 @@ static int record_append_primary_keys(struct flb_logdna *ctx, static flb_sds_t logdna_compose_payload(struct flb_logdna *ctx, const void *data, size_t bytes, - const char *tag, int tag_len) + const char *tag, int tag_len, + struct flb_config *config) { int ret; int len; @@ -192,7 +193,7 @@ static flb_sds_t logdna_compose_payload(struct flb_logdna *ctx, msgpack_pack_str(&mp_pck, 4); msgpack_pack_str_body(&mp_pck, "line", 4); - line_json = flb_msgpack_to_json_str(1024, log_event.body); + line_json = flb_msgpack_to_json_str(1024, log_event.body, config->json_escape_unicode); len = strlen(line_json); msgpack_pack_str(&mp_pck, len); msgpack_pack_str_body(&mp_pck, line_json, len); @@ -204,7 +205,8 @@ static flb_sds_t logdna_compose_payload(struct flb_logdna *ctx, flb_log_event_decoder_destroy(&log_decoder); - json = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + json = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); msgpack_sbuffer_destroy(&mp_sbuf); return json; @@ -382,7 +384,8 @@ static void cb_logdna_flush(struct flb_event_chunk *event_chunk, event_chunk->data, event_chunk->size, event_chunk->tag, - flb_sds_len(event_chunk->tag)); + flb_sds_len(event_chunk->tag), + config); if (!payload) { flb_plg_error(ctx->ins, "cannot compose request payload"); FLB_OUTPUT_RETURN(FLB_RETRY); diff --git a/plugins/out_loki/loki.c b/plugins/out_loki/loki.c index ee7a4313a89..bdec7fd08e8 100644 --- a/plugins/out_loki/loki.c +++ b/plugins/out_loki/loki.c @@ -477,7 +477,8 @@ static void pack_maps(struct flb_loki *ctx, char *tag, int tag_len, msgpack_object *map, struct flb_mp_map_header *mh, - struct mk_list *list) + struct mk_list *list, + struct flb_config *config) { struct mk_list *head; struct flb_loki_kv *kv; @@ -539,7 +540,8 @@ static void pack_maps(struct flb_loki *ctx, */ else { accessed_map_val_json = flb_msgpack_to_json_str(1024, - &accessed_map_kv.val); + &accessed_map_kv.val, + config->json_escape_unicode); if (accessed_map_val_json) { msgpack_pack_str_with_body(mp_pck, accessed_map_val_json, strlen(accessed_map_val_json)); @@ -556,14 +558,16 @@ static void pack_maps(struct flb_loki *ctx, static flb_sds_t pack_structured_metadata(struct flb_loki *ctx, msgpack_packer *mp_pck, char *tag, int tag_len, - msgpack_object *map) + msgpack_object *map, + struct flb_config *config) { struct flb_mp_map_header mh; /* Initialize dynamic map header */ flb_mp_map_header_init(&mh, mp_pck); if (ctx->structured_metadata_map_keys) { pack_maps(ctx, mp_pck, tag, tag_len, map, &mh, - &ctx->structured_metadata_map_keys_list); + &ctx->structured_metadata_map_keys_list, + config); } /* * explicit structured_metadata entries override @@ -1415,7 +1419,8 @@ static int get_tenant_id_from_record(struct flb_loki *ctx, msgpack_object *map, static int pack_record(struct flb_loki *ctx, msgpack_packer *mp_pck, msgpack_object *rec, flb_sds_t *dynamic_tenant_id, - struct flb_mp_accessor *remove_mpa) + struct flb_mp_accessor *remove_mpa, + struct flb_config *config) { int i; int skip = 0; @@ -1506,7 +1511,7 @@ static int pack_record(struct flb_loki *ctx, } if (ctx->out_line_format == FLB_LOKI_FMT_JSON) { - line = flb_msgpack_to_json_str(size_hint, rec); + line = flb_msgpack_to_json_str(size_hint, rec, config->json_escape_unicode); if (!line) { if (tmp_sbuf_data) { flb_free(tmp_sbuf_data); @@ -1635,7 +1640,8 @@ static flb_sds_t loki_compose_payload(struct flb_loki *ctx, char *tag, int tag_len, const void *data, size_t bytes, flb_sds_t *dynamic_tenant_id, - struct flb_mp_accessor *remove_mpa) + struct flb_mp_accessor *remove_mpa, + struct flb_config *config) { // int mp_ok = MSGPACK_UNPACK_SUCCESS; // size_t off = 0; @@ -1726,9 +1732,9 @@ static flb_sds_t loki_compose_payload(struct flb_loki *ctx, /* Append the timestamp */ pack_timestamp(&mp_pck, &log_event.timestamp); - pack_record(ctx, &mp_pck, log_event.body, dynamic_tenant_id, remove_mpa); + pack_record(ctx, &mp_pck, log_event.body, dynamic_tenant_id, remove_mpa, config); if (ctx->structured_metadata || ctx->structured_metadata_map_keys) { - pack_structured_metadata(ctx, &mp_pck, tag, tag_len, NULL); + pack_structured_metadata(ctx, &mp_pck, tag, tag_len, NULL, config); } } } @@ -1763,16 +1769,17 @@ static flb_sds_t loki_compose_payload(struct flb_loki *ctx, /* Append the timestamp */ pack_timestamp(&mp_pck, &log_event.timestamp); - pack_record(ctx, &mp_pck, log_event.body, dynamic_tenant_id, remove_mpa); + pack_record(ctx, &mp_pck, log_event.body, dynamic_tenant_id, remove_mpa, config); if (ctx->structured_metadata || ctx->structured_metadata_map_keys) { - pack_structured_metadata(ctx, &mp_pck, tag, tag_len, log_event.body); + pack_structured_metadata(ctx, &mp_pck, tag, tag_len, log_event.body, config); } } } flb_log_event_decoder_destroy(&log_decoder); - json = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + json = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); msgpack_sbuffer_destroy(&mp_sbuf); @@ -1856,7 +1863,8 @@ static void cb_loki_flush(struct flb_event_chunk *event_chunk, flb_sds_len(event_chunk->tag), event_chunk->data, event_chunk->size, &dynamic_tenant_id->value, - remove_mpa_entry->mpa); + remove_mpa_entry->mpa, + config); if (!payload) { flb_plg_error(ctx->ins, "cannot compose request payload"); @@ -2128,7 +2136,7 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct flb_loki, structured_metadata), "optional structured metadata fields for API requests." }, - + { FLB_CONFIG_MAP_CLIST, "structured_metadata_map_keys", NULL, 0, FLB_TRUE, offsetof(struct flb_loki, structured_metadata_map_keys), @@ -2241,7 +2249,8 @@ static int cb_loki_format_test(struct flb_config *config, payload = loki_compose_payload(ctx, total_records, (char *) tag, tag_len, data, bytes, &dynamic_tenant_id, - ctx->remove_mpa); + ctx->remove_mpa, + config); if (payload == NULL) { if (dynamic_tenant_id != NULL) { flb_sds_destroy(dynamic_tenant_id); diff --git a/plugins/out_nats/nats.c b/plugins/out_nats/nats.c index d2ed1b3caed..e74f1ca7c89 100644 --- a/plugins/out_nats/nats.c +++ b/plugins/out_nats/nats.c @@ -81,7 +81,8 @@ static int cb_nats_init(struct flb_output_instance *ins, struct flb_config *conf static int msgpack_to_json(struct flb_out_nats_config *ctx, const void *data, size_t bytes, const char *tag, int tag_len, - char **out_json, size_t *out_size) + char **out_json, size_t *out_size, + struct flb_config *config) { int i; int map_size; @@ -138,7 +139,7 @@ static int msgpack_to_json(struct flb_out_nats_config *ctx, flb_log_event_decoder_destroy(&log_decoder); - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, config->json_escape_unicode); msgpack_sbuffer_destroy(&mp_sbuf); if (!out_buf) { @@ -186,7 +187,7 @@ static void cb_nats_flush(struct flb_event_chunk *event_chunk, ret = msgpack_to_json(ctx, event_chunk->data, event_chunk->size, event_chunk->tag, flb_sds_len(event_chunk->tag), - &json_msg, &json_len); + &json_msg, &json_len, config); if (ret == -1) { flb_upstream_conn_release(u_conn); FLB_OUTPUT_RETURN(FLB_ERROR); diff --git a/plugins/out_nrlogs/newrelic.c b/plugins/out_nrlogs/newrelic.c index f50b3e048ad..e23ca90c761 100644 --- a/plugins/out_nrlogs/newrelic.c +++ b/plugins/out_nrlogs/newrelic.c @@ -142,7 +142,8 @@ static int package_record(struct flb_time *ts, msgpack_object *map, } static flb_sds_t newrelic_compose_payload(struct flb_newrelic *ctx, - const void *data, size_t bytes) + const void *data, size_t bytes, + struct flb_config *config) { int total_records; flb_sds_t json; @@ -232,7 +233,8 @@ static flb_sds_t newrelic_compose_payload(struct flb_newrelic *ctx, flb_log_event_decoder_destroy(&log_decoder); - json = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + json = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); msgpack_sbuffer_destroy(&mp_sbuf); @@ -368,7 +370,8 @@ static void cb_newrelic_flush(struct flb_event_chunk *event_chunk, /* Format the data to the expected Newrelic Payload */ payload = newrelic_compose_payload(ctx, - event_chunk->data, event_chunk->size); + event_chunk->data, event_chunk->size, + config); if (!payload) { flb_plg_error(ctx->ins, "cannot compose request payload"); FLB_OUTPUT_RETURN(FLB_RETRY); diff --git a/plugins/out_null/null.c b/plugins/out_null/null.c index bf29645de98..b637bcfd9b6 100644 --- a/plugins/out_null/null.c +++ b/plugins/out_null/null.c @@ -123,7 +123,8 @@ static void cb_null_flush(struct flb_event_chunk *event_chunk, event_chunk->size, ctx->out_format, ctx->json_date_format, - ctx->date_key); + ctx->date_key, + config->json_escape_unicode); flb_sds_destroy(json); } diff --git a/plugins/out_opensearch/opensearch.c b/plugins/out_opensearch/opensearch.c index 856b00fa96e..3f52a7abd1f 100644 --- a/plugins/out_opensearch/opensearch.c +++ b/plugins/out_opensearch/opensearch.c @@ -587,7 +587,8 @@ static int opensearch_format(struct flb_config *config, } /* Convert msgpack to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size, + config->json_escape_unicode); msgpack_sbuffer_destroy(&tmp_sbuf); if (!out_buf) { flb_log_event_decoder_destroy(&log_decoder); @@ -886,7 +887,8 @@ static void cb_opensearch_flush(struct flb_event_chunk *event_chunk, /* Convert format */ if (event_chunk->type == FLB_EVENT_TYPE_TRACES) { - pack = flb_msgpack_raw_to_json_sds(event_chunk->data, event_chunk->size); + pack = flb_msgpack_raw_to_json_sds(event_chunk->data, event_chunk->size, + config->json_escape_unicode); if (pack) { ret = 0; diff --git a/plugins/out_oracle_log_analytics/oci_logan.c b/plugins/out_oracle_log_analytics/oci_logan.c index 19e4529ddf3..42ff71938aa 100644 --- a/plugins/out_oracle_log_analytics/oci_logan.c +++ b/plugins/out_oracle_log_analytics/oci_logan.c @@ -1161,7 +1161,8 @@ static int total_flush(struct flb_event_chunk *event_chunk, goto clean_up; } - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); msgpack_sbuffer_destroy(&mp_sbuf); flb_log_event_decoder_destroy(&log_decoder); diff --git a/plugins/out_pgsql/pgsql.c b/plugins/out_pgsql/pgsql.c index e6b24451cab..69a5ebdd14a 100644 --- a/plugins/out_pgsql/pgsql.c +++ b/plugins/out_pgsql/pgsql.c @@ -271,7 +271,8 @@ static void cb_pgsql_flush(struct flb_event_chunk *event_chunk, event_chunk->size, FLB_PACK_JSON_FORMAT_JSON, FLB_PACK_JSON_DATE_DOUBLE, - ctx->timestamp_key); + ctx->timestamp_key, + config->json_escape_unicode); if (json == NULL) { flb_errno(); flb_plg_error(ctx->ins, diff --git a/plugins/out_s3/s3.c b/plugins/out_s3/s3.c index 3b1edafbdab..12c6528b095 100644 --- a/plugins/out_s3/s3.c +++ b/plugins/out_s3/s3.c @@ -1851,7 +1851,7 @@ static void cb_s3_upload(struct flb_config *config, void *data) } static flb_sds_t flb_pack_msgpack_extract_log_key(void *out_context, const char *data, - uint64_t bytes) + uint64_t bytes, struct flb_config *config) { int i; int records = 0; @@ -1958,7 +1958,8 @@ static flb_sds_t flb_pack_msgpack_extract_log_key(void *out_context, const char } else { ret = flb_msgpack_to_json(val_buf + val_offset, - msgpack_size - val_offset, &val); + msgpack_size - val_offset, &val, + config->json_escape_unicode); if (ret < 0) { break; } @@ -2106,14 +2107,16 @@ static void cb_s3_flush(struct flb_event_chunk *event_chunk, if (ctx->log_key) { chunk = flb_pack_msgpack_extract_log_key(ctx, event_chunk->data, - event_chunk->size); + event_chunk->size, + config); } else { chunk = flb_pack_msgpack_to_json_format(event_chunk->data, event_chunk->size, FLB_PACK_JSON_FORMAT_LINES, ctx->json_date_format, - ctx->date_key); + ctx->date_key, + config->json_escape_unicode); } if (chunk == NULL) { flb_plg_error(ctx->ins, "Could not marshal msgpack to output string"); diff --git a/plugins/out_skywalking/skywalking.c b/plugins/out_skywalking/skywalking.c index dd1f8fa0539..3598bd5ff09 100644 --- a/plugins/out_skywalking/skywalking.c +++ b/plugins/out_skywalking/skywalking.c @@ -152,7 +152,8 @@ static void sw_msgpack_pack_kv_int64_t(msgpack_packer* pk, const char* key, } static void sw_msgpack_pack_log_body(msgpack_packer* pk, - msgpack_object* obj, size_t obj_size) + msgpack_object* obj, size_t obj_size, + struct flb_config *config) { int i, j = 0; int log_entry_num = 0; @@ -196,7 +197,8 @@ static void sw_msgpack_pack_log_body(msgpack_packer* pk, value.via.str.ptr, value.via.str.size); } - out_body_str = flb_msgpack_raw_to_json_sds(sbuf.data, sbuf.size); + out_body_str = flb_msgpack_raw_to_json_sds(sbuf.data, sbuf.size, + config->json_escape_unicode); if (!out_body_str) { msgpack_sbuffer_destroy(&sbuf); flb_free(valid_log_entry); @@ -225,7 +227,7 @@ static void sw_msgpack_pack_log_body(msgpack_packer* pk, } static int sw_format(struct flb_output_sw* ctx, const void *data, size_t bytes, - void** buf, size_t* buf_len) + void** buf, size_t* buf_len, struct flb_config *config) { int ret = 0; int chunk_size = 0; @@ -270,10 +272,10 @@ static int sw_format(struct flb_output_sw* ctx, const void *data, size_t bytes, flb_sds_len(ctx->svc_name)); sw_msgpack_pack_kv_str(&pk, "serviceInstance", 15, ctx->svc_inst_name, flb_sds_len(ctx->svc_inst_name)); - sw_msgpack_pack_log_body(&pk, &map, map_size); + sw_msgpack_pack_log_body(&pk, &map, map_size, config); } - out_str = flb_msgpack_raw_to_json_sds(sbuf.data, sbuf.size); + out_str = flb_msgpack_raw_to_json_sds(sbuf.data, sbuf.size, config->json_escape_unicode); if (!out_str) { ret = -1; goto done; @@ -323,7 +325,7 @@ static void cb_sw_flush(struct flb_event_chunk *event_chunk, tmp_ret = sw_format(ctx, event_chunk->data, event_chunk->size, - &buf, &buf_len); + &buf, &buf_len, config); if (tmp_ret != 0) { flb_plg_error(ctx->ins, "failed to create buffer"); FLB_OUTPUT_RETURN(FLB_RETRY); diff --git a/plugins/out_slack/slack.c b/plugins/out_slack/slack.c index 969644c2122..286fe078d85 100644 --- a/plugins/out_slack/slack.c +++ b/plugins/out_slack/slack.c @@ -230,7 +230,7 @@ static void cb_slack_flush(struct flb_event_chunk *event_chunk, flb_sds_destroy(json); /* Re-format mspgack as JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, config->json_escape_unicode); if (!out_buf) { msgpack_sbuffer_destroy(&mp_sbuf); FLB_OUTPUT_RETURN(FLB_RETRY); diff --git a/plugins/out_splunk/splunk.c b/plugins/out_splunk/splunk.c index 970eaa5d8d1..c2531d7b240 100644 --- a/plugins/out_splunk/splunk.c +++ b/plugins/out_splunk/splunk.c @@ -637,7 +637,7 @@ static flb_sds_t get_metadata_auth_header(struct flb_splunk *ctx) static inline int splunk_format(const void *in_buf, size_t in_bytes, char *tag, int tag_len, char **out_buf, size_t *out_size, - struct flb_splunk *ctx) + struct flb_splunk *ctx, struct flb_config *config) { int ret; char *err; @@ -726,7 +726,7 @@ static inline int splunk_format(const void *in_buf, size_t in_bytes, /* Validate packaging */ if (ret != 0) { /* Format invalid record */ - err = flb_msgpack_to_json_str(2048, &map); + err = flb_msgpack_to_json_str(2048, &map, config->json_escape_unicode); if (err) { /* Print error and continue processing other records */ flb_plg_warn(ctx->ins, "could not process or pack record: %s", err); @@ -737,7 +737,8 @@ static inline int splunk_format(const void *in_buf, size_t in_bytes, } /* Format as JSON */ - record = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + record = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); if (!record) { flb_errno(); msgpack_sbuffer_destroy(&mp_sbuf); @@ -898,7 +899,8 @@ static void cb_splunk_flush(struct flb_event_chunk *event_chunk, event_chunk->size, (char *) event_chunk->tag, flb_sds_len(event_chunk->tag), - &buf_data, &buf_size, ctx); + &buf_data, &buf_size, ctx, + config); } if (ret == -1) { @@ -1183,7 +1185,7 @@ static int cb_splunk_format_test(struct flb_config *config, struct flb_splunk *ctx = plugin_context; return splunk_format(data, bytes, (char *) tag, tag_len, - (char**) out_data, out_size,ctx); + (char**) out_data, out_size, ctx, config); } struct flb_output_plugin out_splunk_plugin = { diff --git a/plugins/out_stackdriver/stackdriver.c b/plugins/out_stackdriver/stackdriver.c index 4e1395f2d36..a322e8eca39 100644 --- a/plugins/out_stackdriver/stackdriver.c +++ b/plugins/out_stackdriver/stackdriver.c @@ -1697,7 +1697,8 @@ static int pack_payload(int insert_id_extracted, static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx, int total_records, const char *tag, int tag_len, - const void *data, size_t bytes) + const void *data, size_t bytes, + struct flb_config *config) { int len; int ret; @@ -2576,7 +2577,8 @@ static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx, flb_log_event_decoder_destroy(&log_decoder); /* Convert from msgpack to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); msgpack_sbuffer_destroy(&mp_sbuf); if (!out_buf) { @@ -2604,7 +2606,7 @@ static int stackdriver_format_test(struct flb_config *config, total_records = flb_mp_count(data, bytes); payload = stackdriver_format(ctx, total_records, - (char *) tag, tag_len, data, bytes); + (char *) tag, tag_len, data, bytes, config); if (payload == NULL) { return -1; } @@ -2877,7 +2879,8 @@ static void cb_stackdriver_flush(struct flb_event_chunk *event_chunk, payload_buf = stackdriver_format(ctx, event_chunk->total_events, event_chunk->tag, flb_sds_len(event_chunk->tag), - event_chunk->data, event_chunk->size); + event_chunk->data, event_chunk->size, + config); if (!payload_buf) { #ifdef FLB_HAVE_METRICS cmt_counter_inc(ctx->cmt_failed_requests, diff --git a/plugins/out_stdout/stdout.c b/plugins/out_stdout/stdout.c index f73475476f1..11dc2c6d5a5 100644 --- a/plugins/out_stdout/stdout.c +++ b/plugins/out_stdout/stdout.c @@ -257,7 +257,8 @@ static void cb_stdout_flush(struct flb_event_chunk *event_chunk, event_chunk->size, ctx->out_format, ctx->json_date_format, - ctx->date_key); + ctx->date_key, + config->json_escape_unicode); write(STDOUT_FILENO, json, flb_sds_len(json)); flb_sds_destroy(json); diff --git a/plugins/out_tcp/tcp.c b/plugins/out_tcp/tcp.c index 2ea38aa0c26..7e90d025e1e 100644 --- a/plugins/out_tcp/tcp.c +++ b/plugins/out_tcp/tcp.c @@ -55,7 +55,8 @@ static int cb_tcp_init(struct flb_output_instance *ins, static int compose_payload(struct flb_out_tcp *ctx, const char *tag, int tag_len, const void *in_data, size_t in_size, - void **out_payload, size_t *out_size) + void **out_payload, size_t *out_size, + struct flb_config *config) { int ret; flb_sds_t buf = NULL; @@ -126,7 +127,8 @@ static int compose_payload(struct flb_out_tcp *ctx, in_size, ctx->out_format, ctx->json_date_format, - ctx->date_key); + ctx->date_key, + config->json_escape_unicode); if (!json) { flb_plg_error(ctx->ins, "error formatting JSON payload"); return FLB_ERROR; @@ -164,7 +166,8 @@ static void cb_tcp_flush(struct flb_event_chunk *event_chunk, ret = compose_payload(ctx, event_chunk->tag, flb_sds_len(event_chunk->tag), event_chunk->data, event_chunk->size, - &out_payload, &out_size); + &out_payload, &out_size, + config); if (ret != FLB_OK) { flb_upstream_conn_release(u_conn); return FLB_OUTPUT_RETURN(ret); @@ -244,7 +247,7 @@ static int cb_tcp_format_test(struct flb_config *config, struct flb_out_tcp *ctx = plugin_context; int ret; - ret = compose_payload(ctx, tag, tag_len, data, bytes, out_data, out_size); + ret = compose_payload(ctx, tag, tag_len, data, bytes, out_data, out_size, config); if (ret != FLB_OK) { flb_error("ret=%d", ret); return -1; diff --git a/plugins/out_udp/udp.c b/plugins/out_udp/udp.c index 4ff63122622..2e854100434 100644 --- a/plugins/out_udp/udp.c +++ b/plugins/out_udp/udp.c @@ -127,7 +127,8 @@ static int deliver_chunks_raw(struct flb_out_udp *ctx, static int deliver_chunks_json(struct flb_out_udp *ctx, const char *tag, int tag_len, - const void *in_data, size_t in_size) + const void *in_data, size_t in_size, + struct flb_config *config) { int ret; size_t off = 0; @@ -158,7 +159,8 @@ static int deliver_chunks_json(struct flb_out_udp *ctx, off - previous_offset, ctx->out_format, ctx->json_date_format, - ctx->date_key); + ctx->date_key, + config->json_escape_unicode); if (!json) { flb_plg_error(ctx->ins, "error formatting JSON payload"); @@ -291,7 +293,8 @@ static void cb_udp_flush(struct flb_event_chunk *event_chunk, event_chunk->tag, flb_sds_len(event_chunk->tag), event_chunk->data, - event_chunk->size); + event_chunk->size, + config); } return FLB_OUTPUT_RETURN(ret); diff --git a/plugins/out_vivo_exporter/vivo.c b/plugins/out_vivo_exporter/vivo.c index c46b9ef1d9c..9a6289bc42d 100644 --- a/plugins/out_vivo_exporter/vivo.c +++ b/plugins/out_vivo_exporter/vivo.c @@ -27,7 +27,7 @@ #include "vivo_http.h" #include "vivo_stream.h" -static flb_sds_t format_logs(struct flb_event_chunk *event_chunk) +static flb_sds_t format_logs(struct flb_event_chunk *event_chunk, struct flb_config *config) { struct flb_log_event_decoder log_decoder; struct flb_log_event log_event; @@ -102,7 +102,8 @@ static flb_sds_t format_logs(struct flb_event_chunk *event_chunk) } /* Concatenate by using break lines */ - out_js = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size); + out_js = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size, + config->json_escape_unicode); if (!out_js) { flb_sds_destroy(out_buf); msgpack_sbuffer_destroy(&tmp_sbuf); @@ -130,14 +131,15 @@ static flb_sds_t format_logs(struct flb_event_chunk *event_chunk) } static int logs_event_chunk_append(struct vivo_exporter *ctx, - struct flb_event_chunk *event_chunk) + struct flb_event_chunk *event_chunk, + struct flb_config *config) { size_t len; flb_sds_t json; struct vivo_stream_entry *entry; - json = format_logs(event_chunk); + json = format_logs(event_chunk, config); if (!json) { flb_plg_error(ctx->ins, "cannot convert logs chunk to JSON"); return -1; @@ -159,14 +161,16 @@ static int logs_event_chunk_append(struct vivo_exporter *ctx, static int metrics_traces_event_chunk_append(struct vivo_exporter *ctx, struct vivo_stream *vs, - struct flb_event_chunk *event_chunk) + struct flb_event_chunk *event_chunk, + struct flb_config *config) { size_t len; flb_sds_t json; struct vivo_stream_entry *entry; /* Convert msgpack to readable JSON format */ - json = flb_msgpack_raw_to_json_sds(event_chunk->data, event_chunk->size); + json = flb_msgpack_raw_to_json_sds(event_chunk->data, event_chunk->size, + config->json_escape_unicode); if (!json) { flb_plg_error(ctx->ins, "cannot convert metrics chunk to JSON"); return -1; @@ -264,14 +268,14 @@ static void cb_vivo_flush(struct flb_event_chunk *event_chunk, #ifdef FLB_HAVE_METRICS if (event_chunk->type == FLB_EVENT_TYPE_METRICS) { - ret = metrics_traces_event_chunk_append(ctx, ctx->stream_metrics, event_chunk); + ret = metrics_traces_event_chunk_append(ctx, ctx->stream_metrics, event_chunk, config); } #endif if (event_chunk->type == FLB_EVENT_TYPE_LOGS) { - ret = logs_event_chunk_append(ctx, event_chunk); + ret = logs_event_chunk_append(ctx, event_chunk, config); } else if (event_chunk->type == FLB_EVENT_TYPE_TRACES) { - ret = metrics_traces_event_chunk_append(ctx, ctx->stream_traces, event_chunk); + ret = metrics_traces_event_chunk_append(ctx, ctx->stream_traces, event_chunk, config); } if (ret == 0) { diff --git a/plugins/out_websocket/websocket.c b/plugins/out_websocket/websocket.c index 52865e21a22..b3ffa784915 100644 --- a/plugins/out_websocket/websocket.c +++ b/plugins/out_websocket/websocket.c @@ -254,7 +254,8 @@ static void cb_ws_flush(struct flb_event_chunk *event_chunk, event_chunk->size, ctx->out_format, ctx->json_date_format, - ctx->json_date_key); + ctx->json_date_key, + config->json_escape_unicode); if (!json) { flb_error("[out_ws] error formatting JSON payload"); diff --git a/plugins/processor_content_modifier/cm_utils.c b/plugins/processor_content_modifier/cm_utils.c index e8d3f3e47a3..da68eb98a69 100644 --- a/plugins/processor_content_modifier/cm_utils.c +++ b/plugins/processor_content_modifier/cm_utils.c @@ -150,7 +150,8 @@ cfl_sds_t cm_utils_variant_convert_to_json(struct cfl_variant *value) mpack_writer_destroy(&writer); - json_result = flb_msgpack_raw_to_json_sds(data, size); + /* Using JSON escape here to keep backward compatibility */ + json_result = flb_msgpack_raw_to_json_sds(data, size, FLB_TRUE); MPACK_FREE(data); return json_result; diff --git a/src/flb_config.c b/src/flb_config.c index 407464b4942..9703c94a838 100644 --- a/src/flb_config.c +++ b/src/flb_config.c @@ -145,8 +145,8 @@ struct flb_service_config service_configs[] = { {FLB_CONF_STORAGE_BL_MEM_LIMIT, FLB_CONF_TYPE_STR, offsetof(struct flb_config, storage_bl_mem_limit)}, - {FLB_CONF_STORAGE_BL_FLUSH_ON_SHUTDOWN, - FLB_CONF_TYPE_BOOL, + {FLB_CONF_STORAGE_BL_FLUSH_ON_SHUTDOWN, + FLB_CONF_TYPE_BOOL, offsetof(struct flb_config, storage_bl_flush_on_shutdown)}, {FLB_CONF_STORAGE_MAX_CHUNKS_UP, FLB_CONF_TYPE_INT, @@ -177,6 +177,11 @@ struct flb_service_config service_configs[] = { FLB_CONF_TYPE_INT, offsetof(struct flb_config, sched_base)}, + /* Escape UNicode inside of JSON */ + {FLB_CONF_UNICODE_STR_JSON_ESCAPE, + FLB_CONF_TYPE_INT, + offsetof(struct flb_config, json_escape_unicode)}, + #ifdef FLB_HAVE_STREAM_PROCESSOR {FLB_CONF_STR_STREAMS_FILE, FLB_CONF_TYPE_STR, @@ -301,6 +306,7 @@ struct flb_config *flb_config_init() config->storage_bl_flush_on_shutdown = FLB_FALSE; config->sched_cap = FLB_SCHED_CAP; config->sched_base = FLB_SCHED_BASE; + config->json_escape_unicode = FLB_TRUE; /* reload */ config->ensure_thread_safety_on_hot_reloading = FLB_TRUE; diff --git a/src/flb_help.c b/src/flb_help.c index ea292591d2a..234b121885a 100644 --- a/src/flb_help.c +++ b/src/flb_help.c @@ -825,7 +825,7 @@ flb_sds_t flb_help_build_json_schema(struct flb_config *config) } flb_mp_array_header_end(&mh); - json = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + json = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); msgpack_sbuffer_destroy(&mp_sbuf); return json; diff --git a/src/flb_pack.c b/src/flb_pack.c index df1f9c196dc..7e10c6d9217 100644 --- a/src/flb_pack.c +++ b/src/flb_pack.c @@ -625,7 +625,7 @@ static inline int key_exists_in_map(msgpack_object key, msgpack_object map, int } static int msgpack2json(char *buf, int *off, size_t left, - const msgpack_object *o) + const msgpack_object *o, int escape_unicode) { int i; int dup; @@ -679,7 +679,7 @@ static int msgpack2json(char *buf, int *off, size_t left, case MSGPACK_OBJECT_STR: if (try_to_write(buf, off, left, "\"", 1) && (o->via.str.size > 0 ? - try_to_write_str(buf, off, left, o->via.str.ptr, o->via.str.size) + try_to_write_str(buf, off, left, o->via.str.ptr, o->via.str.size, escape_unicode) : 1/* nothing to do */) && try_to_write(buf, off, left, "\"", 1)) { ret = FLB_TRUE; @@ -689,7 +689,7 @@ static int msgpack2json(char *buf, int *off, size_t left, case MSGPACK_OBJECT_BIN: if (try_to_write(buf, off, left, "\"", 1) && (o->via.bin.size > 0 ? - try_to_write_str(buf, off, left, o->via.bin.ptr, o->via.bin.size) + try_to_write_str(buf, off, left, o->via.bin.ptr, o->via.bin.size, escape_unicode) : 1 /* nothing to do */) && try_to_write(buf, off, left, "\"", 1)) { ret = FLB_TRUE; @@ -726,12 +726,12 @@ static int msgpack2json(char *buf, int *off, size_t left, } if (loop != 0) { msgpack_object* p = o->via.array.ptr; - if (!msgpack2json(buf, off, left, p)) { + if (!msgpack2json(buf, off, left, p, escape_unicode)) { goto msg2json_end; } for (i=1; ikey) || + !msgpack2json(buf, off, left, &(p+i)->key, escape_unicode) || !try_to_write(buf, off, left, ":", 1) || - !msgpack2json(buf, off, left, &(p+i)->val) ) { + !msgpack2json(buf, off, left, &(p+i)->val, escape_unicode) ) { goto msg2json_end; } packed++; @@ -797,7 +797,7 @@ static int msgpack2json(char *buf, int *off, size_t left, * @return success ? a number characters filled : negative value */ int flb_msgpack_to_json(char *json_str, size_t json_size, - const msgpack_object *obj) + const msgpack_object *obj, int escape_unicode) { int ret = -1; int off = 0; @@ -806,12 +806,12 @@ int flb_msgpack_to_json(char *json_str, size_t json_size, return -1; } - ret = msgpack2json(json_str, &off, json_size - 1, obj); + ret = msgpack2json(json_str, &off, json_size - 1, obj, escape_unicode); json_str[off] = '\0'; return ret ? off: ret; } -flb_sds_t flb_msgpack_raw_to_json_sds(const void *in_buf, size_t in_size) +flb_sds_t flb_msgpack_raw_to_json_sds(const void *in_buf, size_t in_size, int escape_unicode) { int ret; size_t off = 0; @@ -846,7 +846,7 @@ flb_sds_t flb_msgpack_raw_to_json_sds(const void *in_buf, size_t in_size) root = &result.data; while (1) { - ret = flb_msgpack_to_json(out_buf, out_size, root); + ret = flb_msgpack_to_json(out_buf, out_size, root, escape_unicode); if (ret <= 0) { realloc_size *= 2; tmp_buf = flb_sds_increase(out_buf, realloc_size); @@ -956,7 +956,7 @@ static int msgpack_pack_formatted_datetime(flb_sds_t out_buf, char time_formatte flb_sds_t flb_pack_msgpack_to_json_format(const char *data, uint64_t bytes, int json_format, int date_format, - flb_sds_t date_key) + flb_sds_t date_key, int escape_unicode) { int i; int ret; @@ -1163,7 +1163,7 @@ flb_sds_t flb_pack_msgpack_to_json_format(const char *data, uint64_t bytes, json_format == FLB_PACK_JSON_FORMAT_STREAM) { /* Encode current record into JSON in a temporary variable */ - out_js = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size); + out_js = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size, escape_unicode); if (!out_js) { flb_sds_destroy(out_buf); msgpack_sbuffer_destroy(&tmp_sbuf); @@ -1219,7 +1219,7 @@ flb_sds_t flb_pack_msgpack_to_json_format(const char *data, uint64_t bytes, /* Format to JSON */ if (json_format == FLB_PACK_JSON_FORMAT_JSON) { - out_buf = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size, escape_unicode); msgpack_sbuffer_destroy(&tmp_sbuf); if (!out_buf) { return NULL; @@ -1244,7 +1244,7 @@ flb_sds_t flb_pack_msgpack_to_json_format(const char *data, uint64_t bytes, * @param data The msgpack_unpacked data. * @return success ? allocated json str ptr : NULL */ -char *flb_msgpack_to_json_str(size_t size, const msgpack_object *obj) +char *flb_msgpack_to_json_str(size_t size, const msgpack_object *obj, int escape_unicode) { int ret; char *buf = NULL; @@ -1265,7 +1265,7 @@ char *flb_msgpack_to_json_str(size_t size, const msgpack_object *obj) } while (1) { - ret = flb_msgpack_to_json(buf, size, obj); + ret = flb_msgpack_to_json(buf, size, obj, escape_unicode); if (ret <= 0) { /* buffer is small. retry.*/ size *= 2; diff --git a/src/flb_record_accessor.c b/src/flb_record_accessor.c index ed66c0f52b7..99cbfd96bbb 100644 --- a/src/flb_record_accessor.c +++ b/src/flb_record_accessor.c @@ -553,7 +553,7 @@ static flb_sds_t ra_translate_keymap(struct flb_ra_parser *rp, flb_sds_t buf, /* Check if is a map or a real bool */ if (v->o.type == MSGPACK_OBJECT_MAP) { /* Convert msgpack map to JSON string */ - js = flb_msgpack_to_json_str(1024, &v->o); + js = flb_msgpack_to_json_str(1024, &v->o, FLB_TRUE); if (js) { len = strlen(js); flb_sds_cat_safe(&buf, js, len); diff --git a/src/flb_sds.c b/src/flb_sds.c index 4ac36ad22e8..7d11ba1682d 100644 --- a/src/flb_sds.c +++ b/src/flb_sds.c @@ -312,7 +312,7 @@ flb_sds_t flb_sds_cat_utf8(flb_sds_t *sds, const char *str, int str_len) while (1) { offset = head->len; - ret = flb_utils_write_str(s, &offset, flb_sds_alloc(s), str, str_len); + ret = flb_utils_write_str(s, &offset, flb_sds_alloc(s), str, str_len, FLB_TRUE); if (ret == FLB_FALSE) { /* realloc */ size = flb_sds_alloc(s) * 2; diff --git a/src/flb_utils.c b/src/flb_utils.c index 56e6877d7bb..6c930704514 100644 --- a/src/flb_utils.c +++ b/src/flb_utils.c @@ -788,7 +788,7 @@ static const struct escape_seq json_escape_table[128] = { * to escape special characters and convert utf-8 byte characters to string * representation. */ -int flb_utils_write_str(char *buf, int *off, size_t size, const char *str, size_t str_len) +static int flb_utils_write_str_escaped(char *buf, int *off, size_t size, const char *str, size_t str_len) { int i, b, ret, len, hex_bytes, utf_sequence_length, utf_sequence_number; int processed_bytes = 0; @@ -1096,7 +1096,139 @@ int flb_utils_write_str(char *buf, int *off, size_t size, const char *str, size_ return FLB_TRUE; } -int flb_utils_write_str_buf(const char *str, size_t str_len, char **out, size_t *out_size) +/* Safely copies raw UTF-8 strings, only escaping essential characters. + * This version correctly implements the repeating SIMD fast path for performance. + */ +static int flb_utils_write_str_raw(char *buf, int *off, size_t size, + const char *str, size_t str_len) +{ + int i, b, vlen, len, utf_len, copypos = 0; + size_t available; + char *p; + off_t offset = 0; + const size_t inst_len = FLB_SIMD_VEC8_INST_LEN; + uint32_t c; + char *seq = NULL; + + available = size - *off; + p = buf + *off; + + /* align length to the nearest multiple of the vector size for safe SIMD processing */ + vlen = str_len & ~(inst_len - 1); + + for (i = 0;;) { + /* + * Process chunks of the input string using SIMD instructions. + * This loop continues as long as it finds "safe" ASCII characters. + */ + for (; i < vlen; i += inst_len) { + flb_vector8 chunk; + flb_vector8_load(&chunk, (const uint8_t *)&str[i]); + + /* If a special character is found, break and switch to the slow path */ + if (flb_vector8_has_le(chunk, (unsigned char) 0x1F) || + flb_vector8_has(chunk, (unsigned char) '"') || + flb_vector8_has(chunk, (unsigned char) '\\') || + flb_vector8_is_highbit_set(chunk)) { + break; + } + } + + /* Copy the 'safe' chunk processed by the SIMD loop so far */ + if (copypos < i) { + if (available < i - copypos) { + return FLB_FALSE; + } + memcpy(p, &str[copypos], i - copypos); + p += i - copypos; + offset += i - copypos; + available -= (i - copypos); + copypos = i; + } + + /* + * Process the next 16-byte chunk character by character. + * This loop runs only for a chunk that contains special characters. + */ + for (b = 0; b < inst_len; b++) { + if (i >= str_len) { + goto done; + } + + c = (uint32_t) str[i]; + len = 0; + seq = NULL; + + /* Handle essential escapes for JSON validity */ + if (c < 128 && json_escape_table[c].seq) { + seq = json_escape_table[c].seq; + len = json_escape_table[c].seq[1] == 'u' ? 6 : 2; + if (available < len) { + return FLB_FALSE; + } + memcpy(p, seq, len); + p += len; + offset += len; + available -= len; + } + else if (c < 0x80) { /* Regular ASCII */ + if (available < 1) { + return FLB_FALSE; + } + *p++ = c; + offset++; + available--; + } + else { /* Multibyte UTF-8 sequence */ + utf_len = flb_utf8_len(&str[i]); + + if (utf_len == 0 || i + utf_len > str_len) { /* Invalid/truncated */ + if (available < 3) { + return FLB_FALSE; + } + memcpy(p, "\xEF\xBF\xBD", 3); /* Standard replacement character */ + p += 3; + offset += 3; + available -= 3; + } + else { /* Valid sequence, copy raw */ + if (available < utf_len) { + return FLB_FALSE; + } + memcpy(p, &str[i], utf_len); + p += utf_len; + offset += utf_len; + available -= utf_len; + i += utf_len - 1; /* Advance loop counter by extra bytes */ + } + } + i++; + } + copypos = i; + } + +done: + *off += offset; + return FLB_TRUE; +} + +/* + * This is the wrapper public function for acting as a wrapper and calls the + * appropriate specialized function based on the escape_unicode flag. + */ +int flb_utils_write_str(char *buf, int *off, size_t size, const char *str, size_t str_len, + int escape_unicode) +{ + if (escape_unicode == FLB_TRUE) { + return flb_utils_write_str_escaped(buf, off, size, str, str_len); + } + else { + return flb_utils_write_str_raw(buf, off, size, str, str_len); + } +} + +int flb_utils_write_str_buf(const char *str, size_t str_len, char **out, size_t *out_size, + int escape_unicode) { int ret; int off; @@ -1113,7 +1245,7 @@ int flb_utils_write_str_buf(const char *str, size_t str_len, char **out, size_t while (1) { off = 0; - ret = flb_utils_write_str(buf, &off, s, str, str_len); + ret = flb_utils_write_str(buf, &off, s, str, str_len, escape_unicode); if (ret == FLB_FALSE) { s += 256; tmp = flb_realloc(buf, s); diff --git a/src/fluent-bit.c b/src/fluent-bit.c index 0e1b8f8840b..0a51efcb478 100644 --- a/src/fluent-bit.c +++ b/src/fluent-bit.c @@ -376,7 +376,8 @@ static void help_format_json(void *help_buf, size_t help_size) { flb_sds_t json; - json = flb_msgpack_raw_to_json_sds(help_buf, help_size); + /* Keep backward compatibility to format help */ + json = flb_msgpack_raw_to_json_sds(help_buf, help_size, FLB_TRUE); printf("%s\n", json); flb_sds_destroy(json); } diff --git a/src/http_server/api/v1/metrics.c b/src/http_server/api/v1/metrics.c index 1f5ed7136db..9f81ce3411e 100644 --- a/src/http_server/api/v1/metrics.c +++ b/src/http_server/api/v1/metrics.c @@ -153,7 +153,7 @@ static void cb_mq_metrics(mk_mq_t *queue, void *data, size_t size) } /* Convert msgpack to JSON */ - out_data = flb_msgpack_raw_to_json_sds(data, size); + out_data = flb_msgpack_raw_to_json_sds(data, size, FLB_TRUE); if (!out_data) { return; } diff --git a/src/http_server/api/v1/plugins.c b/src/http_server/api/v1/plugins.c index c755173e643..4cf1c2f5675 100644 --- a/src/http_server/api/v1/plugins.c +++ b/src/http_server/api/v1/plugins.c @@ -90,7 +90,7 @@ static void cb_plugins(mk_request_t *request, void *data) } /* Export to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); msgpack_sbuffer_destroy(&mp_sbuf); mk_http_status(request, 200); diff --git a/src/http_server/api/v1/storage.c b/src/http_server/api/v1/storage.c index 56add002ed0..33d9e4de918 100644 --- a/src/http_server/api/v1/storage.c +++ b/src/http_server/api/v1/storage.c @@ -105,7 +105,7 @@ static void cb_mq_storage_metrics(mk_mq_t *queue, void *data, size_t size) } /* Convert msgpack to JSON */ - out_data = flb_msgpack_raw_to_json_sds(data, size); + out_data = flb_msgpack_raw_to_json_sds(data, size, FLB_TRUE); if (!out_data) { return; } diff --git a/src/http_server/api/v1/trace.c b/src/http_server/api/v1/trace.c index 1f4021d685b..1ed6f2c32ac 100644 --- a/src/http_server/api/v1/trace.c +++ b/src/http_server/api/v1/trace.c @@ -471,7 +471,7 @@ static void cb_trace(mk_request_t *request, void *data) } /* Export to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); if (out_buf == NULL) { mk_http_status(request, 503); mk_http_done(request); @@ -638,7 +638,7 @@ static void cb_traces(mk_request_t *request, void *data) } /* Export to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); if (out_buf == NULL) { out_buf = flb_sds_create("serialization error"); } diff --git a/src/http_server/api/v1/uptime.c b/src/http_server/api/v1/uptime.c index 12a96a48cce..cb4e567ee84 100644 --- a/src/http_server/api/v1/uptime.c +++ b/src/http_server/api/v1/uptime.c @@ -88,7 +88,7 @@ static void cb_uptime(mk_request_t *request, void *data) uptime_hr(uptime, &mp_pck); /* Export to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); msgpack_sbuffer_destroy(&mp_sbuf); if (!out_buf) { return; diff --git a/src/http_server/api/v2/reload.c b/src/http_server/api/v2/reload.c index 2f8c947cbd9..b57aff14930 100644 --- a/src/http_server/api/v2/reload.c +++ b/src/http_server/api/v2/reload.c @@ -114,7 +114,7 @@ static void handle_reload_request(mk_request_t *request, struct flb_config *conf #endif /* Export to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); msgpack_sbuffer_destroy(&mp_sbuf); if (!out_buf) { mk_http_status(request, 400); @@ -148,7 +148,7 @@ static void handle_get_reload_status(mk_request_t *request, struct flb_config *c msgpack_pack_int64(&mp_pck, config->hot_reloaded_count); /* Export to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); msgpack_sbuffer_destroy(&mp_sbuf); if (!out_buf) { mk_http_status(request, 400); diff --git a/src/http_server/flb_hs_endpoints.c b/src/http_server/flb_hs_endpoints.c index 2ea12a981bb..ffda448bfb1 100644 --- a/src/http_server/flb_hs_endpoints.c +++ b/src/http_server/flb_hs_endpoints.c @@ -93,7 +93,7 @@ static int endpoint_root(struct flb_hs *hs) flb_utils_split_free(list); /* export as JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); msgpack_sbuffer_destroy(&mp_sbuf); if (out_buf) { diff --git a/tests/internal/fuzzers/flb_json_fuzzer.c b/tests/internal/fuzzers/flb_json_fuzzer.c index 916b1a51265..60e6422b58f 100644 --- a/tests/internal/fuzzers/flb_json_fuzzer.c +++ b/tests/internal/fuzzers/flb_json_fuzzer.c @@ -49,7 +49,7 @@ int LLVMFuzzerTestOneInput(unsigned char *data, size_t size) if (ret2 == MSGPACK_UNPACK_SUCCESS) { msgpack_object root = result.data; char *tmp = NULL; - tmp = flb_msgpack_to_json_str(0, &root); + tmp = flb_msgpack_to_json_str(0, &root, FLB_TRUE); if (tmp != NULL) { flb_free(tmp); } @@ -60,7 +60,7 @@ int LLVMFuzzerTestOneInput(unsigned char *data, size_t size) if (decider < 0x30) { flb_sds_t ret_s = flb_pack_msgpack_to_json_format(out_buf, out_size, FLB_PACK_JSON_FORMAT_LINES, - (int)decider, d); + (int)decider, d, FLB_TRUE); free(out_buf); if (ret_s != NULL) { flb_sds_destroy(ret_s); @@ -69,7 +69,7 @@ int LLVMFuzzerTestOneInput(unsigned char *data, size_t size) else { flb_sds_t ret_s = flb_pack_msgpack_to_json_format(out_buf, out_size, FLB_PACK_JSON_FORMAT_LINES, - FLB_PACK_JSON_DATE_EPOCH, NULL); + FLB_PACK_JSON_DATE_EPOCH, NULL, FLB_TRUE); free(out_buf); if (ret_s != NULL) { flb_sds_destroy(ret_s); diff --git a/tests/internal/fuzzers/msgpack_parse_fuzzer.c b/tests/internal/fuzzers/msgpack_parse_fuzzer.c index cbf2ecf1484..02011c7a888 100644 --- a/tests/internal/fuzzers/msgpack_parse_fuzzer.c +++ b/tests/internal/fuzzers/msgpack_parse_fuzzer.c @@ -23,7 +23,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size){ /* target the conversion of raw msgpack to json */ flb_sds_t record; - record = flb_msgpack_raw_to_json_sds(data, size); + record = flb_msgpack_raw_to_json_sds(data, size, FLB_TRUE); flb_sds_destroy(record); return 0; diff --git a/tests/internal/fuzzers/utils_fuzzer.c b/tests/internal/fuzzers/utils_fuzzer.c index 2352adf3e43..8d70d32a144 100644 --- a/tests/internal/fuzzers/utils_fuzzer.c +++ b/tests/internal/fuzzers/utils_fuzzer.c @@ -62,7 +62,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) char *uri = NULL; char *new_dst = NULL; - if (flb_utils_write_str_buf(null_terminated, size, &new_dst, &new_size) == 0) { + if (flb_utils_write_str_buf(null_terminated, size, &new_dst, &new_size, FLB_TRUE) == 0) { flb_free(new_dst); } diff --git a/tests/internal/log_event_decoder.c b/tests/internal/log_event_decoder.c index 6f93edcfb77..cc1509c7f57 100644 --- a/tests/internal/log_event_decoder.c +++ b/tests/internal/log_event_decoder.c @@ -180,7 +180,7 @@ void decode_object() return; } - json = flb_msgpack_to_json_str(4096, event.body); + json = flb_msgpack_to_json_str(4096, event.body, FLB_TRUE); if (!TEST_CHECK(json != NULL)) { TEST_MSG("flb_msgpack_to_json_str error"); return; @@ -250,7 +250,7 @@ void decoder_next() return; } - json = flb_msgpack_to_json_str(4096, event.body); + json = flb_msgpack_to_json_str(4096, event.body, FLB_TRUE); if (!TEST_CHECK(json != NULL)) { TEST_MSG("flb_msgpack_to_json_str error"); return; diff --git a/tests/internal/mp.c b/tests/internal/mp.c index 627d063d72e..7cdf6a3add8 100644 --- a/tests/internal/mp.c +++ b/tests/internal/mp.c @@ -218,7 +218,7 @@ void test_keys_remove_subkey_key() off = 0; msgpack_unpacked_init(&result_final); msgpack_unpack_next(&result_final, out_buf, out_size, &off); - flb_msgpack_to_json(&final_json[0], sizeof(final_json), &result_final.data); + flb_msgpack_to_json(&final_json[0], sizeof(final_json), &result_final.data, FLB_TRUE); if (!TEST_CHECK(strstr(&final_json[0] ,"kubernetes") == NULL)) { TEST_MSG("kubernetes field should be removed"); @@ -310,7 +310,7 @@ void test_remove_sibling_subkeys() off = 0; msgpack_unpacked_init(&result_final); msgpack_unpack_next(&result_final, out_buf, out_size, &off); - flb_msgpack_to_json(&final_json[0], sizeof(final_json), &result_final.data); + flb_msgpack_to_json(&final_json[0], sizeof(final_json), &result_final.data, FLB_TRUE); if (!TEST_CHECK(strstr(&final_json[0] ,"pod_id") == NULL)) { TEST_MSG("pod_id field should be removed"); @@ -411,7 +411,7 @@ void remove_subkey_keys(char *list[], int list_size, int index_start) off = 0; msgpack_unpacked_init(&result_final); msgpack_unpack_next(&result_final, out_buf, out_size, &off); - flb_msgpack_to_json(&final_json[0], sizeof(final_json), &result_final.data); + flb_msgpack_to_json(&final_json[0], sizeof(final_json), &result_final.data, FLB_TRUE); if (!TEST_CHECK(strstr(&final_json[0] ,"kubernetes") == NULL)) { TEST_MSG("kubernetes field should be removed"); @@ -519,8 +519,8 @@ void test_object_to_cfl_to_msgpack() * Convert buf (msgpack 1 buffer) to JSON, and compare the strings * generated by out_buf (msgpack 2 buffer). They must match. */ - buf1 = flb_msgpack_raw_to_json_sds(buf, size); - buf2 = flb_msgpack_raw_to_json_sds(out_buf, out_size); + buf1 = flb_msgpack_raw_to_json_sds(buf, size, FLB_TRUE); + buf2 = flb_msgpack_raw_to_json_sds(out_buf, out_size, FLB_TRUE); ret = strcmp(buf1, buf2); printf("\n>> Compare JSON buf1 v/s JSON buf2 (ret=%i):\n", ret); diff --git a/tests/internal/msgpack_append_message.c b/tests/internal/msgpack_append_message.c index abb487e877c..fe957a98a00 100644 --- a/tests/internal/msgpack_append_message.c +++ b/tests/internal/msgpack_append_message.c @@ -53,7 +53,7 @@ static inline int process_pack(char *pack, size_t size) MSGPACK_OBJECT_STR); TEST_CHECK(ret == 0); - out_buf = flb_msgpack_raw_to_json_sds(appended_buffer, appended_size); + out_buf = flb_msgpack_raw_to_json_sds(appended_buffer, appended_size, FLB_TRUE); TEST_CHECK(out_buf != NULL); p = strstr(out_buf, "\"expanding\":\"injected\""); if (!TEST_CHECK(p != NULL)) { diff --git a/tests/internal/opentelemetry.c b/tests/internal/opentelemetry.c index 5464d3ce1f5..b58fd85f15e 100644 --- a/tests/internal/opentelemetry.c +++ b/tests/internal/opentelemetry.c @@ -185,8 +185,8 @@ static struct test_output *parse_test_output(void *chunk, size_t size) if (record_type == FLB_LOG_EVENT_GROUP_START) { /* Group header */ if (group_idx < output->group_count) { - output->groups[group_idx].metadata = flb_msgpack_to_json_str(1024, event.metadata); - output->groups[group_idx].body = flb_msgpack_to_json_str(1024, event.body); + output->groups[group_idx].metadata = flb_msgpack_to_json_str(1024, event.metadata, FLB_TRUE); + output->groups[group_idx].body = flb_msgpack_to_json_str(1024, event.body, FLB_TRUE); /* Allocate records for this group */ if (output->groups[group_idx].record_count > 0) { @@ -207,8 +207,8 @@ static struct test_output *parse_test_output(void *chunk, size_t size) /* Log record within a group */ if (group_idx < output->group_count && record_idx < output->groups[group_idx].record_count) { - output->groups[group_idx].records[record_idx].metadata = flb_msgpack_to_json_str(1024, event.metadata); - output->groups[group_idx].records[record_idx].body = flb_msgpack_to_json_str(1024, event.body); + output->groups[group_idx].records[record_idx].metadata = flb_msgpack_to_json_str(1024, event.metadata, FLB_TRUE); + output->groups[group_idx].records[record_idx].body = flb_msgpack_to_json_str(1024, event.body, FLB_TRUE); record_idx++; } } @@ -306,7 +306,7 @@ static int validate_extended_output(struct test_output *actual, msgpack_object * /* Validate group metadata */ ret = flb_otel_utils_find_map_entry_by_key(&group_obj->via.map, "metadata", 0, FLB_TRUE); if (ret >= 0) { - expected_meta = flb_msgpack_to_json_str(256, &group_obj->via.map.ptr[ret].val); + expected_meta = flb_msgpack_to_json_str(256, &group_obj->via.map.ptr[ret].val, FLB_TRUE); if (strcmp(expected_meta, actual->groups[i].metadata) != 0) { printf("Group %zu metadata mismatch:\nExpected: %s\nGot: %s\n", i, expected_meta, actual->groups[i].metadata); @@ -319,7 +319,7 @@ static int validate_extended_output(struct test_output *actual, msgpack_object * /* Validate group body */ ret = flb_otel_utils_find_map_entry_by_key(&group_obj->via.map, "body", 0, FLB_TRUE); if (ret >= 0) { - expected_body = flb_msgpack_to_json_str(256, &group_obj->via.map.ptr[ret].val); + expected_body = flb_msgpack_to_json_str(256, &group_obj->via.map.ptr[ret].val, FLB_TRUE); if (strcmp(expected_body, actual->groups[i].body) != 0) { printf("Group %zu body mismatch:\nExpected: %s\nGot: %s\n", i, expected_body, actual->groups[i].body); @@ -357,7 +357,7 @@ static int validate_extended_output(struct test_output *actual, msgpack_object * /* Validate record metadata */ ret = flb_otel_utils_find_map_entry_by_key(&record_obj->via.map, "metadata", 0, FLB_TRUE); if (ret >= 0) { - expected_meta = flb_msgpack_to_json_str(256, &record_obj->via.map.ptr[ret].val); + expected_meta = flb_msgpack_to_json_str(256, &record_obj->via.map.ptr[ret].val, FLB_TRUE); if (strcmp(expected_meta, actual->groups[i].records[j].metadata) != 0) { printf("Group %zu record %zu metadata mismatch:\nExpected: %s\nGot: %s\n", i, j, expected_meta, actual->groups[i].records[j].metadata); @@ -370,7 +370,7 @@ static int validate_extended_output(struct test_output *actual, msgpack_object * /* Validate record body */ ret = flb_otel_utils_find_map_entry_by_key(&record_obj->via.map, "body", 0, FLB_TRUE); if (ret >= 0) { - expected_body = flb_msgpack_to_json_str(256, &record_obj->via.map.ptr[ret].val); + expected_body = flb_msgpack_to_json_str(256, &record_obj->via.map.ptr[ret].val, FLB_TRUE); if (strcmp(expected_body, actual->groups[i].records[j].body) != 0) { printf("Group %zu record %zu body mismatch:\nExpected: %s\nGot: %s\n", i, j, expected_body, actual->groups[i].records[j].body); @@ -545,7 +545,7 @@ void test_opentelemetry_cases() ret = flb_otel_utils_find_map_entry_by_key(&case_obj->via.map, "input", 0, FLB_TRUE); TEST_CHECK(ret >= 0); - input_json = flb_msgpack_to_json_str(1024, &case_obj->via.map.ptr[ret].val); + input_json = flb_msgpack_to_json_str(1024, &case_obj->via.map.ptr[ret].val, FLB_TRUE); TEST_CHECK(input_json != NULL); ret = flb_log_event_encoder_init(&enc, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); @@ -579,22 +579,22 @@ void test_opentelemetry_cases() if (empty_payload == FLB_FALSE && has_groups == FLB_FALSE) { ret = flb_otel_utils_find_map_entry_by_key(&expected->via.map, "group_metadata", 0, FLB_TRUE); TEST_CHECK(ret >= 0); - expect_group_meta = flb_msgpack_to_json_str(256, &expected->via.map.ptr[ret].val); + expect_group_meta = flb_msgpack_to_json_str(256, &expected->via.map.ptr[ret].val, FLB_TRUE); TEST_CHECK(expect_group_meta != NULL); ret = flb_otel_utils_find_map_entry_by_key(&expected->via.map, "group_body", 0, FLB_TRUE); TEST_CHECK(ret >= 0); - expect_group_body = flb_msgpack_to_json_str(256, &expected->via.map.ptr[ret].val); + expect_group_body = flb_msgpack_to_json_str(256, &expected->via.map.ptr[ret].val, FLB_TRUE); TEST_CHECK(expect_group_body != NULL); ret = flb_otel_utils_find_map_entry_by_key(&expected->via.map, "log_metadata", 0, FLB_TRUE); TEST_CHECK(ret >= 0); - expect_log_meta = flb_msgpack_to_json_str(256, &expected->via.map.ptr[ret].val); + expect_log_meta = flb_msgpack_to_json_str(256, &expected->via.map.ptr[ret].val, FLB_TRUE); TEST_CHECK(expect_log_meta != NULL); ret = flb_otel_utils_find_map_entry_by_key(&expected->via.map, "log_body", 0, FLB_TRUE); TEST_CHECK(ret >= 0); - expect_log_body = flb_msgpack_to_json_str(256, &expected->via.map.ptr[ret].val); + expect_log_body = flb_msgpack_to_json_str(256, &expected->via.map.ptr[ret].val, FLB_TRUE); TEST_CHECK(expect_log_body != NULL); } diff --git a/tests/internal/pack.c b/tests/internal/pack.c index aa73ff7defe..abaa403f8e5 100644 --- a/tests/internal/pack.c +++ b/tests/internal/pack.c @@ -283,7 +283,7 @@ void test_json_dup_keys() out_json = flb_pack_msgpack_to_json_format(out_buf, out_size, FLB_PACK_JSON_FORMAT_LINES, FLB_PACK_JSON_DATE_EPOCH, - d); + d, FLB_TRUE); TEST_CHECK(out_json != NULL); TEST_CHECK(strncmp(out_json, data_out, flb_sds_len(out_json)) == 0); @@ -496,7 +496,7 @@ void test_utf8_to_json() json_size = strlen(file_json); - out_buf = flb_msgpack_raw_to_json_sds(file_msgp, msgp_size); + out_buf = flb_msgpack_raw_to_json_sds(file_msgp, msgp_size, FLB_TRUE); TEST_CHECK(out_buf != NULL); out_size = flb_sds_len(out_buf); @@ -717,7 +717,7 @@ void test_json_pack_bug1278() msgpack_pack_str_body(&mp_pck, p_in, len); /* Pack raw string as JSON */ - json = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + json = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); /* Compare expected JSON output */ ret = strcmp(p_out, json); @@ -798,7 +798,7 @@ void test_json_pack_nan() msgpack_sbuffer_destroy(&mp_sbuf); // convert msgpack to json - ret = flb_msgpack_to_json(&json_str[0], sizeof(json_str), &obj); + ret = flb_msgpack_to_json(&json_str[0], sizeof(json_str), &obj, FLB_TRUE); TEST_CHECK(ret >= 0); p = strstr(&json_str[0], "nan"); @@ -809,7 +809,7 @@ void test_json_pack_nan() // convert. nan -> null memset(&json_str[0], 0, sizeof(json_str)); flb_pack_init(&config); - ret = flb_msgpack_to_json(&json_str[0], sizeof(json_str), &obj); + ret = flb_msgpack_to_json(&json_str[0], sizeof(json_str), &obj, FLB_TRUE); TEST_CHECK(ret >= 0); p = strstr(&json_str[0], "null"); @@ -990,7 +990,7 @@ void test_json_date(char* expect, int date_format) ret = flb_pack_msgpack_to_json_format((const char*)&input_msgpack[0], sizeof(input_msgpack), FLB_PACK_JSON_FORMAT_JSON, date_format, - json_key); + json_key, FLB_TRUE); if (!TEST_CHECK(ret != NULL)) { TEST_MSG("flb_pack_msgpack_to_json_format failed"); flb_sds_destroy(json_key); diff --git a/tests/internal/stream_processor.c b/tests/internal/stream_processor.c index aa7bf69beda..4fcd02956f0 100644 --- a/tests/internal/stream_processor.c +++ b/tests/internal/stream_processor.c @@ -886,7 +886,7 @@ static void test_conv_from_str_to_num() goto test_conv_from_str_to_num_end; } - ret = flb_msgpack_to_json(&json[0], sizeof(json), &result.data); + ret = flb_msgpack_to_json(&json[0], sizeof(json), &result.data, FLB_TRUE); if (!TEST_CHECK(ret > 0)) { TEST_MSG("flb_msgpack_to_json failed"); msgpack_unpacked_destroy(&result); diff --git a/tests/internal/utils.c b/tests/internal/utils.c index 232e2461c0b..7d19f97cf76 100644 --- a/tests/internal/utils.c +++ b/tests/internal/utils.c @@ -203,13 +203,19 @@ void test_url_split() } /* test case loop for flb_utils_write_str */ -static void write_str_test_cases_w_buf_size(struct write_str_case *cases, int buf_size); +static void write_str_test_cases_w_buf_size(struct write_str_case *cases, int buf_size, + int escape_unicode); static void write_str_test_cases(struct write_str_case *cases) { - write_str_test_cases_w_buf_size(cases, 100); + write_str_test_cases_w_buf_size(cases, 100, FLB_TRUE); +} + +static void write_raw_str_test_cases(struct write_str_case *cases) { + write_str_test_cases_w_buf_size(cases, 100, FLB_FALSE); } /* test case loop for flb_utils_write_str */ -static void write_str_test_cases_w_buf_size(struct write_str_case *cases, int buf_size) +static void write_str_test_cases_w_buf_size(struct write_str_case *cases, int buf_size, + int escape_unicode) { char *buf = flb_calloc(buf_size + 1, sizeof(char)); int size = buf_size + 1; @@ -220,7 +226,8 @@ static void write_str_test_cases_w_buf_size(struct write_str_case *cases, int bu while (!(tcase->input == 0 && tcase->output == 0)) { memset(buf, 0, size); off = 0; - ret = flb_utils_write_str(buf, &off, buf_size, tcase->input, tcase->input_len); + ret = flb_utils_write_str(buf, &off, buf_size, tcase->input, tcase->input_len, + escape_unicode); if(!TEST_CHECK(ret == tcase->ret)) { TEST_MSG("Input string: %s", tcase->input); @@ -262,30 +269,30 @@ void test_write_str() char jp_expected_output[] = "\\u3042"; off = 0; - ret = flb_utils_write_str(buf, &off, size, "a", 1); + ret = flb_utils_write_str(buf, &off, size, "a", 1, FLB_TRUE); TEST_CHECK(ret == FLB_TRUE); TEST_CHECK(memcmp(buf, "a", off) == 0); off = 0; - ret = flb_utils_write_str(buf, &off, size, "\n", 1); + ret = flb_utils_write_str(buf, &off, size, "\n", 1, FLB_TRUE); TEST_CHECK(ret == FLB_TRUE); TEST_CHECK(memcmp(buf, "\\n", off) == 0); off = 0; - ret = flb_utils_write_str(buf, &off, size, "\xe3\x81\x82", 3); + ret = flb_utils_write_str(buf, &off, size, "\xe3\x81\x82", 3, FLB_TRUE); TEST_CHECK(ret == FLB_TRUE); TEST_CHECK(memcmp(buf, jp_expected_output, off) == 0); /* Truncated bytes: 'buf' should not be touched and off == 0 */ off = 0; - ret = flb_utils_write_str(buf, &off, size, "\xe3\x81\x82\xe3", 1); + ret = flb_utils_write_str(buf, &off, size, "\xe3\x81\x82\xe3", 1, FLB_TRUE); TEST_CHECK(ret == FLB_TRUE); TEST_CHECK(off == 0); TEST_CHECK(memcmp(buf, jp_expected_output, off) == 0); /* Error: buffer too small */ off = 0; - ret = flb_utils_write_str(buf, &off, size, "aaaaaaaaaaa", 11); + ret = flb_utils_write_str(buf, &off, size, "aaaaaaaaaaa", 11, FLB_TRUE); TEST_CHECK(ret == FLB_FALSE); } @@ -388,6 +395,46 @@ void test_write_str_special_bytes() write_str_test_cases(cases); } +void test_write_raw_str_special_bytes() +{ + struct write_str_case cases[] = { + /* + * Input: "你好世界" (12 bytes) + * Output: "你好世界" (raw) + */ + { + "\xE4\xBD\xA0\xE5\xA5\xBD\xE4\xB8\x96\xE7\x95\x8C", 12, + "\xE4\xBD\xA0\xE5\xA5\xBD\xE4\xB8\x96\xE7\x95\x8C", + FLB_TRUE + }, + /* + * Input: "你好我来自一个汉字文化影响的地方" (48 bytes) + * Output: "你好我来自一个汉字文化影响的地方" (raw) + */ + { + "\xE4\xBD\xA0\xE5\xA5\xBD\xE6\x88\x91\xE6\x9D\xA5\xE8\x87\xAA" \ + "\xE4\xB8\x80\xE4\xB8\xAA\xE6\xB1\x89\xE5\xAD\x97\xE6\x96\x87" \ + "\xE5\x8C\x96\xE5\xBD\xB1\xE5\x93\x8D\xE7\x9A\x84\xE5\x9C\xB0" \ + "\xE6\x96\xB9", + 48, + "\xE4\xBD\xA0\xE5\xA5\xBD\xE6\x88\x91\xE6\x9D\xA5\xE8\x87\xAA" \ + "\xE4\xB8\x80\xE4\xB8\xAA\xE6\xB1\x89\xE5\xAD\x97\xE6\x96\x87" \ + "\xE5\x8C\x96\xE5\xBD\xB1\xE5\x93\x8D\xE7\x9A\x84\xE5\x9C\xB0" \ + "\xE6\x96\xB9", + FLB_TRUE + }, + /* Test string with a quote */ + { + "\"hello\"", 7, + "\\\"hello\\\"", + FLB_TRUE + }, + { 0 } + }; + + write_raw_str_test_cases(cases); +} + void test_write_str_invalid_leading_byte_case_2() { @@ -472,7 +519,7 @@ void test_write_str_buffer_overrun() }, { 0 } }; - write_str_test_cases_w_buf_size(cases, 5); + write_str_test_cases_w_buf_size(cases, 5, FLB_TRUE); } struct proxy_url_check { @@ -804,6 +851,7 @@ TEST_LIST = { { "url_split_sds", test_url_split_sds }, { "write_str", test_write_str }, { "write_str_special_bytes", test_write_str_special_bytes }, + { "write_raw_str_special_bytes", test_write_raw_str_special_bytes }, { "test_write_str_invalid_trailing_bytes", test_write_str_invalid_trailing_bytes }, { "test_write_str_invalid_leading_byte", test_write_str_invalid_leading_byte }, { "test_write_str_edge_cases", test_write_str_edge_cases }, diff --git a/tests/runtime/filter_lua.c b/tests/runtime/filter_lua.c index 70bbfc12b43..19540f221a5 100644 --- a/tests/runtime/filter_lua.c +++ b/tests/runtime/filter_lua.c @@ -185,7 +185,7 @@ static char *get_group_metadata(void *chunk, size_t size) return NULL; } - ret = flb_msgpack_to_json(out_buf, out_size, log_event.metadata); + ret = flb_msgpack_to_json(out_buf, out_size, log_event.metadata, FLB_TRUE); if (ret < 0) { flb_sds_destroy(out_buf); flb_log_event_decoder_destroy(&log_decoder); @@ -221,7 +221,7 @@ static char *get_group_body(void *chunk, size_t size) return NULL; } - ret = flb_msgpack_to_json(out_buf, out_size, log_event.body); + ret = flb_msgpack_to_json(out_buf, out_size, log_event.body, FLB_TRUE); if (ret < 0) { flb_sds_destroy(out_buf); flb_log_event_decoder_destroy(&log_decoder); @@ -252,7 +252,7 @@ static char *get_log_body(void *chunk, size_t size) return NULL; } - ret = flb_msgpack_to_json(out_buf, out_size, log_event.body); + ret = flb_msgpack_to_json(out_buf, out_size, log_event.body, FLB_TRUE); if (ret < 0) { flb_sds_destroy(out_buf); flb_log_event_decoder_destroy(&log_decoder); @@ -286,7 +286,7 @@ static char *get_record_metadata(void *chunk, size_t size) return NULL; } - ret = flb_msgpack_to_json(out_buf, out_size, log_event.metadata); + ret = flb_msgpack_to_json(out_buf, out_size, log_event.metadata, FLB_TRUE); if (ret < 0) { flb_sds_destroy(out_buf); flb_log_event_decoder_destroy(&log_decoder); @@ -1365,8 +1365,8 @@ static int cb_check_metadata_array(void *chunk, size_t size, void *data) TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); while ((ret = flb_log_event_decoder_next(&dec, &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - char *meta = flb_msgpack_to_json_str(256, log_event.metadata); - char *body = flb_msgpack_to_json_str(256, log_event.body); + char *meta = flb_msgpack_to_json_str(256, log_event.metadata, FLB_TRUE); + char *body = flb_msgpack_to_json_str(256, log_event.body, FLB_TRUE); TEST_CHECK(meta != NULL && body != NULL); if (meta && body) { diff --git a/tests/runtime/in_opentelemetry.c b/tests/runtime/in_opentelemetry.c index e9efabac9ca..27b5806f11f 100644 --- a/tests/runtime/in_opentelemetry.c +++ b/tests/runtime/in_opentelemetry.c @@ -98,7 +98,7 @@ static char *get_group_metadata(void *chunk, size_t size) return NULL; } - json = flb_msgpack_to_json_str(1024, log_event.metadata); + json = flb_msgpack_to_json_str(1024, log_event.metadata, FLB_TRUE); flb_log_event_decoder_destroy(&log_decoder); return json; } @@ -121,7 +121,7 @@ static char *get_group_body(void *chunk, size_t size) return NULL; } - json = flb_msgpack_to_json_str(1024, log_event.body); + json = flb_msgpack_to_json_str(1024, log_event.body, FLB_TRUE); flb_log_event_decoder_destroy(&log_decoder); return json; } @@ -146,7 +146,7 @@ static char *get_log_body(void *chunk, size_t size) flb_log_event_decoder_next(&log_decoder, &log_event); /* convert log body to json */ - json = flb_msgpack_to_json_str(1024, log_event.body); + json = flb_msgpack_to_json_str(1024, log_event.body, FLB_TRUE); flb_log_event_decoder_destroy(&log_decoder); return json; diff --git a/tests/runtime/in_systemd.c b/tests/runtime/in_systemd.c index 098d5ceb0c4..83c83357f7c 100644 --- a/tests/runtime/in_systemd.c +++ b/tests/runtime/in_systemd.c @@ -32,7 +32,7 @@ static void cb_check_cfl_variant_properties(void *ctx, int ffd, char *result = NULL; /* Convert from msgpack to JSON */ - output = flb_msgpack_raw_to_json_sds(res_data, res_size); + output = flb_msgpack_raw_to_json_sds(res_data, res_size, FLB_TRUE); TEST_CHECK(output != NULL); result = strstr(output, "\"MESSAGE\":\"test native message with multiple values\"");