diff --git a/plugins/out_syslog/syslog.c b/plugins/out_syslog/syslog.c index 4ecc7c4acc3..0922e655ebd 100644 --- a/plugins/out_syslog/syslog.c +++ b/plugins/out_syslog/syslog.c @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include "syslog_conf.h" @@ -545,100 +547,68 @@ static flb_sds_t msgpack_to_sd(struct flb_syslog *ctx, return *s; } +/* Use val array to return its string value unless its a string and return its pointer */ +static void extract_value_from_ra_result(struct flb_ra_value *rval, char** val, + int val_size, int *val_len) { + if (rval->o.type == MSGPACK_OBJECT_BOOLEAN) { + *val = rval->o.via.boolean ? "true" : "false"; + *val_len = rval->o.via.boolean ? 4 : 5; + } + else if (rval->o.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + *val_len = snprintf(*val, val_size, + "%" PRIu64, rval->o.via.u64); + } + else if (rval->o.type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { + *val_len = snprintf(*val, val_size, + "%" PRId64, rval->o.via.i64); + } + else if (rval->o.type == MSGPACK_OBJECT_FLOAT) { + *val_len = snprintf(*val, val_size, + "%f", rval->o.via.f64); + } + else if (rval->o.type == MSGPACK_OBJECT_STR) { + /* String value */ + *val = rval->o.via.str.ptr; + *val_len = rval->o.via.str.size; + } + else if (rval->o.type == MSGPACK_OBJECT_BIN) { + /* Bin value */ + *val = rval->o.via.bin.ptr; + *val_len = rval->o.via.bin.size; + } + else { + *val = NULL; + *val_len = 0; + } +} + static int msgpack_to_syslog(struct flb_syslog *ctx, msgpack_object *o, struct syslog_msg *msg) { int i; - int loop; struct mk_list *head; - struct flb_config_map_val *mv; + struct mk_list *tmp; + struct flb_syslog_sd_key *sd_key_item; + struct flb_ra_value *rval = NULL; + char *val = NULL; + int val_len = 0; + char temp[48] = {0}; if (o == NULL) { return -1; } - loop = o->via.map.size; - if (loop != 0) { - msgpack_object_kv *p = o->via.map.ptr; - - for (i = 0; i < loop; i++) { - char temp[48] = {0}; - const char *key = NULL; - int key_len = 0; - const char *val = NULL; - int val_len = 0; - - msgpack_object *k = &p[i].key; - msgpack_object *v = &p[i].val; - - if (k->type != MSGPACK_OBJECT_BIN && k->type != MSGPACK_OBJECT_STR){ - continue; - } - - if (k->type == MSGPACK_OBJECT_STR) { - key = k->via.str.ptr; - key_len = k->via.str.size; - } - else { - key = k->via.bin.ptr; - key_len = k->via.bin.size; - } - - if (v->type == MSGPACK_OBJECT_MAP) { - if (ctx->sd_keys) { - flb_config_map_foreach(head, mv, ctx->sd_keys) { - if ((key_len == flb_sds_len(mv->val.str)) && - strncmp(key, mv->val.str, flb_sds_len(mv->val.str)) == 0) { - msgpack_to_sd(ctx, &(msg->sd), key, key_len, v); - break; - } - } - } - continue; - } - - if (v->type == MSGPACK_OBJECT_BOOLEAN) { - val = v->via.boolean ? "true" : "false"; - val_len = v->via.boolean ? 4 : 5; - } - else if (v->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + if (ctx->ra_severity_key != NULL) { + if (msg->severity == -1) { + rval = flb_ra_get_value_object(ctx->ra_severity_key, *o); + if (rval) { val = temp; - val_len = snprintf(temp, sizeof(temp) - 1, - "%" PRIu64, v->via.u64); - } - else if (v->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - val = temp; - val_len = snprintf(temp, sizeof(temp) - 1, - "%" PRId64, v->via.i64); - } - else if (v->type == MSGPACK_OBJECT_FLOAT) { - val = temp; - val_len = snprintf(temp, sizeof(temp) - 1, - "%f", v->via.f64); - } - else if (v->type == MSGPACK_OBJECT_STR) { - /* String value */ - val = v->via.str.ptr; - val_len = v->via.str.size; - } - else if (v->type == MSGPACK_OBJECT_BIN) { - /* Bin value */ - val = v->via.bin.ptr; - val_len = v->via.bin.size; - } - - if (!val || !key) { - continue; - } - - if ((ctx->severity_key != NULL) && - flb_sds_cmp(ctx->severity_key, key, key_len) == 0) { - if (msg->severity == -1) { + extract_value_from_ra_result(rval, &val, sizeof(temp) - 1, &val_len); + if (val != NULL) { if ((val_len == 1) && (val[0] >= '0' && val[0] <= '7')) { msg->severity = val[0]-'0'; } else { - int i; for (i=0; syslog_severity[i].name != NULL; i++) { if ((syslog_severity[i].len == val_len) && (!strncasecmp(syslog_severity[i].name, val, val_len))) { @@ -651,10 +621,17 @@ static int msgpack_to_syslog(struct flb_syslog *ctx, msgpack_object *o, } } } + flb_ra_key_value_destroy(rval); } - else if ((ctx->facility_key != NULL) && - flb_sds_cmp(ctx->facility_key, key, key_len) == 0) { - if (msg->facility == -1) { + } + } + if (ctx->ra_facility_key != NULL) { + if (msg->facility == -1) { + rval = flb_ra_get_value_object(ctx->ra_facility_key, *o); + if (rval) { + val = temp; + extract_value_from_ra_result(rval, &val, sizeof(temp) - 1, &val_len); + if (val != NULL) { if ((val_len == 1) && (val[0] >= '0' && val[0] <= '9')) { msg->facility = val[0]-'0'; } @@ -670,7 +647,6 @@ static int msgpack_to_syslog(struct flb_syslog *ctx, msgpack_object *o, } } else { - int i; for (i=0; syslog_facility[i].name != NULL; i++) { if ((syslog_facility[i].len == val_len) && (!strncasecmp(syslog_facility[i].name, val, val_len))) { @@ -683,36 +659,81 @@ static int msgpack_to_syslog(struct flb_syslog *ctx, msgpack_object *o, } } } + flb_ra_key_value_destroy(rval); } - else if ((ctx->hostname_key != NULL) && - flb_sds_cmp(ctx->hostname_key, key, key_len) == 0) { - if (!msg->hostname) { - msg->hostname = flb_sds_create_len(val, val_len); - } + } + } + + if (ctx->ra_hostname_key != NULL) { + rval = flb_ra_get_value_object(ctx->ra_hostname_key, *o); + if (rval) { + val = temp; + extract_value_from_ra_result(rval, &val, sizeof(temp) - 1, &val_len); + if (!msg->hostname && val != NULL) { + msg->hostname = flb_sds_create_len(val, val_len); } - else if ((ctx->appname_key != NULL) && - flb_sds_cmp(ctx->appname_key, key, key_len) == 0) { - if (!msg->appname) { - msg->appname = flb_sds_create_len(val, val_len); - } + flb_ra_key_value_destroy(rval); + } + } + + if (ctx->ra_appname_key != NULL) { + rval = flb_ra_get_value_object(ctx->ra_appname_key, *o); + if (rval) { + val = temp; + extract_value_from_ra_result(rval, &val, sizeof(temp) - 1, &val_len); + if (!msg->appname && val != NULL) { + msg->appname = flb_sds_create_len(val, val_len); } - else if ((ctx->procid_key != NULL) && - flb_sds_cmp(ctx->procid_key, key, key_len) == 0) { - if (!msg->procid) { - msg->procid = flb_sds_create_len(val, val_len); - } + flb_ra_key_value_destroy(rval); + } + } + + if (ctx->ra_procid_key != NULL) { + rval = flb_ra_get_value_object(ctx->ra_procid_key, *o); + if (rval) { + val = temp; + extract_value_from_ra_result(rval, &val, sizeof(temp) - 1, &val_len); + if (!msg->procid && val != NULL) { + msg->procid = flb_sds_create_len(val, val_len); } - else if ((ctx->msgid_key != NULL) && - flb_sds_cmp(ctx->msgid_key, key, key_len) == 0) { - if (!msg->msgid) { - msg->msgid = flb_sds_create_len(val, val_len); - } + flb_ra_key_value_destroy(rval); + } + } + + if (ctx->ra_msgid_key != NULL) { + rval = flb_ra_get_value_object(ctx->ra_msgid_key, *o); + if (rval) { + val = temp; + extract_value_from_ra_result(rval, &val, sizeof(temp) - 1, &val_len); + if (!msg->msgid && val != NULL) { + msg->msgid = flb_sds_create_len(val, val_len); + } + flb_ra_key_value_destroy(rval); + } + } + + if (ctx->ra_message_key != NULL) { + rval = flb_ra_get_value_object(ctx->ra_message_key, *o); + if (rval) { + val = temp; + extract_value_from_ra_result(rval, &val, sizeof(temp) - 1, &val_len); + if (!msg->message && val != NULL) { + msg->message = flb_sds_create_len(val, val_len); } - else if ((ctx->message_key != NULL) && - flb_sds_cmp(ctx->message_key, key, key_len) == 0) { - if (!msg->message) { - msg->message = flb_sds_create_len(val, val_len); + flb_ra_key_value_destroy(rval); + } + } + + if (ctx->ra_sd_keys != NULL) { + mk_list_foreach_safe(head, tmp, ctx->ra_sd_keys) { + sd_key_item = mk_list_entry(head, struct flb_syslog_sd_key, _head); + rval = flb_ra_get_value_object(sd_key_item->ra_sd_key, *o); + if (rval) { + if (rval->o.type == MSGPACK_OBJECT_MAP) { + msgpack_to_sd(ctx, &(msg->sd), sd_key_item->key_normalized, + flb_sds_len(sd_key_item->key_normalized), &rval->o); } + flb_ra_key_value_destroy(rval); } } } diff --git a/plugins/out_syslog/syslog_conf.c b/plugins/out_syslog/syslog_conf.c index 05210d99de4..d7f6f363526 100644 --- a/plugins/out_syslog/syslog_conf.c +++ b/plugins/out_syslog/syslog_conf.c @@ -19,6 +19,11 @@ #include #include +#include +#include +#include + +#include #include "syslog_conf.h" @@ -60,6 +65,69 @@ static int is_valid_facility(struct flb_output_instance *ins, int val, int forma return 0; } +static inline void syslog_normalize_cat(struct flb_ra_parser *rp, flb_sds_t *name) +{ + int sub; + int len; + char tmp[64]; + struct mk_list *s_head; + struct flb_ra_key *key; + struct flb_ra_subentry *entry; + + /* Iterate record accessor keys */ + key = rp->key; + if (rp->type == FLB_RA_PARSER_STRING) { + flb_sds_cat_safe(name, key->name, flb_sds_len(key->name)); + } + else if (rp->type == FLB_RA_PARSER_KEYMAP) { + flb_sds_cat_safe(name, key->name, flb_sds_len(key->name)); + if (mk_list_size(key->subkeys) > 0) { + flb_sds_cat_safe(name, ".", 1); + } + + sub = 0; + mk_list_foreach(s_head, key->subkeys) { + entry = mk_list_entry(s_head, struct flb_ra_subentry, _head); + + if (sub > 0) { + flb_sds_cat_safe(name, ".", 1); + } + if (entry->type == FLB_RA_PARSER_STRING) { + flb_sds_cat_safe(name, entry->str, flb_sds_len(entry->str)); + } + else if (entry->type == FLB_RA_PARSER_ARRAY_ID) { + len = snprintf(tmp, sizeof(tmp) -1, "%d", + entry->array_id); + flb_sds_cat_safe(name, tmp, len); + } + sub++; + } + } +} + +static flb_sds_t syslog_normalize_ra_key_name(struct flb_record_accessor *ra) +{ + int c = 0; + flb_sds_t name; + struct mk_list *head; + struct flb_ra_parser *rp; + + name = flb_sds_create_size(flb_sds_len(ra->pattern)); + if (!name) { + return NULL; + } + + mk_list_foreach(head, &ra->list) { + rp = mk_list_entry(head, struct flb_ra_parser, _head); + if (c > 0) { + flb_sds_cat_safe(&name, ".", 1); + } + syslog_normalize_cat(rp, &name); + c++; + } + + return name; +} struct flb_syslog *flb_syslog_config_create(struct flb_output_instance *ins, struct flb_config *config) @@ -67,6 +135,9 @@ struct flb_syslog *flb_syslog_config_create(struct flb_output_instance *ins, int ret; const char *tmp; struct flb_syslog *ctx = NULL; + struct mk_list *head; + struct flb_config_map_val *mv; + struct flb_syslog_sd_key *sk_key_ra; /* Allocate plugin context */ ctx = flb_calloc(1, sizeof(struct flb_syslog)); @@ -83,8 +154,7 @@ struct flb_syslog *flb_syslog_config_create(struct flb_output_instance *ins, ret = flb_output_config_map_set(ins, (void *) ctx); if (ret == -1) { flb_plg_error(ctx->ins, "configuration error"); - flb_syslog_config_destroy(ctx); - return NULL; + goto error; } /* Set context */ @@ -104,8 +174,7 @@ struct flb_syslog *flb_syslog_config_create(struct flb_output_instance *ins, } else { flb_plg_error(ctx->ins, "unknown syslog mode %s", tmp); - flb_syslog_config_destroy(ctx); - return NULL; + goto error; } } @@ -120,8 +189,7 @@ struct flb_syslog *flb_syslog_config_create(struct flb_output_instance *ins, } else { flb_plg_error(ctx->ins, "unknown syslog format %s", tmp); - flb_syslog_config_destroy(ctx); - return NULL; + goto error; } } @@ -132,14 +200,12 @@ struct flb_syslog *flb_syslog_config_create(struct flb_output_instance *ins, /* validate preset values */ ret = is_valid_severity(ctx->ins, ctx->severity_preset, ctx->parsed_format); if (ret != 0) { - flb_syslog_config_destroy(ctx); - return NULL; + goto error; } ret = is_valid_facility(ctx->ins, ctx->facility_preset, ctx->parsed_format); if (ret != 0) { - flb_syslog_config_destroy(ctx); - return NULL; + goto error; } @@ -153,10 +219,168 @@ struct flb_syslog *flb_syslog_config_create(struct flb_output_instance *ins, } } + if (ctx->severity_key) { + ctx->ra_severity_key = flb_ra_create(ctx->severity_key, FLB_FALSE); + if (ctx->ra_severity_key == NULL) { + flb_plg_error(ins, "could not create record accessor for Severity Key"); + goto error; + } + } + + if (ctx->facility_key) { + ctx->ra_facility_key = flb_ra_create(ctx->facility_key, FLB_FALSE); + if (ctx->ra_facility_key == NULL) { + flb_plg_error(ins, "could not create record accessor for Facility Key"); + goto error; + } + } + + if (ctx->hostname_key) { + ctx->ra_hostname_key = flb_ra_create(ctx->hostname_key, FLB_FALSE); + if (ctx->ra_hostname_key == NULL) { + flb_plg_error(ins, "could not create record accessor for Hostname Key"); + goto error; + } + } + + if (ctx->appname_key) { + ctx->ra_appname_key = flb_ra_create(ctx->appname_key, FLB_FALSE); + if (ctx->ra_appname_key == NULL) { + flb_plg_error(ins, "could not create record accessor for Appname Key"); + goto error; + } + } + + if (ctx->procid_key) { + ctx->ra_procid_key = flb_ra_create(ctx->procid_key, FLB_FALSE); + if (ctx->ra_procid_key == NULL) { + flb_plg_error(ins, "could not create record accessor for Procid Key"); + goto error; + } + } + + if (ctx->msgid_key) { + ctx->ra_msgid_key = flb_ra_create(ctx->msgid_key, FLB_FALSE); + if (ctx->ra_msgid_key == NULL) { + flb_plg_error(ins, "could not create record accessor for Msgid Key"); + goto error; + } + } + + if (ctx->message_key) { + ctx->ra_message_key = flb_ra_create(ctx->message_key, FLB_FALSE); + if (ctx->ra_message_key == NULL) { + flb_plg_error(ins, "could not create record accessor for Message Key"); + goto error; + } + } + + if (ctx->sd_keys) { + /* Initialize sd_keys RAs */ + ctx->ra_sd_keys = flb_malloc(sizeof(struct mk_list)); + if (!ctx->ra_sd_keys) { + flb_errno(); + goto error; + } + mk_list_init(ctx->ra_sd_keys); + flb_config_map_foreach(head, mv, ctx->sd_keys) { + sk_key_ra = flb_malloc(sizeof(struct flb_syslog_sd_key)); + if (!sk_key_ra) { + flb_errno(); + goto error; + } + sk_key_ra->ra_sd_key = flb_ra_create(mv->val.str, FLB_FALSE); + if (sk_key_ra->ra_sd_key == NULL) { + flb_plg_error(ins, "could not create record accessor for SD Key %s", + mv->val.str); + flb_free(sk_key_ra); + goto error; + } + sk_key_ra->key_normalized = syslog_normalize_ra_key_name( + sk_key_ra->ra_sd_key); + if (sk_key_ra->key_normalized == NULL) { + flb_plg_error(ins, "could not normalize name for SD Key %s", + mv->val.str); + flb_ra_destroy(sk_key_ra->ra_sd_key); + flb_free(sk_key_ra); + goto error; + } + mk_list_add(&sk_key_ra->_head, ctx->ra_sd_keys); + } + } + return ctx; + +error: + flb_syslog_config_destroy(ctx); + return NULL; } void flb_syslog_config_destroy(struct flb_syslog *ctx) { + struct mk_list *head; + struct mk_list *tmp; + struct flb_syslog_sd_key *sd_key_item; + + if (!ctx) { + return; + } + + if (ctx->ra_severity_key) { + flb_ra_destroy(ctx->ra_severity_key); + ctx->ra_severity_key = NULL; + } + + if (ctx->ra_facility_key) { + flb_ra_destroy(ctx->ra_facility_key); + ctx->ra_facility_key = NULL; + } + + if (ctx->ra_hostname_key) { + flb_ra_destroy(ctx->ra_hostname_key); + ctx->ra_hostname_key = NULL; + } + + if (ctx->ra_appname_key) { + flb_ra_destroy(ctx->ra_appname_key); + ctx->ra_appname_key = NULL; + } + + if (ctx->ra_procid_key) { + flb_ra_destroy(ctx->ra_procid_key); + ctx->ra_procid_key = NULL; + } + + if (ctx->ra_msgid_key) { + flb_ra_destroy(ctx->ra_msgid_key); + ctx->ra_msgid_key = NULL; + } + + if (ctx->ra_sd_keys) { + mk_list_foreach_safe(head, tmp, ctx->ra_sd_keys) { + sd_key_item = mk_list_entry(head, struct flb_syslog_sd_key, _head); + + if (sd_key_item->ra_sd_key) { + flb_ra_destroy(sd_key_item->ra_sd_key); + sd_key_item->ra_sd_key = NULL; + } + + if (sd_key_item->key_normalized) { + flb_sds_destroy(sd_key_item->key_normalized); + sd_key_item->key_normalized = NULL; + } + + mk_list_del(&sd_key_item->_head); + flb_free(sd_key_item); + } + flb_free(ctx->ra_sd_keys); + ctx->ra_sd_keys = NULL; + } + + if (ctx->ra_message_key) { + flb_ra_destroy(ctx->ra_message_key); + ctx->ra_message_key = NULL; + } + flb_free(ctx); } diff --git a/plugins/out_syslog/syslog_conf.h b/plugins/out_syslog/syslog_conf.h index ab55f197842..e1c979203c8 100644 --- a/plugins/out_syslog/syslog_conf.h +++ b/plugins/out_syslog/syslog_conf.h @@ -23,6 +23,7 @@ #include #include #include +#include #define FLB_SYSLOG_UDP 0 @@ -32,22 +33,55 @@ #define FLB_SYSLOG_RFC3164 0 #define FLB_SYSLOG_RFC5424 1 +struct flb_syslog_sd_key { + struct flb_record_accessor *ra_sd_key; + flb_sds_t key_normalized; /* normalized key name when using ra */ + + struct mk_list _head; +}; + struct flb_syslog { flb_sockfd_t fd; struct flb_upstream *u; flb_sds_t mode; flb_sds_t format; size_t maxsize; + + /* severity_key */ flb_sds_t severity_key; + struct flb_record_accessor *ra_severity_key; + + /* facility_key */ flb_sds_t facility_key; + struct flb_record_accessor *ra_facility_key; + + /* timestamp_key */ flb_sds_t timestamp_key; + + /* hostname_key */ flb_sds_t hostname_key; + struct flb_record_accessor *ra_hostname_key; + + /* appname_key */ flb_sds_t appname_key; + struct flb_record_accessor *ra_appname_key; + + /* procid_key */ flb_sds_t procid_key; + struct flb_record_accessor *ra_procid_key; + + /* msgid_key */ flb_sds_t msgid_key; + struct flb_record_accessor *ra_msgid_key; + + /* sd_keys */ struct mk_list *sd_keys; int allow_longer_sd_id; + struct mk_list *ra_sd_keys; + + /* message_key */ flb_sds_t message_key; + struct flb_record_accessor *ra_message_key; /* Preset */ int severity_preset; diff --git a/tests/runtime/out_syslog.c b/tests/runtime/out_syslog.c index aa9edab33c7..507fc14a2f8 100644 --- a/tests/runtime/out_syslog.c +++ b/tests/runtime/out_syslog.c @@ -1608,6 +1608,65 @@ void flb_test_malformed_longer_sd_id_rfc5424() test_ctx_destroy(ctx); } +void flb_test_nested_keys_rfc5424() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"log.appname\":\"fluent-bit\", \"p_key\":\"1234\", \"nested_values\": {\"msg\":\"hello world\", \"hostname.key\":\"localhost\", \"sd_key\": {\"logtype\": \"access\",\"clustername\": \"mycluster\",\"namespace\": \"mynamespace\"}}}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", + "<14>1 1970-01-01T00:00:01.000000Z localhost fluent-bit 1234 - [nested_values.sd_key logtype=\"access\" clustername=\"mycluster\" namespace=\"mynamespace\"] hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc5424", + "syslog_message_key", "$nested_values['msg']", + "syslog_sd_key", "$nested_values['sd_key']", + "syslog_appname_key", "log.appname", + "syslog_procid_key", "$p_key", + "syslog_hostname_key", "$nested_values['hostname.key']", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + TEST_LIST = { /* rfc3164 */ /* procid_key, msgid_key, sd_key are not supported */ @@ -1639,6 +1698,7 @@ TEST_LIST = { {"format_msgid_preset_rfc5424", flb_test_msgid_preset_rfc5424}, {"allow_longer_sd_id_rfc5424", flb_test_allow_longer_sd_id_rfc5424}, {"malformed_longer_sd_id_rfc5424", flb_test_malformed_longer_sd_id_rfc5424}, + {"nested_values_rfc5424", flb_test_nested_keys_rfc5424}, {NULL, NULL} };