Skip to content

out_syslog: handle multivalued structured data #8355

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

out_syslog: handle multivalued structured data #8355

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
189 changes: 105 additions & 84 deletions plugins/out_syslog/syslog.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -380,12 +380,109 @@ static flb_sds_t syslog_rfc3164 (flb_sds_t *s, struct flb_time *tms,
return *s;
}

static flb_sds_t msgpack_value_to_sd(struct flb_syslog *ctx,
flb_sds_t *s, const char* key, int key_len,
msgpack_object *v)
{
char temp[48] = {0};
flb_sds_t tmp;
int n, start_len, end_len;
const char *val = NULL;
int val_len = 0;

if (v->type == MSGPACK_OBJECT_BOOLEAN) {
val = v->via.boolean ? "true" : "false";
val_len = v->via.boolean ? 4 : 5;
}
else if (v->type == MSGPACK_OBJECT_POSITIVE_INTEGER) {
val = temp;
val_len = snprintf(temp, sizeof(temp) - 1,
"%" PRIu64, v->via.u64);
}
else if (v->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) {
val = temp;
val_len = snprintf(temp, sizeof(temp) - 1,
"%" PRId64, v->via.i64);
}
else if (v->type == MSGPACK_OBJECT_FLOAT) {
val = temp;
val_len = snprintf(temp, sizeof(temp) - 1,
"%f", v->via.f64);
}
else if (v->type == MSGPACK_OBJECT_STR) {
/* String value */
val = v->via.str.ptr;
val_len = v->via.str.size;
}
else if (v->type == MSGPACK_OBJECT_BIN) {
/* Bin value */
val = v->via.bin.ptr;
val_len = v->via.bin.size;
}

if (!val || !key) {
return NULL;
}

tmp = flb_sds_cat(*s, " " , 1);
if (!tmp) {
return NULL;
}
*s = tmp;

start_len = flb_sds_len(*s);
if (ctx->allow_longer_sd_id != FLB_TRUE && key_len > 32 ) {
/*
* RFC5424 defines
* PARAM-NAME = SD-NAME
* SD-NAME = 1*32PRINTUSASCII
* ; except '=', SP, ']', %d34 (")
*
* https://www.rfc-editor.org/rfc/rfc5424#section-6
*/
key_len = 32;
}
tmp = flb_sds_cat(*s, key, key_len);
if (!tmp) {
return NULL;
}
*s = tmp;

end_len = flb_sds_len(*s);
for(n=start_len; n < end_len; n++) {
if (!rfc5424_sp_name[(unsigned char)(*s)[n]]) {
(*s)[n] = '_';
}
}

tmp = flb_sds_cat(*s, "=\"" , 2);
if (!tmp) {
return NULL;
}
*s = tmp;

tmp = flb_sds_cat_esc(*s, val , val_len,
rfc5424_sp_value, sizeof(rfc5424_sp_value));
if (!tmp) {
return NULL;
}
*s = tmp;

tmp = flb_sds_cat(*s, "\"" , 1);
if (!tmp) {
return NULL;
}
*s = tmp;

return *s;
}

static flb_sds_t msgpack_to_sd(struct flb_syslog *ctx,
flb_sds_t *s, const char *sd, int sd_len,
msgpack_object *o)
{
flb_sds_t tmp;
int i;
int i, idx;
int loop;
int n, start_len, end_len;

Expand Down Expand Up @@ -430,11 +527,8 @@ static flb_sds_t msgpack_to_sd(struct flb_syslog *ctx,
if (loop != 0) {
msgpack_object_kv *p = o->via.map.ptr;
for (i = 0; i < loop; i++) {
char temp[48] = {0};
const char *key = NULL;
int key_len = 0;
const char *val = NULL;
int val_len = 0;

msgpack_object *k = &p[i].key;
msgpack_object *v = &p[i].val;
Expand All @@ -452,89 +546,16 @@ static flb_sds_t msgpack_to_sd(struct flb_syslog *ctx,
key_len = k->via.bin.size;
}

if (v->type == MSGPACK_OBJECT_BOOLEAN) {
val = v->via.boolean ? "true" : "false";
val_len = v->via.boolean ? 4 : 5;
}
else if (v->type == MSGPACK_OBJECT_POSITIVE_INTEGER) {
val = temp;
val_len = snprintf(temp, sizeof(temp) - 1,
"%" PRIu64, v->via.u64);
}
else if (v->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) {
val = temp;
val_len = snprintf(temp, sizeof(temp) - 1,
"%" PRId64, v->via.i64);
}
else if (v->type == MSGPACK_OBJECT_FLOAT) {
val = temp;
val_len = snprintf(temp, sizeof(temp) - 1,
"%f", v->via.f64);
}
else if (v->type == MSGPACK_OBJECT_STR) {
/* String value */
val = v->via.str.ptr;
val_len = v->via.str.size;
}
else if (v->type == MSGPACK_OBJECT_BIN) {
/* Bin value */
val = v->via.bin.ptr;
val_len = v->via.bin.size;
}

if (!val || !key) {
continue;
}

tmp = flb_sds_cat(*s, " " , 1);
if (!tmp) {
return NULL;
}
*s = tmp;

start_len = flb_sds_len(*s);
if (ctx->allow_longer_sd_id != FLB_TRUE && key_len > 32 ) {
/*
* RFC5424 defines
* PARAM-NAME = SD-NAME
* SD-NAME = 1*32PRINTUSASCII
* ; except '=', SP, ']', %d34 (")
*
* https://www.rfc-editor.org/rfc/rfc5424#section-6
*/
key_len = 32;
}
tmp = flb_sds_cat(*s, key, key_len);
if (!tmp) {
return NULL;
}
*s = tmp;

end_len = flb_sds_len(*s);
for(n=start_len; n < end_len; n++) {
if (!rfc5424_sp_name[(unsigned char)(*s)[n]]) {
(*s)[n] = '_';
if (v->type == MSGPACK_OBJECT_ARRAY) {
for (idx = 0 ;
idx < v->via.array.size;
idx++) {
msgpack_value_to_sd(ctx, s, key, key_len, &v->via.array.ptr[idx]);
}
}

tmp = flb_sds_cat(*s, "=\"" , 2);
if (!tmp) {
return NULL;
}
*s = tmp;

tmp = flb_sds_cat_esc(*s, val , val_len,
rfc5424_sp_value, sizeof(rfc5424_sp_value));
if (!tmp) {
return NULL;
}
*s = tmp;

tmp = flb_sds_cat(*s, "\"" , 1);
if (!tmp) {
return NULL;
else {
msgpack_value_to_sd(ctx, s, key, key_len, v);
}
*s = tmp;
}
}

Expand Down
57 changes: 57 additions & 0 deletions tests/runtime/out_syslog.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1608,6 +1608,62 @@ void flb_test_malformed_longer_sd_id_rfc5424()
test_ctx_destroy(ctx);
}

void flb_test_sd_key_multivalued_param_rfc5424()
{
struct test_ctx *ctx;
int ret;
int num;

char *buf = "[1, {\"msg\":\"hello world\", \"sd_key\": {\"logtype\": \"access\",\"clustername\": \"mycluster\",\"namespace\": \"mynamespace\", \"tag\": [ \"tag1\", \"tag2\", \"tag3\" ]}}]";
size_t size = strlen(buf);

char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z",
"<14>1 1970-01-01T00:00:01.000000Z - - - - [sd_key logtype=\"access\" clustername=\"mycluster\" namespace=\"mynamespace\" tag=\"tag1\" tag=\"tag2\" tag=\"tag3\"] hello world"};
struct str_list expected = {
.size = sizeof(expected_strs)/sizeof(char*),
.lists = &expected_strs[0],
};

clear_output_num();

ctx = test_ctx_create();
if (!TEST_CHECK(ctx != NULL)) {
TEST_MSG("test_ctx_create failed");
exit(EXIT_FAILURE);
}

ret = flb_output_set(ctx->flb, ctx->o_ffd,
"match", "*",
"syslog_format", "rfc5424",
"syslog_message_key", "msg",
"syslog_sd_key", "sd_key",
NULL);
TEST_CHECK(ret == 0);

ret = flb_output_set_test(ctx->flb, ctx->o_ffd,
"formatter", cb_check_str_list,
&expected, NULL);
TEST_CHECK(ret == 0);

/* Start the engine */
ret = flb_start(ctx->flb);
TEST_CHECK(ret == 0);

/* Ingest data sample */
ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size);
TEST_CHECK(ret >= 0);

/* waiting to flush */
flb_time_msleep(500);

num = get_output_num();
if (!TEST_CHECK(num > 0)) {
TEST_MSG("no outputs");
}

test_ctx_destroy(ctx);
}

TEST_LIST = {
/* rfc3164 */
/* procid_key, msgid_key, sd_key are not supported */
Expand Down Expand Up @@ -1639,6 +1695,7 @@ TEST_LIST = {
{"format_msgid_preset_rfc5424", flb_test_msgid_preset_rfc5424},
{"allow_longer_sd_id_rfc5424", flb_test_allow_longer_sd_id_rfc5424},
{"malformed_longer_sd_id_rfc5424", flb_test_malformed_longer_sd_id_rfc5424},
{"format_sd_key_multivalued_rfc5424", flb_test_sd_key_multivalued_param_rfc5424},
{NULL, NULL}
};