Skip to content

Commit 560b645

Browse files
bplessisbplessis
committed
out_syslog: handle multivalued structured data
When generating a syslog SD-ELEMENT it is permitted to repeat a SD-PARAM multiple times: https://datatracker.ietf.org/doc/html/rfc5424#section-6.3.3 This is used by loggly to pass tags alongside syslog messages for example https://documentation.solarwinds.com/en/success_center/loggly/content/admin/tags.htm This proposal convert array-typed object into multiple SD-PARAM elements to be able to satisfy this need. Signed-off-by: Benoit Plessis <[email protected]>
1 parent cd497e3 commit 560b645

File tree

2 files changed

+162
-84
lines changed

2 files changed

+162
-84
lines changed

‎plugins/out_syslog/syslog.c‎

Lines changed: 105 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -380,12 +380,109 @@ static flb_sds_t syslog_rfc3164 (flb_sds_t *s, struct flb_time *tms,
380380
return *s;
381381
}
382382

383+
static flb_sds_t msgpack_value_to_sd(struct flb_syslog *ctx,
384+
flb_sds_t *s, const char* key, int key_len,
385+
msgpack_object *v)
386+
{
387+
char temp[48] = {0};
388+
flb_sds_t tmp;
389+
int n, start_len, end_len;
390+
const char *val = NULL;
391+
int val_len = 0;
392+
393+
if (v->type == MSGPACK_OBJECT_BOOLEAN) {
394+
val = v->via.boolean ? "true" : "false";
395+
val_len = v->via.boolean ? 4 : 5;
396+
}
397+
else if (v->type == MSGPACK_OBJECT_POSITIVE_INTEGER) {
398+
val = temp;
399+
val_len = snprintf(temp, sizeof(temp) - 1,
400+
"%" PRIu64, v->via.u64);
401+
}
402+
else if (v->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) {
403+
val = temp;
404+
val_len = snprintf(temp, sizeof(temp) - 1,
405+
"%" PRId64, v->via.i64);
406+
}
407+
else if (v->type == MSGPACK_OBJECT_FLOAT) {
408+
val = temp;
409+
val_len = snprintf(temp, sizeof(temp) - 1,
410+
"%f", v->via.f64);
411+
}
412+
else if (v->type == MSGPACK_OBJECT_STR) {
413+
/* String value */
414+
val = v->via.str.ptr;
415+
val_len = v->via.str.size;
416+
}
417+
else if (v->type == MSGPACK_OBJECT_BIN) {
418+
/* Bin value */
419+
val = v->via.bin.ptr;
420+
val_len = v->via.bin.size;
421+
}
422+
423+
if (!val || !key) {
424+
return NULL;
425+
}
426+
427+
tmp = flb_sds_cat(*s, " " , 1);
428+
if (!tmp) {
429+
return NULL;
430+
}
431+
*s = tmp;
432+
433+
start_len = flb_sds_len(*s);
434+
if (ctx->allow_longer_sd_id != FLB_TRUE && key_len > 32 ) {
435+
/*
436+
* RFC5424 defines
437+
* PARAM-NAME = SD-NAME
438+
* SD-NAME = 1*32PRINTUSASCII
439+
* ; except '=', SP, ']', %d34 (")
440+
*
441+
* https://www.rfc-editor.org/rfc/rfc5424#section-6
442+
*/
443+
key_len = 32;
444+
}
445+
tmp = flb_sds_cat(*s, key, key_len);
446+
if (!tmp) {
447+
return NULL;
448+
}
449+
*s = tmp;
450+
451+
end_len = flb_sds_len(*s);
452+
for(n=start_len; n < end_len; n++) {
453+
if (!rfc5424_sp_name[(unsigned char)(*s)[n]]) {
454+
(*s)[n] = '_';
455+
}
456+
}
457+
458+
tmp = flb_sds_cat(*s, "=\"" , 2);
459+
if (!tmp) {
460+
return NULL;
461+
}
462+
*s = tmp;
463+
464+
tmp = flb_sds_cat_esc(*s, val , val_len,
465+
rfc5424_sp_value, sizeof(rfc5424_sp_value));
466+
if (!tmp) {
467+
return NULL;
468+
}
469+
*s = tmp;
470+
471+
tmp = flb_sds_cat(*s, "\"" , 1);
472+
if (!tmp) {
473+
return NULL;
474+
}
475+
*s = tmp;
476+
477+
return *s;
478+
}
479+
383480
static flb_sds_t msgpack_to_sd(struct flb_syslog *ctx,
384481
flb_sds_t *s, const char *sd, int sd_len,
385482
msgpack_object *o)
386483
{
387484
flb_sds_t tmp;
388-
int i;
485+
int i, idx;
389486
int loop;
390487
int n, start_len, end_len;
391488

@@ -430,11 +527,8 @@ static flb_sds_t msgpack_to_sd(struct flb_syslog *ctx,
430527
if (loop != 0) {
431528
msgpack_object_kv *p = o->via.map.ptr;
432529
for (i = 0; i < loop; i++) {
433-
char temp[48] = {0};
434530
const char *key = NULL;
435531
int key_len = 0;
436-
const char *val = NULL;
437-
int val_len = 0;
438532

439533
msgpack_object *k = &p[i].key;
440534
msgpack_object *v = &p[i].val;
@@ -452,89 +546,16 @@ static flb_sds_t msgpack_to_sd(struct flb_syslog *ctx,
452546
key_len = k->via.bin.size;
453547
}
454548

455-
if (v->type == MSGPACK_OBJECT_BOOLEAN) {
456-
val = v->via.boolean ? "true" : "false";
457-
val_len = v->via.boolean ? 4 : 5;
458-
}
459-
else if (v->type == MSGPACK_OBJECT_POSITIVE_INTEGER) {
460-
val = temp;
461-
val_len = snprintf(temp, sizeof(temp) - 1,
462-
"%" PRIu64, v->via.u64);
463-
}
464-
else if (v->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) {
465-
val = temp;
466-
val_len = snprintf(temp, sizeof(temp) - 1,
467-
"%" PRId64, v->via.i64);
468-
}
469-
else if (v->type == MSGPACK_OBJECT_FLOAT) {
470-
val = temp;
471-
val_len = snprintf(temp, sizeof(temp) - 1,
472-
"%f", v->via.f64);
473-
}
474-
else if (v->type == MSGPACK_OBJECT_STR) {
475-
/* String value */
476-
val = v->via.str.ptr;
477-
val_len = v->via.str.size;
478-
}
479-
else if (v->type == MSGPACK_OBJECT_BIN) {
480-
/* Bin value */
481-
val = v->via.bin.ptr;
482-
val_len = v->via.bin.size;
483-
}
484-
485-
if (!val || !key) {
486-
continue;
487-
}
488-
489-
tmp = flb_sds_cat(*s, " " , 1);
490-
if (!tmp) {
491-
return NULL;
492-
}
493-
*s = tmp;
494-
495-
start_len = flb_sds_len(*s);
496-
if (ctx->allow_longer_sd_id != FLB_TRUE && key_len > 32 ) {
497-
/*
498-
* RFC5424 defines
499-
* PARAM-NAME = SD-NAME
500-
* SD-NAME = 1*32PRINTUSASCII
501-
* ; except '=', SP, ']', %d34 (")
502-
*
503-
* https://www.rfc-editor.org/rfc/rfc5424#section-6
504-
*/
505-
key_len = 32;
506-
}
507-
tmp = flb_sds_cat(*s, key, key_len);
508-
if (!tmp) {
509-
return NULL;
510-
}
511-
*s = tmp;
512-
513-
end_len = flb_sds_len(*s);
514-
for(n=start_len; n < end_len; n++) {
515-
if (!rfc5424_sp_name[(unsigned char)(*s)[n]]) {
516-
(*s)[n] = '_';
549+
if (v->type == MSGPACK_OBJECT_ARRAY) {
550+
for (idx = 0 ;
551+
idx < v->via.array.size;
552+
idx++) {
553+
msgpack_value_to_sd(ctx, s, key, key_len, &v->via.array.ptr[idx]);
517554
}
518555
}
519-
520-
tmp = flb_sds_cat(*s, "=\"" , 2);
521-
if (!tmp) {
522-
return NULL;
523-
}
524-
*s = tmp;
525-
526-
tmp = flb_sds_cat_esc(*s, val , val_len,
527-
rfc5424_sp_value, sizeof(rfc5424_sp_value));
528-
if (!tmp) {
529-
return NULL;
530-
}
531-
*s = tmp;
532-
533-
tmp = flb_sds_cat(*s, "\"" , 1);
534-
if (!tmp) {
535-
return NULL;
556+
else {
557+
msgpack_value_to_sd(ctx, s, key, key_len, v);
536558
}
537-
*s = tmp;
538559
}
539560
}
540561

‎tests/runtime/out_syslog.c‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1608,6 +1608,62 @@ void flb_test_malformed_longer_sd_id_rfc5424()
16081608
test_ctx_destroy(ctx);
16091609
}
16101610

1611+
void flb_test_sd_key_multivalued_param_rfc5424()
1612+
{
1613+
struct test_ctx *ctx;
1614+
int ret;
1615+
int num;
1616+
1617+
char *buf = "[1, {\"msg\":\"hello world\", \"sd_key\": {\"logtype\": \"access\",\"clustername\": \"mycluster\",\"namespace\": \"mynamespace\", \"tag\": [ \"tag1\", \"tag2\", \"tag3\" ]}}]";
1618+
size_t size = strlen(buf);
1619+
1620+
char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z",
1621+
"<14>1 1970-01-01T00:00:01.000000Z - - - - [sd_key logtype=\"access\" clustername=\"mycluster\" namespace=\"mynamespace\" tag=\"tag1\" tag=\"tag2\" tag=\"tag3\"] hello world"};
1622+
struct str_list expected = {
1623+
.size = sizeof(expected_strs)/sizeof(char*),
1624+
.lists = &expected_strs[0],
1625+
};
1626+
1627+
clear_output_num();
1628+
1629+
ctx = test_ctx_create();
1630+
if (!TEST_CHECK(ctx != NULL)) {
1631+
TEST_MSG("test_ctx_create failed");
1632+
exit(EXIT_FAILURE);
1633+
}
1634+
1635+
ret = flb_output_set(ctx->flb, ctx->o_ffd,
1636+
"match", "*",
1637+
"syslog_format", "rfc5424",
1638+
"syslog_message_key", "msg",
1639+
"syslog_sd_key", "sd_key",
1640+
NULL);
1641+
TEST_CHECK(ret == 0);
1642+
1643+
ret = flb_output_set_test(ctx->flb, ctx->o_ffd,
1644+
"formatter", cb_check_str_list,
1645+
&expected, NULL);
1646+
TEST_CHECK(ret == 0);
1647+
1648+
/* Start the engine */
1649+
ret = flb_start(ctx->flb);
1650+
TEST_CHECK(ret == 0);
1651+
1652+
/* Ingest data sample */
1653+
ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size);
1654+
TEST_CHECK(ret >= 0);
1655+
1656+
/* waiting to flush */
1657+
flb_time_msleep(500);
1658+
1659+
num = get_output_num();
1660+
if (!TEST_CHECK(num > 0)) {
1661+
TEST_MSG("no outputs");
1662+
}
1663+
1664+
test_ctx_destroy(ctx);
1665+
}
1666+
16111667
TEST_LIST = {
16121668
/* rfc3164 */
16131669
/* procid_key, msgid_key, sd_key are not supported */
@@ -1639,6 +1695,7 @@ TEST_LIST = {
16391695
{"format_msgid_preset_rfc5424", flb_test_msgid_preset_rfc5424},
16401696
{"allow_longer_sd_id_rfc5424", flb_test_allow_longer_sd_id_rfc5424},
16411697
{"malformed_longer_sd_id_rfc5424", flb_test_malformed_longer_sd_id_rfc5424},
1698+
{"format_sd_key_multivalued_rfc5424", flb_test_sd_key_multivalued_param_rfc5424},
16421699
{NULL, NULL}
16431700
};
16441701

0 commit comments

Comments
 (0)