Skip to content

Commit 4b27323

Browse files
bluebikebluebike
committed
core: and flb_iconv faculty to do charset encodings
Make possible to have other encodings that UTF-8 in fluent-bit, by creating object flb_iconv-library. Enabled by -D FLB_ICONV=Yes (default No) Requires libiconv-library (can be embedded to libc) Signed-off-by: Jukka Pihl <[email protected]>
1 parent c5ce877 commit 4b27323

File tree

4 files changed

+267
-0
lines changed

4 files changed

+267
-0
lines changed

‎CMakeLists.txt‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ option(FLB_SYSTEM_STRPTIME "Use strptime in system libc" Yes)
6868
option(FLB_STATIC_CONF "Build binary using static configuration")
6969
option(FLB_STREAM_PROCESSOR "Enable Stream Processor" Yes)
7070
option(FLB_CORO_STACK_SIZE "Set coroutine stack size")
71+
option(FLB_ICONV "Enable iconv string decoding" No)
7172

7273
# Metrics: Experimental Feature, disabled by default on 0.12 series
7374
# but enabled in the upcoming 0.13 release. Note that development
@@ -153,6 +154,7 @@ if(FLB_ALL)
153154
# Global
154155
set(FLB_DEBUG 1)
155156
set(FLB_TLS 1)
157+
set(FLB_ICONV 1)
156158

157159
# Input plugins
158160
set(FLB_IN_CPU 1)
@@ -182,6 +184,7 @@ if(FLB_ALL)
182184
set(FLB_OUT_STDOUT 1)
183185
set(FLB_OUT_LIB 1)
184186
set(FLB_OUT_FLOWCOUNTER 1)
187+
185188
endif()
186189

187190
if(FLB_DEV)
@@ -577,6 +580,67 @@ if(FLB_INOTIFY)
577580
endif()
578581
endif()
579582

583+
# iconv
584+
if(FLB_ICONV)
585+
586+
if(CMAKE_VERSION VERSION_LESS 3.11.0)
587+
message("cmake older than 3.11.0 .. no FindIconv module... testing here")
588+
if(NOT DEFINED Iconv_IS_BUILT_IN)
589+
if(DEFINED Iconv_INCLUDE_DIR AND NOT DEFINED Iconv_LIBRARY)
590+
check_c_source_compiles(
591+
"
592+
#include <stddef.h>
593+
#include <iconv.h>
594+
int main() {
595+
char *a, *b;
596+
size_t i, j;
597+
iconv_t ic;
598+
ic = iconv_open(\"to\", \"from\");
599+
iconv(ic, &a, &i, &b, &j);
600+
iconv_close(ic);
601+
}
602+
" Iconv_IS_BUILT_IN)
603+
endif()
604+
endif()
605+
if(NOT Iconv_IS_BUILT_IN)
606+
if(DEFINED Iconv_INCLUDE_DIR AND NOT DEFINED Iconv_LIBRARIES)
607+
set(CMAKE_REQUIRED_LIBRARIES "iconv")
608+
check_c_source_compiles(
609+
"
610+
#include <stddef.h>
611+
#include <iconv.h>
612+
int main() {
613+
char *a, *b;
614+
size_t i, j;
615+
iconv_t ic;
616+
ic = iconv_open(\"to\", \"from\");
617+
iconv(ic, &a, &i, &b, &j);
618+
iconv_close(ic);
619+
}
620+
" Iconv_FOUND)
621+
if(Iconv_FOUND)
622+
set(Iconv_LIBRARIES "iconv")
623+
endif()
624+
endif()
625+
endif()
626+
627+
else()
628+
include(FindIconv)
629+
endif()
630+
631+
if(Iconv_IS_BUILT_IN)
632+
message("Iconv_IS_BUILT_IN builtin is defined")
633+
FLB_DEFINITION(FLB_HAVE_ICONV)
634+
elseif(Iconv_LIBRARIES)
635+
message("Iconv_LIBRARIES defined '${Iconv_LIBRARIES}'")
636+
FLB_DEFINITION(FLB_HAVE_ICONV)
637+
else()
638+
message("FLB_ICONV defined iconv not given")
639+
set(FLB_ICONV NO)
640+
endif()
641+
642+
endif()
643+
580644
configure_file(
581645
"${PROJECT_SOURCE_DIR}/include/fluent-bit/flb_info.h.in"
582646
"${PROJECT_SOURCE_DIR}/include/fluent-bit/flb_info.h"

‎include/fluent-bit/flb_iconv.h‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2+
3+
/* Fluent Bit
4+
* ==========
5+
* Copyright (C) 2019 The Fluent Bit Authors
6+
* Copyright (C) 2015-2018 Treasure Data Inc.
7+
*
8+
* Licensed under the Apache License, Version 2.0 (the "License");
9+
* you may not use this file except in compliance with the License.
10+
* You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing, software
15+
* distributed under the License is distributed on an "AS IS" BASIS,
16+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
* See the License for the specific language governing permissions and
18+
* limitations under the License.
19+
*/
20+
21+
#ifndef FLB_ICONV_H
22+
#define FLB_ICONV_H
23+
24+
#include <iconv.h>
25+
26+
27+
#define FLB_ICONV_SUCCESS 0
28+
#define FLB_ICONV_NOT_CHANGED 1
29+
#define FLB_ICONV_FAILURE -1
30+
31+
#define FLB_ICONV_ACCEPT_NOT_CHANGED 0x01
32+
33+
struct flb_iconv {
34+
iconv_t conv;
35+
};
36+
37+
struct flb_iconv *flb_iconv_open(char *to, char *from);
38+
39+
int flb_iconv_execute(struct flb_iconv *ic,
40+
char *str, size_t slen,
41+
char **result, size_t *result_len,
42+
unsigned int flags);
43+
44+
void flb_iconv_close(struct flb_iconv *ic);
45+
46+
#endif

‎src/CMakeLists.txt‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,20 @@ if(FLB_TLS)
7878
)
7979
endif()
8080

81+
# iconv
82+
if(FLB_ICONV)
83+
set(src
84+
${src}
85+
"flb_iconv.c"
86+
)
87+
if(NOT Iconv_IS_BUILT_IN)
88+
set(extra_libs
89+
${extra_libs}
90+
${Iconv_LIBRARIES}
91+
)
92+
endif()
93+
endif()
94+
8195
if(FLB_PROXY_GO)
8296
set(src
8397
${src}

‎src/flb_iconv.c‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2+
3+
/* Fluent Bit
4+
* ==========
5+
* Copyright (C) 2019 The Fluent Bit Authors
6+
* Copyright (C) 2015-2018 Treasure Data Inc.
7+
*
8+
* Licensed under the Apache License, Version 2.0 (the "License");
9+
* you may not use this file except in compliance with the License.
10+
* You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing, software
15+
* distributed under the License is distributed on an "AS IS" BASIS,
16+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
* See the License for the specific language governing permissions and
18+
* limitations under the License.
19+
*/
20+
21+
#include <stdio.h>
22+
#include <iconv.h>
23+
#include <string.h>
24+
#include <time.h>
25+
#include <ctype.h>
26+
27+
#include <fluent-bit/flb_macros.h>
28+
#include <fluent-bit/flb_config.h>
29+
#include <fluent-bit/flb_error.h>
30+
#include <fluent-bit/flb_iconv.h>
31+
#include <fluent-bit/flb_mem.h>
32+
33+
struct flb_iconv *flb_iconv_open(char *to, char *from) {
34+
iconv_t ic;
35+
struct flb_iconv *c;
36+
ic = iconv_open(to,from);
37+
if (ic == (iconv_t) -1) {
38+
return NULL;
39+
}
40+
c = flb_calloc(sizeof(struct flb_iconv),1);
41+
c->conv = ic;
42+
return c;
43+
}
44+
45+
void flb_iconv_close(struct flb_iconv *c) {
46+
if(c) {
47+
iconv_close(c->conv);
48+
flb_free(c);
49+
}
50+
}
51+
52+
53+
int flb_iconv_execute(struct flb_iconv *c,
54+
char *str, size_t slen,
55+
char **result, size_t *result_len,
56+
unsigned int flags)
57+
{
58+
char *outbuf;
59+
char *inp;
60+
char *outp;
61+
size_t outlen;
62+
size_t outroom;
63+
size_t outleft;
64+
size_t ret;
65+
size_t inleft;
66+
size_t outdone;
67+
68+
if(slen == 0) {
69+
if(flags & FLB_ICONV_ACCEPT_NOT_CHANGED) {
70+
*result = NULL;
71+
*result_len = 0;
72+
return FLB_ICONV_NOT_CHANGED;
73+
} else {
74+
outp = flb_malloc(1);
75+
if(outp == NULL) {
76+
return FLB_ICONV_FAILURE;
77+
}
78+
*result = outp;
79+
*result_len = 0;
80+
return FLB_ICONV_SUCCESS;
81+
}
82+
}
83+
84+
inp = str;
85+
inleft = slen;
86+
87+
outroom = slen + (slen / 2) + 3; // just add something
88+
outbuf = flb_malloc(outroom);
89+
if(outbuf == NULL) {
90+
return FLB_ICONV_FAILURE;
91+
}
92+
outp = outbuf;
93+
outlen = 0;
94+
outleft = outroom - 1;
95+
96+
iconv(c->conv, NULL, NULL, NULL, NULL);
97+
98+
while(inleft > 0) {
99+
ret = iconv(c->conv, &inp, &inleft, &outp, &outleft);
100+
if(ret == -1) {
101+
switch(errno) {
102+
case EILSEQ: // bad input sequence of char (ignore)
103+
flb_warn("[flb_iconv] bad input char 0x%02x", *inp & 0xff);
104+
inp++;
105+
inleft--;
106+
break;
107+
case EINVAL: // imcomplete char (ignore)
108+
flb_warn("[flb_iconv] incomplete char 0x%02x", *inp & 0xff);
109+
inp++;
110+
inleft--;
111+
break;
112+
113+
case E2BIG:
114+
outdone = outp - outbuf;
115+
outroom = outroom * 2;
116+
outbuf = flb_realloc(outbuf, outroom);
117+
if(outbuf == NULL) {
118+
return FLB_ICONV_FAILURE;
119+
}
120+
outp = outbuf + outdone;
121+
outleft = outroom - outdone - 1;
122+
break;
123+
default:
124+
flb_error("[flb_iconv] unknown error: %d", errno);
125+
break;
126+
}
127+
} else {
128+
break;
129+
}
130+
}
131+
outdone = outp - outbuf;
132+
if((flags & FLB_ICONV_ACCEPT_NOT_CHANGED) && (slen == outdone) && (memcmp(str,outbuf,slen) == 0)) {
133+
flb_free(outbuf);
134+
*result = NULL;
135+
*result_len = 0;
136+
return FLB_ICONV_NOT_CHANGED;
137+
} else {
138+
*result = outbuf;
139+
*result_len = outdone;
140+
return FLB_ICONV_SUCCESS;
141+
}
142+
}
143+

0 commit comments

Comments
 (0)