Skip to content

Commit d17c93f

Browse files
bluebikebluebike
committed
core: and flb_iconv faculty to do charset encodings
Make possible to have other encodings that UTF-8 in fluent-bit, by creating object flb_iconv-library. Enabled by -D FLB_ICONV=Yes (default No) Requires libiconv-library (can be embedded to libc) Signed-off-by: Jukka Pihl <[email protected]>
1 parent c845804 commit d17c93f

File tree

4 files changed

+267
-0
lines changed

4 files changed

+267
-0
lines changed

‎CMakeLists.txt‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ option(FLB_SYSTEM_STRPTIME "Use strptime in system libc" Yes)
6767
option(FLB_STATIC_CONF "Build binary using static configuration")
6868
option(FLB_STREAM_PROCESSOR "Enable Stream Processor" Yes)
6969
option(FLB_CORO_STACK_SIZE "Set coroutine stack size")
70+
option(FLB_ICONV "Enable iconv string decoding" No)
7071

7172
# Metrics: Experimental Feature, disabled by default on 0.12 series
7273
# but enabled in the upcoming 0.13 release. Note that development
@@ -150,6 +151,7 @@ if(FLB_ALL)
150151
# Global
151152
set(FLB_DEBUG 1)
152153
set(FLB_TLS 1)
154+
set(FLB_ICONV 1)
153155

154156
# Input plugins
155157
set(FLB_IN_CPU 1)
@@ -179,6 +181,7 @@ if(FLB_ALL)
179181
set(FLB_OUT_STDOUT 1)
180182
set(FLB_OUT_LIB 1)
181183
set(FLB_OUT_FLOWCOUNTER 1)
184+
182185
endif()
183186

184187
if(FLB_DEV)
@@ -593,6 +596,67 @@ if(FLB_INOTIFY)
593596
endif()
594597
endif()
595598

599+
# iconv
600+
if(FLB_ICONV)
601+
602+
if(CMAKE_VERSION VERSION_LESS 3.11.0)
603+
message("cmake older than 3.11.0 .. no FindIconv module... testing here")
604+
if(NOT DEFINED Iconv_IS_BUILT_IN)
605+
if(DEFINED Iconv_INCLUDE_DIR AND NOT DEFINED Iconv_LIBRARY)
606+
check_c_source_compiles(
607+
"
608+
#include <stddef.h>
609+
#include <iconv.h>
610+
int main() {
611+
char *a, *b;
612+
size_t i, j;
613+
iconv_t ic;
614+
ic = iconv_open(\"to\", \"from\");
615+
iconv(ic, &a, &i, &b, &j);
616+
iconv_close(ic);
617+
}
618+
" Iconv_IS_BUILT_IN)
619+
endif()
620+
endif()
621+
if(NOT Iconv_IS_BUILT_IN)
622+
if(DEFINED Iconv_INCLUDE_DIR AND NOT DEFINED Iconv_LIBRARIES)
623+
set(CMAKE_REQUIRED_LIBRARIES "iconv")
624+
check_c_source_compiles(
625+
"
626+
#include <stddef.h>
627+
#include <iconv.h>
628+
int main() {
629+
char *a, *b;
630+
size_t i, j;
631+
iconv_t ic;
632+
ic = iconv_open(\"to\", \"from\");
633+
iconv(ic, &a, &i, &b, &j);
634+
iconv_close(ic);
635+
}
636+
" Iconv_FOUND)
637+
if(Iconv_FOUND)
638+
set(Iconv_LIBRARIES "iconv")
639+
endif()
640+
endif()
641+
endif()
642+
643+
else()
644+
include(FindIconv)
645+
endif()
646+
647+
if(Iconv_IS_BUILT_IN)
648+
message("Iconv_IS_BUILT_IN builtin is defined")
649+
FLB_DEFINITION(FLB_HAVE_ICONV)
650+
elseif(Iconv_LIBRARIES)
651+
message("Iconv_LIBRARIES defined '${Iconv_LIBRARIES}'")
652+
FLB_DEFINITION(FLB_HAVE_ICONV)
653+
else()
654+
message("FLB_ICONV defined iconv not given")
655+
set(FLB_ICONV NO)
656+
endif()
657+
658+
endif()
659+
596660
configure_file(
597661
"${PROJECT_SOURCE_DIR}/include/fluent-bit/flb_info.h.in"
598662
"${PROJECT_SOURCE_DIR}/include/fluent-bit/flb_info.h"

‎include/fluent-bit/flb_iconv.h‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2+
3+
/* Fluent Bit
4+
* ==========
5+
* Copyright (C) 2019 The Fluent Bit Authors
6+
* Copyright (C) 2015-2018 Treasure Data Inc.
7+
*
8+
* Licensed under the Apache License, Version 2.0 (the "License");
9+
* you may not use this file except in compliance with the License.
10+
* You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing, software
15+
* distributed under the License is distributed on an "AS IS" BASIS,
16+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
* See the License for the specific language governing permissions and
18+
* limitations under the License.
19+
*/
20+
21+
#ifndef FLB_ICONV_H
22+
#define FLB_ICONV_H
23+
24+
#include <iconv.h>
25+
26+
27+
#define FLB_ICONV_SUCCESS 0
28+
#define FLB_ICONV_NOT_CHANGED 1
29+
#define FLB_ICONV_FAILURE -1
30+
31+
#define FLB_ICONV_ACCEPT_NOT_CHANGED 0x01
32+
33+
struct flb_iconv {
34+
iconv_t conv;
35+
};
36+
37+
struct flb_iconv *flb_iconv_open(char *to, char *from);
38+
39+
int flb_iconv_execute(struct flb_iconv *ic,
40+
char *str, size_t slen,
41+
char **result, size_t *result_len,
42+
unsigned int flags);
43+
44+
void flb_iconv_close(struct flb_iconv *ic);
45+
46+
#endif

‎src/CMakeLists.txt‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,20 @@ if(FLB_TLS)
7676
)
7777
endif()
7878

79+
# iconv
80+
if(FLB_ICONV)
81+
set(src
82+
${src}
83+
"flb_iconv.c"
84+
)
85+
if(NOT Iconv_IS_BUILT_IN)
86+
set(extra_libs
87+
${extra_libs}
88+
${Iconv_LIBRARIES}
89+
)
90+
endif()
91+
endif()
92+
7993
if(FLB_PROXY_GO)
8094
set(src
8195
${src}

‎src/flb_iconv.c‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2+
3+
/* Fluent Bit
4+
* ==========
5+
* Copyright (C) 2019 The Fluent Bit Authors
6+
* Copyright (C) 2015-2018 Treasure Data Inc.
7+
*
8+
* Licensed under the Apache License, Version 2.0 (the "License");
9+
* you may not use this file except in compliance with the License.
10+
* You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing, software
15+
* distributed under the License is distributed on an "AS IS" BASIS,
16+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
* See the License for the specific language governing permissions and
18+
* limitations under the License.
19+
*/
20+
21+
#include <stdio.h>
22+
#include <iconv.h>
23+
#include <string.h>
24+
#include <time.h>
25+
#include <ctype.h>
26+
27+
#include <fluent-bit/flb_macros.h>
28+
#include <fluent-bit/flb_config.h>
29+
#include <fluent-bit/flb_error.h>
30+
#include <fluent-bit/flb_iconv.h>
31+
#include <fluent-bit/flb_mem.h>
32+
33+
struct flb_iconv *flb_iconv_open(char *to, char *from) {
34+
iconv_t ic;
35+
struct flb_iconv *c;
36+
ic = iconv_open(to,from);
37+
if (ic == (iconv_t) -1) {
38+
return NULL;
39+
}
40+
c = flb_calloc(sizeof(struct flb_iconv),1);
41+
c->conv = ic;
42+
return c;
43+
}
44+
45+
void flb_iconv_close(struct flb_iconv *c) {
46+
if(c) {
47+
iconv_close(c->conv);
48+
flb_free(c);
49+
}
50+
}
51+
52+
53+
int flb_iconv_execute(struct flb_iconv *c,
54+
char *str, size_t slen,
55+
char **result, size_t *result_len,
56+
unsigned int flags)
57+
{
58+
char *outbuf;
59+
char *inp;
60+
char *outp;
61+
size_t outlen;
62+
size_t outroom;
63+
size_t outleft;
64+
size_t ret;
65+
size_t inleft;
66+
size_t outdone;
67+
68+
if(slen == 0) {
69+
if(flags & FLB_ICONV_ACCEPT_NOT_CHANGED) {
70+
*result = NULL;
71+
*result_len = 0;
72+
return FLB_ICONV_NOT_CHANGED;
73+
} else {
74+
outp = flb_malloc(1);
75+
if(outp == NULL) {
76+
return FLB_ICONV_FAILURE;
77+
}
78+
*result = outp;
79+
*result_len = 0;
80+
return FLB_ICONV_SUCCESS;
81+
}
82+
}
83+
84+
inp = str;
85+
inleft = slen;
86+
87+
outroom = slen + (slen / 2) + 3; // just add something
88+
outbuf = flb_malloc(outroom);
89+
if(outbuf == NULL) {
90+
return FLB_ICONV_FAILURE;
91+
}
92+
outp = outbuf;
93+
outlen = 0;
94+
outleft = outroom - 1;
95+
96+
iconv(c->conv, NULL, NULL, NULL, NULL);
97+
98+
while(inleft > 0) {
99+
ret = iconv(c->conv, &inp, &inleft, &outp, &outleft);
100+
if(ret == -1) {
101+
switch(errno) {
102+
case EILSEQ: // bad input sequence of char (ignore)
103+
flb_warn("[flb_iconv] bad input char 0x%02x", *inp & 0xff);
104+
inp++;
105+
inleft--;
106+
break;
107+
case EINVAL: // imcomplete char (ignore)
108+
flb_warn("[flb_iconv] incomplete char 0x%02x", *inp & 0xff);
109+
inp++;
110+
inleft--;
111+
break;
112+
113+
case E2BIG:
114+
outdone = outp - outbuf;
115+
outroom = outroom * 2;
116+
outbuf = flb_realloc(outbuf, outroom);
117+
if(outbuf == NULL) {
118+
return FLB_ICONV_FAILURE;
119+
}
120+
outp = outbuf + outdone;
121+
outleft = outroom - outdone - 1;
122+
break;
123+
default:
124+
flb_error("[flb_iconv] unknown error: %d", errno);
125+
break;
126+
}
127+
} else {
128+
break;
129+
}
130+
}
131+
outdone = outp - outbuf;
132+
if((flags & FLB_ICONV_ACCEPT_NOT_CHANGED) && (slen == outdone) && (memcmp(str,outbuf,slen) == 0)) {
133+
flb_free(outbuf);
134+
*result = NULL;
135+
*result_len = 0;
136+
return FLB_ICONV_NOT_CHANGED;
137+
} else {
138+
*result = outbuf;
139+
*result_len = outdone;
140+
return FLB_ICONV_SUCCESS;
141+
}
142+
}
143+

0 commit comments

Comments
 (0)