Skip to content

Commit 5e034c2

Browse files
bluebikebluebike
committed
core: and flb_iconv faculty to do charset encodings
Make possible to have other encodings that UTF-8 in fluent-bit, by creating object flb_iconv-library. Enabled by -D FLB_ICONV=Yes (default No) Requires libiconv-library (can be embedded to libc) Signed-off-by: Jukka Pihl <[email protected]>
1 parent b56f9b1 commit 5e034c2

File tree

4 files changed

+267
-0
lines changed

4 files changed

+267
-0
lines changed

‎CMakeLists.txt‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ option(FLB_SYSTEM_STRPTIME "Use strptime in system libc" Yes)
7373
option(FLB_STATIC_CONF "Build binary using static configuration")
7474
option(FLB_STREAM_PROCESSOR "Enable Stream Processor" Yes)
7575
option(FLB_CORO_STACK_SIZE "Set coroutine stack size")
76+
option(FLB_ICONV "Enable iconv string decoding" No)
7677

7778
# Metrics: Experimental Feature, disabled by default on 0.12 series
7879
# but enabled in the upcoming 0.13 release. Note that development
@@ -174,6 +175,7 @@ if(FLB_ALL)
174175
# Global
175176
set(FLB_DEBUG 1)
176177
set(FLB_TLS 1)
178+
set(FLB_ICONV 1)
177179

178180
# Input plugins
179181
set(FLB_IN_CPU 1)
@@ -203,6 +205,7 @@ if(FLB_ALL)
203205
set(FLB_OUT_STDOUT 1)
204206
set(FLB_OUT_LIB 1)
205207
set(FLB_OUT_FLOWCOUNTER 1)
208+
206209
endif()
207210

208211
if(FLB_DEV)
@@ -649,6 +652,67 @@ if(FLB_INOTIFY)
649652
endif()
650653
endif()
651654

655+
# iconv
656+
if(FLB_ICONV)
657+
658+
if(CMAKE_VERSION VERSION_LESS 3.11.0)
659+
message("cmake older than 3.11.0 .. no FindIconv module... testing here")
660+
if(NOT DEFINED Iconv_IS_BUILT_IN)
661+
if(DEFINED Iconv_INCLUDE_DIR AND NOT DEFINED Iconv_LIBRARY)
662+
check_c_source_compiles(
663+
"
664+
#include <stddef.h>
665+
#include <iconv.h>
666+
int main() {
667+
char *a, *b;
668+
size_t i, j;
669+
iconv_t ic;
670+
ic = iconv_open(\"to\", \"from\");
671+
iconv(ic, &a, &i, &b, &j);
672+
iconv_close(ic);
673+
}
674+
" Iconv_IS_BUILT_IN)
675+
endif()
676+
endif()
677+
if(NOT Iconv_IS_BUILT_IN)
678+
if(DEFINED Iconv_INCLUDE_DIR AND NOT DEFINED Iconv_LIBRARIES)
679+
set(CMAKE_REQUIRED_LIBRARIES "iconv")
680+
check_c_source_compiles(
681+
"
682+
#include <stddef.h>
683+
#include <iconv.h>
684+
int main() {
685+
char *a, *b;
686+
size_t i, j;
687+
iconv_t ic;
688+
ic = iconv_open(\"to\", \"from\");
689+
iconv(ic, &a, &i, &b, &j);
690+
iconv_close(ic);
691+
}
692+
" Iconv_FOUND)
693+
if(Iconv_FOUND)
694+
set(Iconv_LIBRARIES "iconv")
695+
endif()
696+
endif()
697+
endif()
698+
699+
else()
700+
include(FindIconv)
701+
endif()
702+
703+
if(Iconv_IS_BUILT_IN)
704+
message("Iconv_IS_BUILT_IN builtin is defined")
705+
FLB_DEFINITION(FLB_HAVE_ICONV)
706+
elseif(Iconv_LIBRARIES)
707+
message("Iconv_LIBRARIES defined '${Iconv_LIBRARIES}'")
708+
FLB_DEFINITION(FLB_HAVE_ICONV)
709+
else()
710+
message("FLB_ICONV defined iconv not given")
711+
set(FLB_ICONV NO)
712+
endif()
713+
714+
endif()
715+
652716
configure_file(
653717
"${PROJECT_SOURCE_DIR}/include/fluent-bit/flb_info.h.in"
654718
"${PROJECT_SOURCE_DIR}/include/fluent-bit/flb_info.h"

‎include/fluent-bit/flb_iconv.h‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2+
3+
/* Fluent Bit
4+
* ==========
5+
* Copyright (C) 2019 The Fluent Bit Authors
6+
* Copyright (C) 2015-2018 Treasure Data Inc.
7+
*
8+
* Licensed under the Apache License, Version 2.0 (the "License");
9+
* you may not use this file except in compliance with the License.
10+
* You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing, software
15+
* distributed under the License is distributed on an "AS IS" BASIS,
16+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
* See the License for the specific language governing permissions and
18+
* limitations under the License.
19+
*/
20+
21+
#ifndef FLB_ICONV_H
22+
#define FLB_ICONV_H
23+
24+
#include <iconv.h>
25+
26+
27+
#define FLB_ICONV_SUCCESS 0
28+
#define FLB_ICONV_NOT_CHANGED 1
29+
#define FLB_ICONV_FAILURE -1
30+
31+
#define FLB_ICONV_ACCEPT_NOT_CHANGED 0x01
32+
33+
struct flb_iconv {
34+
iconv_t conv;
35+
};
36+
37+
struct flb_iconv *flb_iconv_open(char *to, char *from);
38+
39+
int flb_iconv_execute(struct flb_iconv *ic,
40+
char *str, size_t slen,
41+
char **result, size_t *result_len,
42+
unsigned int flags);
43+
44+
void flb_iconv_close(struct flb_iconv *ic);
45+
46+
#endif

‎src/CMakeLists.txt‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,20 @@ if(FLB_TLS)
8282
)
8383
endif()
8484

85+
# iconv
86+
if(FLB_ICONV)
87+
set(src
88+
${src}
89+
"flb_iconv.c"
90+
)
91+
if(NOT Iconv_IS_BUILT_IN)
92+
set(extra_libs
93+
${extra_libs}
94+
${Iconv_LIBRARIES}
95+
)
96+
endif()
97+
endif()
98+
8599
if(FLB_PROXY_GO)
86100
set(src
87101
${src}

‎src/flb_iconv.c‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2+
3+
/* Fluent Bit
4+
* ==========
5+
* Copyright (C) 2019 The Fluent Bit Authors
6+
* Copyright (C) 2015-2018 Treasure Data Inc.
7+
*
8+
* Licensed under the Apache License, Version 2.0 (the "License");
9+
* you may not use this file except in compliance with the License.
10+
* You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing, software
15+
* distributed under the License is distributed on an "AS IS" BASIS,
16+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
* See the License for the specific language governing permissions and
18+
* limitations under the License.
19+
*/
20+
21+
#include <stdio.h>
22+
#include <iconv.h>
23+
#include <string.h>
24+
#include <time.h>
25+
#include <ctype.h>
26+
27+
#include <fluent-bit/flb_macros.h>
28+
#include <fluent-bit/flb_config.h>
29+
#include <fluent-bit/flb_error.h>
30+
#include <fluent-bit/flb_iconv.h>
31+
#include <fluent-bit/flb_mem.h>
32+
33+
struct flb_iconv *flb_iconv_open(char *to, char *from) {
34+
iconv_t ic;
35+
struct flb_iconv *c;
36+
ic = iconv_open(to,from);
37+
if (ic == (iconv_t) -1) {
38+
return NULL;
39+
}
40+
c = flb_calloc(sizeof(struct flb_iconv),1);
41+
c->conv = ic;
42+
return c;
43+
}
44+
45+
void flb_iconv_close(struct flb_iconv *c) {
46+
if(c) {
47+
iconv_close(c->conv);
48+
flb_free(c);
49+
}
50+
}
51+
52+
53+
int flb_iconv_execute(struct flb_iconv *c,
54+
char *str, size_t slen,
55+
char **result, size_t *result_len,
56+
unsigned int flags)
57+
{
58+
char *outbuf;
59+
char *inp;
60+
char *outp;
61+
size_t outlen;
62+
size_t outroom;
63+
size_t outleft;
64+
size_t ret;
65+
size_t inleft;
66+
size_t outdone;
67+
68+
if(slen == 0) {
69+
if(flags & FLB_ICONV_ACCEPT_NOT_CHANGED) {
70+
*result = NULL;
71+
*result_len = 0;
72+
return FLB_ICONV_NOT_CHANGED;
73+
} else {
74+
outp = flb_malloc(1);
75+
if(outp == NULL) {
76+
return FLB_ICONV_FAILURE;
77+
}
78+
*result = outp;
79+
*result_len = 0;
80+
return FLB_ICONV_SUCCESS;
81+
}
82+
}
83+
84+
inp = str;
85+
inleft = slen;
86+
87+
outroom = slen + (slen / 2) + 3; // just add something
88+
outbuf = flb_malloc(outroom);
89+
if(outbuf == NULL) {
90+
return FLB_ICONV_FAILURE;
91+
}
92+
outp = outbuf;
93+
outlen = 0;
94+
outleft = outroom - 1;
95+
96+
iconv(c->conv, NULL, NULL, NULL, NULL);
97+
98+
while(inleft > 0) {
99+
ret = iconv(c->conv, &inp, &inleft, &outp, &outleft);
100+
if(ret == -1) {
101+
switch(errno) {
102+
case EILSEQ: // bad input sequence of char (ignore)
103+
flb_warn("[flb_iconv] bad input char 0x%02x", *inp & 0xff);
104+
inp++;
105+
inleft--;
106+
break;
107+
case EINVAL: // imcomplete char (ignore)
108+
flb_warn("[flb_iconv] incomplete char 0x%02x", *inp & 0xff);
109+
inp++;
110+
inleft--;
111+
break;
112+
113+
case E2BIG:
114+
outdone = outp - outbuf;
115+
outroom = outroom * 2;
116+
outbuf = flb_realloc(outbuf, outroom);
117+
if(outbuf == NULL) {
118+
return FLB_ICONV_FAILURE;
119+
}
120+
outp = outbuf + outdone;
121+
outleft = outroom - outdone - 1;
122+
break;
123+
default:
124+
flb_error("[flb_iconv] unknown error: %d", errno);
125+
break;
126+
}
127+
} else {
128+
break;
129+
}
130+
}
131+
outdone = outp - outbuf;
132+
if((flags & FLB_ICONV_ACCEPT_NOT_CHANGED) && (slen == outdone) && (memcmp(str,outbuf,slen) == 0)) {
133+
flb_free(outbuf);
134+
*result = NULL;
135+
*result_len = 0;
136+
return FLB_ICONV_NOT_CHANGED;
137+
} else {
138+
*result = outbuf;
139+
*result_len = outdone;
140+
return FLB_ICONV_SUCCESS;
141+
}
142+
}
143+

0 commit comments

Comments
 (0)