Skip to content

Commit 3379df9

Browse files
edsiperedsiper
authored
filter_kubernetes: new option 'use_tag_for_meta' to use tag for metadata (#4062)
The following patch adds a new option called 'use_tag_for_meta' which allows to enrich the metadata only by using the information coming from the record tags. This feature is useful only if you don't want to talk to API server or Kubelet for metadata. The data enrichment depends heavily on a right setup for Tail and the regular expression to extract the proper components. Usage example: --- fluent-bit.conf --- [INPUT] name tail path /var/log/containers/*.log tag kube.<pod>.<namespace>.<container> tag_regex ^/var/log/containers/(?:[^/]+/)?(?<pod>.+)_(?<namespace>.+)_(?<container>.+)\.log$ [FILTER] name kubernetes match kube.* kube_tag_prefix kube. regex_parser kube-name use_tag_for_meta on --- eof --- In any of your parsers file, append the following entries: --- parsers.conf --- [PARSER] name kube-name format regex regex (?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9]))\.(?<namespace_name>[^_]+)\.(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})$ --- eof --- Assuming that one of your log files has the proper common name it will be parsed and enriched as follows: - file name /var/log/containers/traefik-97b44b794-f6sp6_kube-system_traefik-5ce550068d69ec7db2ba4cd9342bb04d79686da97cb802dd8e1eb19487ff727b.log - record output tag : kube.traefik-97b44b794-f6sp6.kube-system.traefik-987cea4dac49e14e64cd6caa4dfcc5610669e6838bd199fa396167a4adcbb4c0: record: [1630903216.378076554, {"log"=>"..." "time="2021-09-06T02:35:53Z" "kubernetes"=>{"pod_name"=>"traefik-97b44b794-f6sp6", "namespace_name"=>"kube-system", "container_name"=>"traefik", "docker_id"=>"5ce550068d69ec7db2ba4cd9342bb04d79686da97cb802dd8e1eb19487ff727b" } } ] Signed-off-by: Eduardo Silva <[email protected]>
1 parent 2ed0c40 commit 3379df9

File tree

4 files changed

+86
-6
lines changed

4 files changed

+86
-6
lines changed

‎plugins/filter_kubernetes/kube_conf.c‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,10 @@ struct flb_kube *flb_kube_conf_create(struct flb_filter_instance *ins,
8787
/* Get Kubernetes API server */
8888
url = flb_filter_get_property("kube_url", ins);
8989

90-
if (ctx->use_kubelet) {
90+
if (ctx->use_tag_for_meta) {
91+
ctx->api_https = FLB_FALSE;
92+
}
93+
else if (ctx->use_kubelet) {
9194
ctx->api_host = flb_strdup(FLB_KUBELET_HOST);
9295
ctx->api_port = ctx->kubelet_port;
9396
ctx->api_https = FLB_TRUE;
@@ -184,8 +187,10 @@ struct flb_kube *flb_kube_conf_create(struct flb_filter_instance *ins,
184187
}
185188
}
186189

187-
flb_plg_info(ctx->ins, "https=%i host=%s port=%i",
188-
ctx->api_https, ctx->api_host, ctx->api_port);
190+
if (!ctx->use_tag_for_meta) {
191+
flb_plg_info(ctx->ins, "https=%i host=%s port=%i",
192+
ctx->api_https, ctx->api_host, ctx->api_port);
193+
}
189194
return ctx;
190195
}
191196

‎plugins/filter_kubernetes/kube_conf.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,7 @@ struct flb_kube {
154154
int dns_retries;
155155
int dns_wait_time;
156156

157+
int use_tag_for_meta;
157158
int use_kubelet;
158159
int kubelet_port;
159160

‎plugins/filter_kubernetes/kube_meta.c‎

Lines changed: 70 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -833,6 +833,62 @@ static int search_item_in_items(struct flb_kube_meta *meta,
833833
return ret;
834834
}
835835

836+
837+
static int merge_meta_from_tag(struct flb_kube *ctx, struct flb_kube_meta *meta,
838+
char **out_buf, size_t *out_size)
839+
{
840+
msgpack_sbuffer mp_sbuf;
841+
msgpack_packer mp_pck;
842+
struct flb_mp_map_header mh;
843+
844+
/* Initialize output msgpack buffer */
845+
msgpack_sbuffer_init(&mp_sbuf);
846+
msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write);
847+
848+
flb_mp_map_header_init(&mh, &mp_pck);
849+
850+
if (meta->podname != NULL) {
851+
flb_mp_map_header_append(&mh);
852+
msgpack_pack_str(&mp_pck, 8);
853+
msgpack_pack_str_body(&mp_pck, "pod_name", 8);
854+
msgpack_pack_str(&mp_pck, meta->podname_len);
855+
msgpack_pack_str_body(&mp_pck, meta->podname, meta->podname_len);
856+
}
857+
858+
if (meta->namespace != NULL) {
859+
flb_mp_map_header_append(&mh);
860+
msgpack_pack_str(&mp_pck, 14);
861+
msgpack_pack_str_body(&mp_pck, "namespace_name", 14);
862+
msgpack_pack_str(&mp_pck, meta->namespace_len);
863+
msgpack_pack_str_body(&mp_pck, meta->namespace, meta->namespace_len);
864+
}
865+
866+
if (meta->container_name != NULL) {
867+
flb_mp_map_header_append(&mh);
868+
msgpack_pack_str(&mp_pck, 14);
869+
msgpack_pack_str_body(&mp_pck, "container_name", 14);
870+
msgpack_pack_str(&mp_pck, meta->container_name_len);
871+
msgpack_pack_str_body(&mp_pck, meta->container_name,
872+
meta->container_name_len);
873+
}
874+
if (meta->docker_id != NULL) {
875+
flb_mp_map_header_append(&mh);
876+
msgpack_pack_str(&mp_pck, 9);
877+
msgpack_pack_str_body(&mp_pck, "docker_id", 9);
878+
msgpack_pack_str(&mp_pck, meta->docker_id_len);
879+
msgpack_pack_str_body(&mp_pck, meta->docker_id,
880+
meta->docker_id_len);
881+
}
882+
883+
flb_mp_map_header_end(&mh);
884+
885+
/* Set outgoing msgpack buffer */
886+
*out_buf = mp_sbuf.data;
887+
*out_size = mp_sbuf.size;
888+
889+
return 0;
890+
}
891+
836892
static int merge_meta(struct flb_kube_meta *meta, struct flb_kube *ctx,
837893
const char *api_buf, size_t api_size,
838894
char **out_buf, size_t *out_size)
@@ -1191,6 +1247,7 @@ static inline int extract_meta(struct flb_kube *ctx,
11911247
}
11921248
kube_tag_str = tag + kube_tag_len;
11931249
kube_tag_len = tag_len - kube_tag_len;
1250+
11941251
n = flb_regex_do(ctx->regex, kube_tag_str, kube_tag_len, &result);
11951252
}
11961253

@@ -1265,10 +1322,15 @@ static int get_and_merge_meta(struct flb_kube *ctx, struct flb_kube_meta *meta,
12651322
char *api_buf;
12661323
size_t api_size;
12671324

1268-
if (ctx->use_kubelet) {
1325+
if (ctx->use_tag_for_meta) {
1326+
ret = merge_meta_from_tag(ctx, meta, out_buf, out_size);
1327+
return ret;
1328+
}
1329+
else if (ctx->use_kubelet) {
12691330
ret = get_pods_from_kubelet(ctx, meta->namespace, meta->podname,
12701331
&api_buf, &api_size);
1271-
} else {
1332+
}
1333+
else {
12721334
ret = get_api_server_info(ctx, meta->namespace, meta->podname,
12731335
&api_buf, &api_size);
12741336
}
@@ -1366,12 +1428,17 @@ int flb_kube_meta_init(struct flb_kube *ctx, struct flb_config *config)
13661428
return 0;
13671429
}
13681430

1431+
if (ctx->use_tag_for_meta) {
1432+
flb_plg_info(ctx->ins, "no network access required (OK)");
1433+
return 0;
1434+
}
1435+
13691436
/* Init network */
13701437
flb_kube_network_init(ctx, config);
13711438

13721439
/* Gather local info */
13731440
ret = get_local_pod_info(ctx);
1374-
if (ret == FLB_TRUE) {
1441+
if (ret == FLB_TRUE && !ctx->use_tag_for_meta) {
13751442
flb_plg_info(ctx->ins, "local POD info OK");
13761443

13771444
ret = wait_for_dns(ctx);

‎plugins/filter_kubernetes/kubernetes.c‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -823,6 +823,13 @@ static struct flb_config_map config_map[] = {
823823
0, FLB_TRUE, offsetof(struct flb_kube, cache_use_docker_id),
824824
"fetch K8s meta when docker_id is changed"
825825
},
826+
827+
{
828+
FLB_CONFIG_MAP_BOOL, "use_tag_for_meta", "false",
829+
0, FLB_TRUE, offsetof(struct flb_kube, use_tag_for_meta),
830+
"use tag associated to retrieve metadata instead of kube-server"
831+
},
832+
826833
/*
827834
* Enable the feature for using kubelet to get pods information
828835
*/

0 commit comments

Comments
 (0)