diff --git a/include/fluent-bit/flb_aws_credentials.h b/include/fluent-bit/flb_aws_credentials.h index 36652ead45a..cacaa492ad4 100644 --- a/include/fluent-bit/flb_aws_credentials.h +++ b/include/fluent-bit/flb_aws_credentials.h @@ -34,6 +34,14 @@ /* 5 second timeout for credential related http requests */ #define FLB_AWS_CREDENTIAL_NET_TIMEOUT 5 +/* IoT Credentials Environment Variables */ +#define AWS_IOT_KEY_FILE "AWS_IOT_KEY_FILE" +#define AWS_IOT_CERT_FILE "AWS_IOT_CERT_FILE" +#define AWS_IOT_CA_CERT_FILE "AWS_IOT_CA_CERT_FILE" +#define AWS_IOT_CREDENTIALS_ENDPOINT "AWS_IOT_CREDENTIALS_ENDPOINT" +#define AWS_IOT_THING_NAME "AWS_IOT_THING_NAME" +#define AWS_IOT_ROLE_ALIAS "AWS_IOT_ROLE_ALIAS" + /* * A structure that wraps the sensitive data needed to sign an AWS request */ @@ -225,6 +233,11 @@ struct flb_aws_provider *flb_eks_provider_create(struct flb_config *config, flb_aws_client_generator *generator); +/* + * IoT Provider + */ +struct flb_aws_provider *flb_iot_provider_create(struct flb_config *config, + struct flb_aws_client_generator *generator); /* * STS Assume Role Provider. diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index d86ebdb10f3..fd7d12af5a0 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -364,7 +364,6 @@ REGISTER_OUT_PLUGIN("out_es") REGISTER_OUT_PLUGIN("out_exit") REGISTER_OUT_PLUGIN("out_file") REGISTER_OUT_PLUGIN("out_logrotate") - REGISTER_OUT_PLUGIN("out_forward") REGISTER_OUT_PLUGIN("out_http") REGISTER_OUT_PLUGIN("out_influxdb") diff --git a/plugins/out_logrotate/logrotate.c b/plugins/out_logrotate/logrotate.c index dba8563a91d..af6ae222338 100644 --- a/plugins/out_logrotate/logrotate.c +++ b/plugins/out_logrotate/logrotate.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -79,7 +80,7 @@ struct flb_logrotate_conf { size_t max_size; /* Max file size */ int max_files; /* Maximum number of rotated files to keep */ int gzip; /* Whether to gzip rotated files */ - size_t current_file_size; /* Current file size in bytes */ + struct flb_hash_table *file_sizes; /* Hash table to store file size per filename */ struct flb_output_instance *ins; }; @@ -123,10 +124,20 @@ static int cb_logrotate_init(struct flb_output_instance *ins, ctx->delimiter = NULL; ctx->label_delimiter = NULL; ctx->template = NULL; - ctx->current_file_size = 0; /* Initialize file size counter */ + + /* Initialize hash table to store file sizes per filename */ + ctx->file_sizes = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, 64, 0); + if (!ctx->file_sizes) { + flb_errno(); + flb_free(ctx); + return -1; + } ret = flb_output_config_map_set(ins, (void *) ctx); if (ret == -1) { + if (ctx->file_sizes) { + flb_hash_table_destroy(ctx->file_sizes); + } flb_free(ctx); return -1; } @@ -162,6 +173,9 @@ static int cb_logrotate_init(struct flb_output_instance *ins, } else { flb_plg_error(ctx->ins, "unknown format %s. abort.", tmp); + if (ctx->file_sizes) { + flb_hash_table_destroy(ctx->file_sizes); + } flb_free(ctx); return -1; } @@ -353,13 +367,14 @@ static int template_output(FILE *fp, struct flb_time *tm, msgpack_object *obj, return 0; } -static int plain_output(FILE *fp, msgpack_object *obj, size_t alloc_size) +static int plain_output(FILE *fp, msgpack_object *obj, size_t alloc_size, int escape_unicode) { char *buf; - buf = flb_msgpack_to_json_str(alloc_size, obj); + buf = flb_msgpack_to_json_str(alloc_size, obj, escape_unicode); if (buf) { - fprintf(fp, "%s" NEWLINE, buf); + fprintf(fp, "%s" NEWLINE, + buf); flb_free(buf); } return 0; @@ -513,25 +528,50 @@ static int mkpath(struct flb_output_instance *ins, const char *dir) #endif } -/* Function to check if file size exceeds max size in MB */ -static int should_rotate_file(struct flb_logrotate_conf *ctx) +/* Function to check if file size exceeds max size for a specific file */ +static int should_rotate_file(struct flb_logrotate_conf *ctx, const char *filename) { - if (ctx->current_file_size >= ctx->max_size) { - flb_plg_info(ctx->ins, "going to rotate file: current size=%zu max size=%zu", ctx->current_file_size, ctx->max_size); + size_t file_size = 0; + void *out_buf; + size_t out_size; + int ret; + + /* Get file size from hash table */ + ret = flb_hash_table_get(ctx->file_sizes, filename, strlen(filename), + &out_buf, &out_size); + if (ret == 0 && out_size == sizeof(size_t)) { + file_size = *(size_t *)out_buf; + } + + if (file_size >= ctx->max_size) { + flb_plg_info(ctx->ins, "going to rotate file %s: current size=%zu max size=%zu", + filename, file_size, ctx->max_size); return 1; } else { - flb_plg_debug(ctx->ins, "file should not be rotated: current size=%zu max size=%zu", ctx->current_file_size, ctx->max_size); + flb_plg_debug(ctx->ins, "file %s should not be rotated: current size=%zu max size=%zu", + filename, file_size, ctx->max_size); return 0; } } -/* Function to update file size counter using current file position */ -static void update_file_size_counter(struct flb_logrotate_conf *ctx, FILE *fp) +/* Function to update file size counter for a specific file */ +static void update_file_size_counter(struct flb_logrotate_conf *ctx, + const char *filename, FILE *fp) { struct stat st; + size_t file_size; + int ret; + if (fstat(fileno(fp), &st) == 0 && st.st_size >= 0) { - ctx->current_file_size = (size_t) st.st_size; + file_size = (size_t) st.st_size; + + /* Store or update file size in hash table */ + ret = flb_hash_table_add(ctx->file_sizes, filename, strlen(filename), + &file_size, sizeof(size_t)); + if (ret == -1) { + flb_plg_warn(ctx->ins, "failed to update file size for %s", filename); + } } } @@ -710,11 +750,21 @@ static int rotate_file(struct flb_logrotate_conf *ctx, const char *filename) char timestamp[32]; char rotated_filename[PATH_MAX]; char gzip_filename[PATH_MAX]; + size_t file_size = 0; + void *out_buf; + size_t out_size; int ret = 0; + /* Get file size from hash table for logging */ + ret = flb_hash_table_get(ctx->file_sizes, filename, strlen(filename), + &out_buf, &out_size); + if (ret == 0 && out_size == sizeof(size_t)) { + file_size = *(size_t *)out_buf; + } + /* Log rotation event */ flb_plg_info(ctx->ins, "rotating file: %s (current size: %zu bytes)", - filename, ctx->current_file_size); + filename, file_size); /* Generate timestamp */ generate_timestamp(timestamp, sizeof(timestamp)); @@ -750,6 +800,62 @@ static int rotate_file(struct flb_logrotate_conf *ctx, const char *filename) return 0; } +/* Function to validate if a filename matches the rotation pattern format + * Valid formats: + * - base_filename.YYYYMMDD_HHMMSS (15 chars after pattern) + * - base_filename.YYYYMMDD_HHMMSS.gz (18 chars after pattern) + */ +static int is_valid_rotation_filename(const char *filename, const char *pattern) +{ + size_t pattern_len = strlen(pattern); + size_t filename_len = strlen(filename); + const char *suffix; + size_t suffix_len; + int i; + + /* Check that filename starts with pattern */ + if (strncmp(filename, pattern, pattern_len) != 0) { + return 0; + } + + /* Get the suffix after the pattern */ + suffix = filename + pattern_len; + suffix_len = filename_len - pattern_len; + + /* Must be exactly 15 or 18 characters */ + if (suffix_len != 15 && suffix_len != 18) { + return 0; + } + + /* For 18 characters, must end with .gz */ + if (suffix_len == 18) { + if (strcmp(suffix + 15, ".gz") != 0) { + return 0; + } + } + + /* Validate timestamp format: YYYYMMDD_HHMMSS + * - 8 digits (YYYYMMDD) + * - underscore at position 8 + * - 6 digits (HHMMSS) + */ + for (i = 0; i < 8; i++) { + if (suffix[i] < '0' || suffix[i] > '9') { + return 0; + } + } + if (suffix[8] != '_') { + return 0; + } + for (i = 9; i < 15; i++) { + if (suffix[i] < '0' || suffix[i] > '9') { + return 0; + } + } + + return 1; +} + /* Function to clean up old rotated files */ static int cleanup_old_files(struct flb_logrotate_conf *ctx, const char *directory, const char *base_filename) { @@ -772,7 +878,7 @@ static int cleanup_old_files(struct flb_logrotate_conf *ctx, const char *directo /* Count matching files */ while ((entry = readdir(dir)) != NULL) { - if (strncmp(entry->d_name, pattern, strlen(pattern)) == 0) { + if (is_valid_rotation_filename(entry->d_name, pattern)) { file_count++; } } @@ -793,7 +899,7 @@ static int cleanup_old_files(struct flb_logrotate_conf *ctx, const char *directo rewinddir(dir); i = 0; while ((entry = readdir(dir)) != NULL && i < file_count) { - if (strncmp(entry->d_name, pattern, strlen(pattern)) == 0) { + if (is_valid_rotation_filename(entry->d_name, pattern)) { snprintf(full_path, PATH_MAX - 1, "%s" FLB_PATH_SEPARATOR "%s", directory, entry->d_name); files[i] = flb_strdup(full_path); @@ -853,6 +959,7 @@ static void cb_logrotate_flush(struct flb_event_chunk *event_chunk, char out_file[PATH_MAX]; char *buf; long file_pos; + bool have_directory; char *out_file_copy; char directory[PATH_MAX]; @@ -884,7 +991,9 @@ static void cb_logrotate_flush(struct flb_event_chunk *event_chunk, } /* Check if file needs rotation based on current size counter */ - if (should_rotate_file(ctx)) { + if (should_rotate_file(ctx, out_file)) { + have_directory = false; + directory[0] = '\0'; /* Extract directory and base filename for cleanup */ out_file_copy = flb_strdup(out_file); if (out_file_copy) { @@ -897,6 +1006,7 @@ static void cb_logrotate_flush(struct flb_event_chunk *event_chunk, directory[PATH_MAX - 1] = '\0'; #endif flb_free(out_file_copy); + have_directory = true; } /* Get base filename for cleanup */ @@ -910,10 +1020,12 @@ static void cb_logrotate_flush(struct flb_event_chunk *event_chunk, /* Rotate the file */ if (rotate_file(ctx, out_file) == 0) { - /* Reset file size counter after rotation */ - ctx->current_file_size = 0; + /* Remove file size entry from hash table after rotation */ + flb_hash_table_del(ctx->file_sizes, out_file); /* Clean up old rotated files */ - cleanup_old_files(ctx, directory, base_filename); + if (have_directory) { + cleanup_old_files(ctx, directory, base_filename); + } } } @@ -941,8 +1053,15 @@ static void cb_logrotate_flush(struct flb_event_chunk *event_chunk, } /* Initialize file size counter if this is a new file */ - if (ctx->current_file_size == 0) { - update_file_size_counter(ctx, fp); + { + void *out_buf; + size_t out_size; + int ret = flb_hash_table_get(ctx->file_sizes, out_file, strlen(out_file), + &out_buf, &out_size); + if (ret != 0) { + /* File not in hash table, initialize it */ + update_file_size_counter(ctx, out_file, fp); + } } /* @@ -955,6 +1074,8 @@ static void cb_logrotate_flush(struct flb_event_chunk *event_chunk, if (event_chunk->type == FLB_INPUT_METRICS) { print_metrics_text(ctx->ins, fp, event_chunk->data, event_chunk->size); + /* Update file size counter */ + update_file_size_counter(ctx, out_file, fp); fclose(fp); FLB_OUTPUT_RETURN(FLB_OK); } @@ -979,7 +1100,7 @@ static void cb_logrotate_flush(struct flb_event_chunk *event_chunk, } while (total < event_chunk->size); /* Update file size counter */ - update_file_size_counter(ctx, fp); + update_file_size_counter(ctx, out_file, fp); fclose(fp); FLB_OUTPUT_RETURN(FLB_OK); } @@ -1008,7 +1129,8 @@ static void cb_logrotate_flush(struct flb_event_chunk *event_chunk, switch (ctx->format){ case FLB_OUT_LOGROTATE_FMT_JSON: - buf = flb_msgpack_to_json_str(alloc_size, log_event.body); + buf = flb_msgpack_to_json_str(alloc_size, log_event.body, + config->json_escape_unicode); if (buf) { fprintf(fp, "%s: [%"PRIu64".%09lu, %s]" NEWLINE, event_chunk->tag, @@ -1040,7 +1162,7 @@ static void cb_logrotate_flush(struct flb_event_chunk *event_chunk, log_event.body, ctx); break; case FLB_OUT_LOGROTATE_FMT_PLAIN: - plain_output(fp, log_event.body, alloc_size); + plain_output(fp, log_event.body, alloc_size, config->json_escape_unicode); break; case FLB_OUT_LOGROTATE_FMT_TEMPLATE: template_output(fp, @@ -1053,7 +1175,7 @@ static void cb_logrotate_flush(struct flb_event_chunk *event_chunk, flb_log_event_decoder_destroy(&log_decoder); /* Update file size counter */ - update_file_size_counter(ctx, fp); + update_file_size_counter(ctx, out_file, fp); fclose(fp); FLB_OUTPUT_RETURN(FLB_OK); @@ -1067,6 +1189,11 @@ static int cb_logrotate_exit(void *data, struct flb_config *config) return 0; } + /* Destroy hash table */ + if (ctx->file_sizes) { + flb_hash_table_destroy(ctx->file_sizes); + } + flb_free(ctx); return 0; } diff --git a/src/aws/CMakeLists.txt b/src/aws/CMakeLists.txt index 941e811b633..de7d491cb47 100644 --- a/src/aws/CMakeLists.txt +++ b/src/aws/CMakeLists.txt @@ -15,6 +15,7 @@ set(src "flb_aws_imds.c" "flb_aws_credentials_http.c" "flb_aws_credentials_profile.c" + "flb_aws_credentials_iot.c" ) message(STATUS "=== AWS Credentials ===") diff --git a/src/aws/flb_aws_credentials.c b/src/aws/flb_aws_credentials.c index 75f13b111f2..94e2515279a 100644 --- a/src/aws/flb_aws_credentials.c +++ b/src/aws/flb_aws_credentials.c @@ -51,7 +51,6 @@ static struct flb_aws_provider *standard_chain_create(struct flb_config int eks_irsa, char *profile); - /* * The standard credential provider chain: * 1. Environment variables @@ -59,6 +58,7 @@ static struct flb_aws_provider *standard_chain_create(struct flb_config * 3. EKS OIDC * 4. EC2 IMDS * 5. ECS HTTP credentials endpoint + * 6. IoT credentials endpoint * * This provider will evaluate each provider in order, returning the result * from the first provider that returns valid credentials. @@ -566,6 +566,14 @@ static struct flb_aws_provider *standard_chain_create(struct flb_config mk_list_add(&sub_provider->_head, &implementation->sub_providers); + /* IoT Provider - check early since it requires specific environment variables */ + sub_provider = flb_iot_provider_create(config, generator); + if (sub_provider) { + /* IoT provider can fail if we are not running in IoT */ + mk_list_add(&sub_provider->_head, &implementation->sub_providers); + flb_debug("[aws_credentials] Initialized IoT Provider in standard chain"); + } + flb_debug("[aws_credentials] creating profile %s provider", profile); sub_provider = flb_profile_provider_create(profile); if (sub_provider) { diff --git a/src/aws/flb_aws_credentials_iot.c b/src/aws/flb_aws_credentials_iot.c new file mode 100644 index 00000000000..0f27f688bef --- /dev/null +++ b/src/aws/flb_aws_credentials_iot.c @@ -0,0 +1,655 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +/* IoT Provider */ +struct flb_aws_provider_iot { + struct flb_aws_credentials *creds; + time_t next_refresh; + + struct flb_aws_client *client; + + /* IoT specific configuration */ + char *key_file; + char *cert_file; + char *ca_cert_file; + char *credentials_endpoint; + char *thing_name; + char *role_alias; + + /* TLS configuration for IoT certificates */ + struct flb_tls *tls; + + /* Static header for thing name */ + struct flb_aws_header thing_name_header; +}; + +/* Forward declarations */ +static int iot_credentials_request(struct flb_aws_provider_iot *implementation); +static struct flb_aws_credentials *flb_parse_iot_credentials(char *response, size_t response_len, time_t *expiration); + +struct flb_aws_credentials *get_credentials_fn_iot(struct flb_aws_provider *provider) +{ + struct flb_aws_credentials *creds = NULL; + int refresh = FLB_FALSE; + struct flb_aws_provider_iot *implementation = provider->implementation; + + flb_debug("[aws_credentials] Requesting credentials from the " + "IoT provider.."); + + /* a negative next_refresh means that auto-refresh is disabled */ + if (implementation->next_refresh > 0 + && time(NULL) > implementation->next_refresh) { + refresh = FLB_TRUE; + } + if (!implementation->creds || refresh == FLB_TRUE) { + if (try_lock_provider(provider)) { + flb_debug("[aws_credentials] IoT Provider: Refreshing credential " + "cache."); + iot_credentials_request(implementation); + unlock_provider(provider); + } + } + + if (!implementation->creds) { + /* + * We failed to lock the provider and creds are unset. This means that + * another co-routine is performing the refresh. + */ + flb_warn("[aws_credentials] No cached credentials are available and " + "a credential refresh is already in progress. The current " + "co-routine will retry."); + + return NULL; + } + + creds = flb_calloc(1, sizeof(struct flb_aws_credentials)); + if (!creds) { + goto error; + } + + creds->access_key_id = flb_sds_create(implementation->creds->access_key_id); + if (!creds->access_key_id) { + goto error; + } + + creds->secret_access_key = flb_sds_create(implementation->creds-> + secret_access_key); + if (!creds->secret_access_key) { + goto error; + } + + if (implementation->creds->session_token) { + creds->session_token = flb_sds_create(implementation->creds-> + session_token); + if (!creds->session_token) { + goto error; + } + } else { + creds->session_token = NULL; + } + + return creds; + +error: + flb_errno(); + flb_aws_credentials_destroy(creds); + return NULL; +} + +int refresh_fn_iot(struct flb_aws_provider *provider) { + int ret = -1; + struct flb_aws_provider_iot *implementation = provider->implementation; + + flb_debug("[aws_credentials] Refresh called on the IoT provider"); + + if (try_lock_provider(provider)) { + ret = iot_credentials_request(implementation); + unlock_provider(provider); + } + return ret; +} + +int init_fn_iot(struct flb_aws_provider *provider) { + int ret = -1; + struct flb_aws_provider_iot *implementation = provider->implementation; + + flb_debug("[aws_credentials] Init called on the IoT provider"); + + implementation->client->debug_only = FLB_TRUE; + + if (try_lock_provider(provider)) { + ret = iot_credentials_request(implementation); + unlock_provider(provider); + } + + implementation->client->debug_only = FLB_FALSE; + return ret; +} + +void sync_fn_iot(struct flb_aws_provider *provider) { + struct flb_aws_provider_iot *implementation = provider->implementation; + + flb_debug("[aws_credentials] Sync called on the IoT provider"); + /* Remove async flag */ + flb_stream_disable_async_mode(&implementation->client->upstream->base); +} + +void async_fn_iot(struct flb_aws_provider *provider) { + struct flb_aws_provider_iot *implementation = provider->implementation; + + flb_debug("[aws_credentials] Async called on the IoT provider"); + /* Add async flag */ + flb_stream_enable_async_mode(&implementation->client->upstream->base); +} + +void upstream_set_fn_iot(struct flb_aws_provider *provider, + struct flb_output_instance *ins) { + struct flb_aws_provider_iot *implementation = provider->implementation; + + flb_debug("[aws_credentials] upstream_set called on the IoT provider"); + /* Associate output and upstream */ + flb_output_upstream_set(implementation->client->upstream, ins); +} + +void destroy_fn_iot(struct flb_aws_provider *provider) { + struct flb_aws_provider_iot *implementation = provider->implementation; + + if (implementation) { + if (implementation->creds) { + flb_aws_credentials_destroy(implementation->creds); + } + + if (implementation->client) { + flb_aws_client_destroy(implementation->client); + } + + if (implementation->tls) { + flb_tls_destroy(implementation->tls); + } + + if (implementation->key_file) { + flb_free(implementation->key_file); + } + if (implementation->cert_file) { + flb_free(implementation->cert_file); + } + if (implementation->ca_cert_file) { + flb_free(implementation->ca_cert_file); + } + if (implementation->credentials_endpoint) { + flb_free(implementation->credentials_endpoint); + } + if (implementation->thing_name) { + flb_free(implementation->thing_name); + } + if (implementation->role_alias) { + flb_free(implementation->role_alias); + } + + flb_free(implementation); + provider->implementation = NULL; + } + + return; +} + +static struct flb_aws_provider_vtable iot_provider_vtable = { + .get_credentials = get_credentials_fn_iot, + .init = init_fn_iot, + .refresh = refresh_fn_iot, + .destroy = destroy_fn_iot, + .sync = sync_fn_iot, + .async = async_fn_iot, + .upstream_set = upstream_set_fn_iot, +}; + +struct flb_aws_provider *flb_iot_provider_create(struct flb_config *config, + struct flb_aws_client_generator *generator) +{ + struct flb_aws_provider_iot *implementation = NULL; + struct flb_aws_provider *provider = NULL; + struct flb_upstream *upstream = NULL; + char *endpoint_path = NULL; + flb_sds_t protocol = NULL; + flb_sds_t host = NULL; + flb_sds_t port_sds = NULL; + int port = 443; + int ret; + + /* Check if IoT environment variables are set */ + char *key_file = getenv(AWS_IOT_KEY_FILE); + char *cert_file = getenv(AWS_IOT_CERT_FILE); + char *ca_cert_file = getenv(AWS_IOT_CA_CERT_FILE); + char *credentials_endpoint = getenv(AWS_IOT_CREDENTIALS_ENDPOINT); + char *thing_name = getenv(AWS_IOT_THING_NAME); + char *role_alias = getenv(AWS_IOT_ROLE_ALIAS); + + if (!key_file || !cert_file || !ca_cert_file || !credentials_endpoint || + !thing_name || !role_alias) { + flb_debug("[aws_credentials] Not initializing IoT provider because " + "required environment variables are not set"); + return NULL; + } + + provider = flb_calloc(1, sizeof(struct flb_aws_provider)); + if (!provider) { + flb_errno(); + return NULL; + } + + pthread_mutex_init(&provider->lock, NULL); + + implementation = flb_calloc(1, sizeof(struct flb_aws_provider_iot)); + if (!implementation) { + flb_free(provider); + flb_errno(); + return NULL; + } + + provider->provider_vtable = &iot_provider_vtable; + provider->implementation = implementation; + + /* Store IoT configuration */ + implementation->key_file = flb_strdup(key_file); + implementation->cert_file = flb_strdup(cert_file); + implementation->ca_cert_file = flb_strdup(ca_cert_file); + implementation->credentials_endpoint = flb_strdup(credentials_endpoint); + implementation->thing_name = flb_strdup(thing_name); + implementation->role_alias = flb_strdup(role_alias); + + /* Ensure credentials_endpoint has http or https scheme, default to https:// if missing */ + if (strncmp(credentials_endpoint, "http://", 7) != 0 && + strncmp(credentials_endpoint, "https://", 8) != 0) { + flb_sds_t tmp = flb_sds_create_size(strlen(credentials_endpoint) + 8 + 1); + if (!tmp) { + flb_error("[aws_credentials] Failed to allocate memory for credentials_endpoint"); + goto error; + } + flb_sds_cat(tmp, "https://", 8); + flb_sds_cat(tmp, credentials_endpoint, strlen(credentials_endpoint)); + flb_free(implementation->credentials_endpoint); + implementation->credentials_endpoint = tmp; + credentials_endpoint = implementation->credentials_endpoint; + } + + /* Parse the credentials endpoint URL */ + ret = flb_utils_url_split_sds(credentials_endpoint, &protocol, &host, &port_sds, &endpoint_path); + if (ret < 0) { + flb_error("[aws_credentials] Invalid IoT credentials endpoint URL: %s", credentials_endpoint); + goto error; + } + + if (port_sds != NULL) { + port = atoi(port_sds); + if (port == 0) { + flb_error("[aws_credentials] Invalid port in IoT credentials endpoint: %s", port_sds); + goto error; + } + } + + /* Create TLS configuration for IoT certificates */ + flb_debug("[aws_credentials] Creating TLS instance with cert: %s, key: %s, ca: %s", + implementation->cert_file, implementation->key_file, implementation->ca_cert_file); + + implementation->tls = flb_tls_create(FLB_TLS_CLIENT_MODE, + FLB_TRUE, + FLB_TRUE, /* debug - enable TLS debug */ + NULL, /* vhost */ + NULL, /* ca_path */ + implementation->ca_cert_file, + implementation->cert_file, + implementation->key_file, + NULL); /* key_passwd */ + if (!implementation->tls) { + flb_error("[aws_credentials] Failed to create TLS instance for IoT Provider"); + goto error; + } + + flb_debug("[aws_credentials] TLS instance created successfully"); + + /* Create upstream connection */ + flb_debug("[aws_credentials] Creating upstream connection to %s:%d", host, port); + upstream = flb_upstream_create(config, host, port, FLB_IO_TLS, implementation->tls); + if (!upstream) { + flb_error("[aws_credentials] IoT Provider: connection initialization error"); + goto error; + } + + flb_debug("[aws_credentials] Upstream connection created successfully"); + + upstream->base.net.connect_timeout = FLB_AWS_CREDENTIAL_NET_TIMEOUT; + + implementation->client = generator->create(); + if (!implementation->client) { + flb_aws_provider_destroy(provider); + flb_upstream_destroy(upstream); + flb_error("[aws_credentials] IoT Provider: client creation error"); + return NULL; + } + + implementation->client->name = "iot_provider_client"; + implementation->client->has_auth = FLB_FALSE; + implementation->client->provider = NULL; + implementation->client->region = NULL; + implementation->client->service = NULL; + implementation->client->port = port; + implementation->client->flags = 0; + implementation->client->proxy = NULL; + implementation->client->upstream = upstream; + + flb_debug("[aws_credentials] IoT client configured: name=%s, port=%d, has_auth=%d", + implementation->client->name, implementation->client->port, implementation->client->has_auth); + + /* Set up the thing name header */ + implementation->thing_name_header.key = "x-amzn-iot-thingname"; + implementation->thing_name_header.key_len = 22; + implementation->thing_name_header.val = implementation->thing_name; + implementation->thing_name_header.val_len = strlen(implementation->thing_name); + + flb_debug("[aws_credentials] Setting IoT thing name header: %s = %s", + implementation->thing_name_header.key, implementation->thing_name_header.val); + + /* Set the static headers for the client */ + implementation->client->static_headers = &implementation->thing_name_header; + implementation->client->static_headers_len = 1; + + cleanup: + flb_sds_destroy(protocol); + flb_sds_destroy(host); + flb_sds_destroy(port_sds); + flb_sds_destroy(endpoint_path); + return provider; + error: + flb_aws_provider_destroy(provider); + provider = NULL; + goto cleanup; +} + +static int iot_credentials_request(struct flb_aws_provider_iot *implementation) +{ + struct flb_aws_credentials *creds = NULL; + struct flb_http_client *c = NULL; + time_t expiration; + flb_sds_t uri = NULL; + int ret; + + flb_debug("[aws_credentials] Calling IoT credentials endpoint.."); + + /* Construct the URI for the IoT credentials request */ + uri = flb_sds_create_size(256); + if (!uri) { + flb_errno(); + return -1; + } + + uri = flb_sds_printf(&uri, "/role-aliases/%s/credentials", implementation->role_alias); + if (!uri) { + return -1; + } + + /* Make the HTTP request */ + flb_debug("[aws_credentials] Making IoT credentials request to: %s", uri); + flb_debug("[aws_credentials] Client headers count: %d", implementation->client->static_headers_len); + if (implementation->client->static_headers_len > 0) { + flb_debug("[aws_credentials] Client header: %s = %s", + implementation->client->static_headers[0].key, + implementation->client->static_headers[0].val); + } + + c = implementation->client->client_vtable->request(implementation->client, FLB_HTTP_GET, + uri, NULL, 0, NULL, 0); + + flb_sds_destroy(uri); + + if (!c) { + flb_error("[aws_credentials] IoT credentials request failed - no response"); + return -1; + } + + flb_debug("[aws_credentials] IoT credentials response status: %d", c->resp.status); + flb_debug("[aws_credentials] IoT credentials response size: %zu", c->resp.payload_size); + + if (c->resp.status != 200) { + flb_error("[aws_credentials] IoT credentials request failed with status: %d", c->resp.status); + if (c->resp.payload_size > 0) { + flb_aws_print_error_code(c->resp.payload, c->resp.payload_size, + "IoTCredentialsProvider"); + } + flb_http_client_destroy(c); + return -1; + } + + /* Debug: Log the actual response from IoT credentials endpoint */ + flb_debug("[aws_credentials] IoT credentials response (size: %zu): %.*s", + c->resp.payload_size, (int)c->resp.payload_size, c->resp.payload); + + /* Parse the credentials response - IoT endpoint may have different format */ + creds = flb_parse_iot_credentials(c->resp.payload, c->resp.payload_size, &expiration); + if (!creds) { + flb_debug("[aws_credentials] Failed to parse IoT credentials response"); + flb_http_client_destroy(c); + return -1; + } + + /* Destroy existing credentials */ + flb_aws_credentials_destroy(implementation->creds); + implementation->creds = NULL; + + implementation->creds = creds; + implementation->next_refresh = expiration - FLB_AWS_REFRESH_WINDOW; + flb_http_client_destroy(c); + + return 0; +} + +/* + * Parse IoT credentials response. + * AWS IoT credentials endpoint returns a JSON response with credentials. + * The format may be different from standard AWS credentials endpoints. + */ +static struct flb_aws_credentials *flb_parse_iot_credentials(char *response, size_t response_len, time_t *expiration) +{ + jsmntok_t *tokens = NULL; + const jsmntok_t *t = NULL; + char *current_token = NULL; + jsmn_parser parser; + int tokens_size = 50; + size_t size; + int ret; + struct flb_aws_credentials *creds = NULL; + int i = 0; + int len; + flb_sds_t tmp; + + /* + * Remove/reset existing value of expiration. + * Expiration should be in the response, but it is not + * strictly speaking needed. Fluent Bit logs a warning if it is missing. + */ + *expiration = -1; + + jsmn_init(&parser); + + size = sizeof(jsmntok_t) * tokens_size; + tokens = flb_calloc(1, size); + if (!tokens) { + goto error; + } + + ret = jsmn_parse(&parser, response, response_len, tokens, tokens_size); + + if (ret == JSMN_ERROR_INVAL || ret == JSMN_ERROR_PART) { + flb_error("[aws_credentials] Could not parse IoT credentials response - invalid JSON."); + goto error; + } + + /* Shouldn't happen, but just in case, check for too many tokens error */ + if (ret == JSMN_ERROR_NOMEM) { + flb_error("[aws_credentials] Could not parse IoT credentials response - response contained more tokens than expected."); + goto error; + } + + /* return value is number of tokens parsed */ + tokens_size = ret; + + creds = flb_calloc(1, sizeof(struct flb_aws_credentials)); + if (!creds) { + flb_errno(); + goto error; + } + + /* + * jsmn will create an array of tokens like: + * key, value, key, value + * For IoT credentials, the structure is: + * {"credentials": {"accessKeyId": "...", "secretAccessKey": "...", ...}} + */ + while (i < (tokens_size - 1)) { + t = &tokens[i]; + + if (t->start == -1 || t->end == -1 || (t->start == 0 && t->end == 0)) { + break; + } + + if (t->type == JSMN_STRING) { + current_token = &response[t->start]; + len = t->end - t->start; + + /* Check for credentials wrapper object */ + if (strncmp(current_token, "credentials", len) == 0) { + /* Skip the credentials object - we'll process its contents */ + i++; + continue; + } + + /* Check for AccessKeyId field (case insensitive) */ + if (strncmp(current_token, "accessKeyId", len) == 0 || + strncmp(current_token, "AccessKeyId", len) == 0) { + i++; + t = &tokens[i]; + current_token = &response[t->start]; + len = t->end - t->start; + if (creds->access_key_id != NULL) { + flb_error("[aws_credentials] Trying to double allocate access_key_id"); + goto error; + } + creds->access_key_id = flb_sds_create_len(current_token, len); + if (!creds->access_key_id) { + flb_errno(); + goto error; + } + continue; + } + /* Check for SecretAccessKey field (case insensitive) */ + if (strncmp(current_token, "secretAccessKey", len) == 0 || + strncmp(current_token, "SecretAccessKey", len) == 0) { + i++; + t = &tokens[i]; + current_token = &response[t->start]; + len = t->end - t->start; + if (creds->secret_access_key != NULL) { + flb_error("[aws_credentials] Trying to double allocate secret_access_key"); + goto error; + } + creds->secret_access_key = flb_sds_create_len(current_token, len); + if (!creds->secret_access_key) { + flb_errno(); + goto error; + } + continue; + } + /* Check for Token field (session token) - case insensitive */ + if (strncmp(current_token, "sessionToken", len) == 0 || + strncmp(current_token, "Token", len) == 0) { + i++; + t = &tokens[i]; + current_token = &response[t->start]; + len = t->end - t->start; + if (creds->session_token != NULL) { + flb_error("[aws_credentials] Trying to double allocate session_token"); + goto error; + } + creds->session_token = flb_sds_create_len(current_token, len); + if (!creds->session_token) { + flb_errno(); + goto error; + } + continue; + } + /* Check for Expiration field (case insensitive) */ + if (strncmp(current_token, "expiration", len) == 0 || + strncmp(current_token, "Expiration", len) == 0) { + i++; + t = &tokens[i]; + current_token = &response[t->start]; + len = t->end - t->start; + tmp = flb_sds_create_len(current_token, len); + if (!tmp) { + flb_errno(); + goto error; + } + *expiration = flb_aws_cred_expiration(tmp); + if (*expiration < 0) { + flb_warn("[aws_credentials] '%s' was invalid or could not be parsed. Disabling auto-refresh of credentials.", tmp); + } + flb_sds_destroy(tmp); + } + } + + i++; + } + + if (creds->access_key_id == NULL) { + flb_error("[aws_credentials] Missing AccessKeyId field in IoT credentials response"); + goto error; + } + + if (creds->secret_access_key == NULL) { + flb_error("[aws_credentials] Missing SecretAccessKey field in IoT credentials response"); + goto error; + } + + flb_debug("[aws_credentials] Successfully parsed IoT credentials - AccessKeyId: %s, Expiration: %ld", + creds->access_key_id, *expiration); + + flb_free(tokens); + return creds; + +error: + flb_aws_credentials_destroy(creds); + flb_free(tokens); + return NULL; +} diff --git a/tests/runtime/out_logrotate.c b/tests/runtime/out_logrotate.c index e96ba211122..42ed8b57aa3 100644 --- a/tests/runtime/out_logrotate.c +++ b/tests/runtime/out_logrotate.c @@ -38,7 +38,7 @@ static int recursive_delete_directory(const char *dir_path) { DIR *dir; struct dirent *entry; - struct stat statbuf; + struct stat statbuf; char path[PATH_MAX]; int ret = 0; @@ -337,7 +337,8 @@ void flb_test_logrotate_max_files_cleanup(void) } sleep(1); /* Wait for flush */ file_count = count_files_in_directory(TEST_LOGPATH, TEST_LOGFILE); - TEST_CHECK(file_count <= 4); + TEST_ASSERT(file_count >= 0); + TEST_CHECK(file_count <= 4); } flb_stop(ctx); @@ -345,7 +346,8 @@ void flb_test_logrotate_max_files_cleanup(void) /* Check that only Max_Files + 1 files exist (current + rotated) */ file_count = count_files_in_directory(TEST_LOGPATH, TEST_LOGFILE); - TEST_CHECK(file_count <= 4); /* Current file + 2 rotated files */ + TEST_ASSERT(file_count >= 0); + TEST_CHECK(file_count <= 4); /* Current file + 3 rotated files (max_files=3) */ /* Clean up directory and all contents */ recursive_delete_directory(TEST_LOGPATH);