diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 298d316cb76..dea2b83c34c 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -53,6 +53,8 @@ jobs: - "-DFLB_SANITIZE_THREAD=On" - "-DFLB_SIMD=On" - "-DFLB_SIMD=Off" + - "-DFLB_SQLDB=On" + - "-DFLB_SQLDB=Off" - "-DFLB_ARROW=On" cmake_version: - "3.31.6" diff --git a/plugins/filter_checklist/checklist.c b/plugins/filter_checklist/checklist.c index 210f26065ca..976f9a5559c 100644 --- a/plugins/filter_checklist/checklist.c +++ b/plugins/filter_checklist/checklist.c @@ -21,13 +21,13 @@ #include #include #include -#include #include #include #include "checklist.h" #include +#ifdef FLB_HAVE_SQLDB static int db_init(struct checklist *ctx) { int ret; @@ -124,6 +124,7 @@ static int db_check(struct checklist *ctx, char *buf, size_t size) return match; } +#endif static int load_file_patterns(struct checklist *ctx) { @@ -175,9 +176,11 @@ static int load_file_patterns(struct checklist *ctx) if (ctx->mode == CHECK_EXACT_MATCH) { ret = flb_hash_table_add(ctx->ht, buf, len, "", 0); } +#ifdef FLB_HAVE_SQLDB else if (ctx->mode == CHECK_PARTIAL_MATCH) { ret = db_insert(ctx, buf, len); } +#endif if (ret >= 0) { flb_plg_debug(ctx->ins, "file list: line=%i adds value='%s'", line, buf); @@ -210,7 +213,13 @@ static int init_config(struct checklist *ctx) ctx->mode = CHECK_EXACT_MATCH; } else if (strcasecmp(tmp, "partial") == 0) { +#ifdef FLB_HAVE_SQLDB ctx->mode = CHECK_PARTIAL_MATCH; +#else + flb_plg_error(ctx->ins, + "'mode=partial' requires FLB_HAVE_SQLDB enabled at build time"); + return -1; +#endif } } @@ -223,12 +232,14 @@ static int init_config(struct checklist *ctx) return -1; } } +#ifdef FLB_HAVE_SQLDB else if (ctx->mode == CHECK_PARTIAL_MATCH) { ret = db_init(ctx); if (ret < 0) { return -1; } } +#endif /* record accessor pattern / key name */ ctx->ra_lookup_key = flb_ra_create(ctx->lookup_key, FLB_TRUE); @@ -284,6 +295,11 @@ static int cb_checklist_init(struct flb_filter_instance *ins, } ret = init_config(ctx); + if (ret == -1) { + flb_filter_set_context(ins, NULL); + flb_free(ctx); + return -1; + } return 0; } @@ -504,9 +520,11 @@ static int cb_checklist_filter(const void *data, size_t bytes, found = FLB_TRUE; } } +#ifdef FLB_HAVE_SQLDB else if (ctx->mode == CHECK_PARTIAL_MATCH) { found = db_check(ctx, cmp_buf, cmp_size); } +#endif if (cmp_buf && cmp_buf != (char *) rval->o.via.str.ptr) { flb_free(cmp_buf); @@ -592,11 +610,13 @@ static int cb_exit(void *data, struct flb_config *config) flb_hash_table_destroy(ctx->ht); } +#ifdef FLB_HAVE_SQLDB if (ctx->db) { sqlite3_finalize(ctx->stmt_insert); sqlite3_finalize(ctx->stmt_check); flb_sqldb_close(ctx->db); } +#endif flb_free(ctx); return 0; diff --git a/plugins/filter_checklist/checklist.h b/plugins/filter_checklist/checklist.h index 5fb797c5459..f73e0d3b284 100644 --- a/plugins/filter_checklist/checklist.h +++ b/plugins/filter_checklist/checklist.h @@ -21,14 +21,19 @@ #define FLB_FILTER_CHECK_H #include -#include #include #include +#ifdef FLB_HAVE_SQLDB +#include +#endif + #define LINE_SIZE 2048 #define CHECK_HASH_TABLE_SIZE 100000 #define CHECK_EXACT_MATCH 0 /* exact string match */ +#ifdef FLB_HAVE_SQLDB #define CHECK_PARTIAL_MATCH 1 /* partial match */ +#endif /* plugin context */ struct checklist { @@ -41,9 +46,11 @@ struct checklist { struct mk_list *records; /* internal */ +#ifdef FLB_HAVE_SQLDB struct flb_sqldb *db; sqlite3_stmt *stmt_insert; sqlite3_stmt *stmt_check; +#endif struct flb_hash_table *ht; struct flb_record_accessor *ra_lookup_key; struct flb_filter_instance *ins; diff --git a/plugins/in_blob/blob.c b/plugins/in_blob/blob.c index c3fb8918e62..e9ab9645f19 100644 --- a/plugins/in_blob/blob.c +++ b/plugins/in_blob/blob.c @@ -40,9 +40,12 @@ #include #include "blob.h" -#include "blob_db.h" #include "blob_file.h" +#ifdef FLB_HAVE_SQLDB +#include "blob_db.h" +#endif + #include "win32_glob.c" /* Define missing GLOB_TILDE if not exists */ @@ -739,7 +742,10 @@ static int in_blob_exit(void *in_context, struct flb_config *config) return 0; } +#ifdef FLB_HAVE_SQLDB blob_db_close(ctx); +#endif + blob_file_list_remove_all(ctx); flb_log_event_encoder_destroy(ctx->log_encoder); flb_free(ctx); diff --git a/plugins/in_blob/blob.h b/plugins/in_blob/blob.h index 5f18ca23b93..0333dbddf6e 100644 --- a/plugins/in_blob/blob.h +++ b/plugins/in_blob/blob.h @@ -21,10 +21,13 @@ #define FLB_IN_BLOB_H #include -#include #include #include +#ifdef FLB_HAVE_SQLDB +#include +#endif + #define POST_UPLOAD_ACTION_NONE 0 #define POST_UPLOAD_ACTION_DELETE 1 #define POST_UPLOAD_ACTION_EMIT_LOG 2 diff --git a/plugins/out_azure_blob/azure_blob.c b/plugins/out_azure_blob/azure_blob.c index a650809c4d4..19f3ca13d69 100644 --- a/plugins/out_azure_blob/azure_blob.c +++ b/plugins/out_azure_blob/azure_blob.c @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include @@ -35,7 +34,6 @@ #include #include "azure_blob.h" -#include "azure_blob_db.h" #include "azure_blob_uri.h" #include "azure_blob_conf.h" #include "azure_blob_appendblob.h" @@ -43,6 +41,10 @@ #include "azure_blob_http.h" #include "azure_blob_store.h" +#ifdef FLB_HAVE_SQLDB +#include "azure_blob_db.h" +#endif + #define CREATE_BLOB 1337 /* thread_local_storage for workers */ @@ -248,6 +250,7 @@ static int create_blob(struct flb_azure_blob *ctx, char *name) return FLB_OK; } +#ifdef FLB_HAVE_SQLDB static int delete_blob(struct flb_azure_blob *ctx, char *name) { int ret; @@ -322,6 +325,7 @@ static int delete_blob(struct flb_azure_blob *ctx, char *name) flb_upstream_conn_release(u_conn); return FLB_OK; } +#endif static int http_send_blob(struct flb_config *config, struct flb_azure_blob *ctx, flb_sds_t ref_name, @@ -488,16 +492,23 @@ static int send_blob(struct flb_config *config, if (!block_id) { flb_plg_error(ctx->ins, "could not generate block id"); flb_free(generated_random_string); - cfl_sds_destroy(ref_name); + flb_sds_destroy(ref_name); return FLB_RETRY; } uri = azb_block_blob_uri(ctx, tag, block_id, ms, generated_random_string); ref_name = flb_sds_printf(&ref_name, "file=%s.%" PRIu64, name, ms); } else if (event_type == FLB_EVENT_TYPE_BLOBS) { +#ifdef FLB_HAVE_SQLDB block_id = azb_block_blob_id_blob(ctx, name, part_id); uri = azb_block_blob_uri(ctx, name, block_id, 0, generated_random_string); ref_name = flb_sds_printf(&ref_name, "file=%s:%" PRIu64, name, part_id); +#else + flb_plg_error(ctx->ins, "FLB_EVENT_TYPE_BLOBS requires FLB_HAVE_SQLDB enabled at build time"); + flb_free(generated_random_string); + flb_sds_destroy(ref_name); + return FLB_ERROR; +#endif } } @@ -772,6 +783,7 @@ static int cb_azure_blob_init(struct flb_output_instance *ins, return 0; } +#ifdef FLB_HAVE_SQLDB static int blob_chunk_register_parts(struct flb_azure_blob *ctx, uint64_t file_id, size_t total_size) { int ret; @@ -1171,6 +1183,7 @@ static int azb_timer_create(struct flb_azure_blob *ctx) return 0; } +#endif /** * Azure Blob Storage ingestion callback function @@ -1660,6 +1673,7 @@ static void cb_azure_blob_flush(struct flb_event_chunk *event_chunk, } } else if (event_chunk->type == FLB_EVENT_TYPE_BLOBS) { +#ifdef FLB_HAVE_SQLDB /* * For Blob types, we use the flush callback to enqueue the file, then cb_azb_blob_file_upload() * takes care of the rest like reading the file and uploading it to Azure. @@ -1668,6 +1682,9 @@ static void cb_azure_blob_flush(struct flb_event_chunk *event_chunk, if (ret == -1) { FLB_OUTPUT_RETURN(FLB_RETRY); } +#else + ret = FLB_ERROR; +#endif } if (json){ @@ -1728,11 +1745,13 @@ static int cb_worker_init(void *data, struct flb_config *config) FLB_TLS_SET(worker_info, info); } +#ifdef FLB_HAVE_SQLDB ret = azb_timer_create(ctx); if (ret == -1) { flb_plg_error(ctx->ins, "failed to create upload timer"); return -1; } +#endif return 0; } diff --git a/plugins/out_azure_blob/azure_blob.h b/plugins/out_azure_blob/azure_blob.h index 8699dda54f8..7c51eb410b0 100644 --- a/plugins/out_azure_blob/azure_blob.h +++ b/plugins/out_azure_blob/azure_blob.h @@ -23,7 +23,10 @@ #include #include #include + +#ifdef FLB_HAVE_SQLDB #include +#endif /* Content-Type */ #define AZURE_BLOB_CT "Content-Type" diff --git a/plugins/out_azure_blob/azure_blob_blockblob.c b/plugins/out_azure_blob/azure_blob_blockblob.c index 860401b2623..9d70e28fc8e 100644 --- a/plugins/out_azure_blob/azure_blob_blockblob.c +++ b/plugins/out_azure_blob/azure_blob_blockblob.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include diff --git a/plugins/out_azure_blob/azure_blob_conf.c b/plugins/out_azure_blob/azure_blob_conf.c index ea883a01852..0d935c8a9d6 100644 --- a/plugins/out_azure_blob/azure_blob_conf.c +++ b/plugins/out_azure_blob/azure_blob_conf.c @@ -20,10 +20,14 @@ #include #include #include +#include #include "azure_blob.h" #include "azure_blob_conf.h" + +#ifdef FLB_HAVE_SQLDB #include "azure_blob_db.h" +#endif #include #include @@ -762,6 +766,7 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in } } +#ifdef FLB_HAVE_SQLDB /* database file for blob signal handling */ if (ctx->database_file) { ctx->db = azb_db_open(ctx, ctx->database_file); @@ -771,6 +776,7 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in } pthread_mutex_init(&ctx->file_upload_commit_file_parts, NULL); +#endif flb_plg_info(ctx->ins, "account_name=%s, container_name=%s, blob_type=%s, emulator_mode=%s, endpoint=%s, auth_type=%s", @@ -826,7 +832,9 @@ void flb_azure_blob_conf_destroy(struct flb_azure_blob *ctx) flb_upstream_destroy(ctx->u); } - +#ifdef FLB_HAVE_SQLDB azb_db_close(ctx); +#endif + flb_free(ctx); } diff --git a/src/flb_blob_db.c b/src/flb_blob_db.c index fe6bbcf05ee..a525b544987 100644 --- a/src/flb_blob_db.c +++ b/src/flb_blob_db.c @@ -17,10 +17,11 @@ * limitations under the License. */ -#ifdef FLB_HAVE_SQLDB +#include +#include +#ifdef FLB_HAVE_SQLDB #include -#include static int prepare_stmts(struct flb_blob_db *context) { @@ -1411,6 +1412,16 @@ int flb_blob_db_close(struct flb_blob_db *context) return FLB_BLOB_DB_ERROR_NO_BACKEND_AVAILABLE; } +int flb_blob_db_lock(struct flb_blob_db *context) +{ + return FLB_BLOB_DB_ERROR_NO_BACKEND_AVAILABLE; +} + +int flb_blob_db_unlock(struct flb_blob_db *context) +{ + return FLB_BLOB_DB_ERROR_NO_BACKEND_AVAILABLE; +} + int flb_blob_db_file_exists(struct flb_blob_db *context, char *path, uint64_t *id) @@ -1455,6 +1466,13 @@ int flb_blob_db_file_delivery_attempts(struct flb_blob_db *context, return FLB_BLOB_DB_ERROR_NO_BACKEND_AVAILABLE; } +int flb_blob_file_update_remote_id(struct flb_blob_db *context, + uint64_t id, + cfl_sds_t remote_id) +{ + return FLB_BLOB_DB_ERROR_NO_BACKEND_AVAILABLE; +} + int flb_blob_db_file_get_next_aborted(struct flb_blob_db *context, uint64_t *id, uint64_t *delivery_attempts, @@ -1467,6 +1485,13 @@ int flb_blob_db_file_get_next_aborted(struct flb_blob_db *context, return FLB_BLOB_DB_ERROR_NO_BACKEND_AVAILABLE; } +int flb_blob_db_file_part_update_remote_id(struct flb_blob_db *context, + uint64_t id, + cfl_sds_t remote_id) +{ + return FLB_BLOB_DB_ERROR_NO_BACKEND_AVAILABLE; +} + int flb_blob_db_file_get_next_stale(struct flb_blob_db *context, uint64_t *id, cfl_sds_t *path, @@ -1512,12 +1537,12 @@ int flb_blob_db_file_part_get_next(struct flb_blob_db *context, cfl_sds_t *file_path, cfl_sds_t *destination, cfl_sds_t *remote_file_id, - cfl_sds_t *tag) + cfl_sds_t *tag, + int *part_count) { return FLB_BLOB_DB_ERROR_NO_BACKEND_AVAILABLE; } - int flb_blob_db_file_part_uploaded(struct flb_blob_db *context, uint64_t id) { return FLB_BLOB_DB_ERROR_NO_BACKEND_AVAILABLE; diff --git a/tests/runtime/filter_checklist.c b/tests/runtime/filter_checklist.c index 72d1e481589..5a3d0af3d9d 100644 --- a/tests/runtime/filter_checklist.c +++ b/tests/runtime/filter_checklist.c @@ -348,6 +348,7 @@ void flb_test_ignore_case(void) test_ctx_destroy(ctx); } +#ifdef FLB_HAVE_SQLDB void flb_test_mode_partial(void) { int ret; @@ -394,12 +395,15 @@ void flb_test_mode_partial(void) test_ctx_destroy(ctx); } +#endif TEST_LIST = { {"lookup_key", flb_test_lookup_key}, {"lookup_keys", flb_test_lookup_keys}, {"records", flb_test_records}, {"ignore_case", flb_test_ignore_case}, +#ifdef FLB_HAVE_SQLDB {"mode_partial", flb_test_mode_partial}, +#endif {NULL, NULL} }; diff --git a/tests/runtime_shell/CMakeLists.txt b/tests/runtime_shell/CMakeLists.txt index 30ae87a6b96..493a978d6f8 100644 --- a/tests/runtime_shell/CMakeLists.txt +++ b/tests/runtime_shell/CMakeLists.txt @@ -6,7 +6,6 @@ configure_file( set(UNIT_TESTS_SH custom_calyptia.sh in_dummy_expect.sh - in_tail_expect.sh in_http_tls_expect.sh in_syslog_tcp_tls_expect.sh in_syslog_tcp_plaintext_expect.sh @@ -17,6 +16,12 @@ set(UNIT_TESTS_SH processor_invalid.sh ) +if(FLB_SQLDB) +set(UNIT_TESTS_SH + ${UNIT_TESTS_SH} + in_tail_expect.sh) +endif() + # Prepare list of unit tests foreach(script ${UNIT_TESTS_SH}) add_test(NAME ${script}