custom_calyptia: switch to YAML for Fleet config (#9698)

* custom_calyptia: switch to YAML for Fleet config

Signed-off-by: Patrick Stephens <[email protected]>

* in_calyptia_fleet: code review comments

Signed-off-by: Patrick Stephens <[email protected]>

---------

Signed-off-by: Patrick Stephens <[email protected]>

Author: Pat

.github/workflows/unit-tests.yaml

@@ -78,6 +78,7 @@ jobs:
           CC: ${{ matrix.compiler }}
           CXX: ${{ matrix.compiler }}
           FLB_OPT: ${{ matrix.flb_option }}
+          CALYPTIA_FLEET_TOKEN: ${{ secrets.CALYPTIA_FLEET_TOKEN }}
 
   run-macos-unit-tests:
     # We chain this after Linux one as there are costs and restrictions associated
@@ -111,6 +112,7 @@ jobs:
           CC: gcc
           CXX: g++
           FLB_OPT: ${{ matrix.flb_option }}
+          CALYPTIA_FLEET_TOKEN: ${{ secrets.CALYPTIA_FLEET_TOKEN }}
 
   run-aarch64-unit-tests:
     # Ensure for OSS Fluent Bit repo we enable usage of Actuated runners for ARM builds, for forks it should keep existing ubuntu-latest usage.
@@ -167,6 +169,7 @@ jobs:
         env:
           CC: ${{ matrix.config.compiler }}
           CXX: ${{ matrix.config.compiler }}
+          CALYPTIA_FLEET_TOKEN: ${{ secrets.CALYPTIA_FLEET_TOKEN }}
 
   run-qemu-ubuntu-unit-tests:
     # We chain this after Linux one as there are CPU time costs for QEMU emulation
@@ -217,7 +220,6 @@ jobs:
             make -j $nparallel
             ctest -j $nparallel --build-run-dir . --output-on-failure
 
-
   # Required check looks at this so do not remove
   run-all-unit-tests:
     if: always()

include/fluent-bit/calyptia/calyptia_constants.h

@@ -36,8 +36,10 @@
 #define CALYPTIA_ENDPOINT_TRACE   "/v1/traces/%s"
 
 #define CALYPTIA_ENDPOINT_FLEETS "/v1/fleets"
-#define CALYPTIA_ENDPOINT_FLEET_CONFIG_INI "/v1/fleets/%s/config?format=ini"
+#define CALYPTIA_ENDPOINT_FLEET_CONFIG_INI "/v1/fleets/%s/config?format=ini&config_format=ini"
+#define CALYPTIA_ENDPOINT_FLEET_CONFIG_YAML "/v1/fleets/%s/config?format=yaml&config_format=yaml"
 #define CALYPTIA_ENDPOINT_FLEET_FILES "/v1/fleets/%s/files"
+#define CALYPTIA_ENDPOINT_FLEET_BY_NAME "/v1/search?project_id=%s&resource=fleet&term=%s&exact=true"
 
 /* Storage */
 #define CALYPTIA_SESSION_FILE     "session.CALYPTIA"

plugins/custom_calyptia/calyptia.c

@@ -232,6 +232,7 @@ int set_fleet_input_properties(struct calyptia *ctx, struct flb_input_instance *
     flb_input_set_property(fleet, "host", ctx->cloud_host);
     flb_input_set_property(fleet, "port", ctx->cloud_port);
     flb_input_set_property(fleet, "config_dir", ctx->fleet_config_dir);
+    flb_input_set_property(fleet, "fleet_config_legacy_format", ctx->fleet_config_legacy_format == 1 ? "on" : "off");
 
     /* Set TLS properties */
     flb_input_set_property(fleet, "tls", ctx->cloud_tls == 1 ? "on" : "off");
@@ -617,7 +618,7 @@ static int cb_calyptia_init(struct flb_custom_instance *ins,
             flb_free(ctx);
             return -1;
         }
-        ctx->machine_id_auto_configured = 1;
+        ctx->machine_id_auto_configured = FLB_TRUE;
     }
 
     /* input collector */
@@ -774,6 +775,11 @@ static struct flb_config_map config_map[] = {
      0, FLB_TRUE, offsetof(struct calyptia, register_retry_on_flush),
      "Retry agent registration on flush if failed on init."
     },
+    {
+     FLB_CONFIG_MAP_BOOL, "fleet_config_legacy_format", "true",
+     0, FLB_TRUE, offsetof(struct calyptia, fleet_config_legacy_format),
+     "If set, use legacy (TOML) format for configuration files."
+    },
     /* EOF */
     {0}
 };

plugins/custom_calyptia/calyptia.h

@@ -20,8 +20,6 @@
 #ifndef FLB_CALYPTIA_H
 #define FLB_CALYPTIA_H
 
-#include <stdbool.h>
-
 struct calyptia {
     /* config map options */
     flb_sds_t api_key;
@@ -55,7 +53,8 @@ struct calyptia {
     flb_sds_t fleet_max_http_buffer_size;
     flb_sds_t fleet_interval_sec;
     flb_sds_t fleet_interval_nsec;
-    bool register_retry_on_flush;   /* retry registration on flush if failed */
+    int register_retry_on_flush;          /* retry registration on flush if failed */
+    int fleet_config_legacy_format;       /* Fleet config format to use: INI (true) or YAML (false) */
 };
 
 int set_fleet_input_properties(struct calyptia *ctx, struct flb_input_instance *fleet);