Add option for FileManagerStrategy delete stale data (#64)

* Add option for FileManagerStrategy discard stale data

Signed-off-by: Jesse Ikawa <[email protected]>

* Add unit tests

Signed-off-by: Jesse Ikawa <[email protected]>

Author: jikawa-az

README.md

@@ -124,6 +124,7 @@ An example configuration file called `sample_configuration.yaml` is provided. Wh
 | publish_topic_names | Whether or not to include topic name information in the log messsages that are uploaded to AWS CloudWatch Logs | *bool* | true/false | true |
 | storage_directory | The location where all offline metrics will be stored | *string* | string | ~/.ros/cwlogs/ |
 | storage_limit | The maximum size of all offline storage files in KB. Once this limit is reached offline logs will start to be deleted oldest first. | *int* | number | 1048576 |
+| delete_stale_data | Whether or not to delete log batch data that are over 14 days old, which are rejected by [AWS PutLogEvents](https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html). | *bool* | true/false | false |
 | aws_client_configuration | AWS region configuration | *std::string* | *region*: "us-west-2"/"us-east-1"/"us-east-2"/etc. | region: us-west-2 |
 
 ### Advanced Configuration Parameters

cloudwatch_logger/config/sample_configuration.yaml

@@ -44,6 +44,10 @@ storage_directory: "~/.ros/cwlogs/"
 # The maximum size of all files in offline storage in KB
 storage_limit: 1048576
 
+# whether or not to automatically delete log data > 14 days old
+#default value is: false
+delete_stale_data: false
+
 # This is the AWS Client Configuration used by the AWS service client in the Node. If given the node will load the
 # provided configuration when initializing the client.
 aws_client_configuration:

cloudwatch_logger/include/cloudwatch_logger/log_node_param_helper.h

@@ -51,6 +51,7 @@ constexpr char kNodeParamStorageDirectory[] = "storage_directory";
 constexpr char kNodeParamFileExtension[] = "file_extension";
 constexpr char kNodeParamMaximumFileSize[] = "maximum_file_size";
 constexpr char kNodeParamStorageLimit[] = "storage_limit";
+constexpr char kNodeParamDeleteStaleData[] = "delete_stale_data";
 
 /** Default values for parameters **/
 constexpr char kNodeLogGroupNameDefaultValue[] = "ros_log_group";
@@ -231,6 +232,22 @@ void ReadOption(
   const size_t & default_value,
   size_t & option_value);
 
+/**
+ * Fetch a single size_t option
+ *
+ * @param parameter_reader to retrieve the parameters from
+ * @param option_key the parameter key to read
+ * @param default_value a default value if the parameter doesn't exist or is unreadble
+ * @param option_value the size_t value for this option
+ * @return an error code that indicates whether the parameter was read successfully or not,
+ * as returned by \p parameter_reader
+ */
+void ReadOption(
+  const std::shared_ptr<Aws::Client::ParameterReaderInterface>& parameter_reader,
+  const std::string & option_key,
+  const bool & default_value,
+  bool & option_value);
+
 }  // namespace Utils
 }  // namespace CloudWatchLogs
 }  // namespace Aws

cloudwatch_logger/src/log_node_param_helper.cpp

@@ -335,6 +335,12 @@ void ReadFileManagerStrategyOptions(
     kNodeParamStorageLimit,
     Aws::FileManagement::kDefaultFileManagerStrategyOptions.storage_limit_in_kb,
     file_manager_strategy_options.storage_limit_in_kb);
+
+  ReadOption(
+    parameter_reader,
+    kNodeParamDeleteStaleData,
+    Aws::FileManagement::kDefaultFileManagerStrategyOptions.delete_stale_data,
+    file_manager_strategy_options.delete_stale_data);
 }
 
 void ReadOption(
@@ -383,6 +389,29 @@ void ReadOption(
   }
 }
 
+void ReadOption(
+  const std::shared_ptr<Aws::Client::ParameterReaderInterface>& parameter_reader,
+  const std::string & option_key,
+  const bool & default_value,
+  bool & option_value)
+{
+  Aws::AwsError ret = parameter_reader->ReadParam(ParameterPath(option_key), option_value);
+  switch (ret) {
+    case Aws::AwsError::AWS_ERR_NOT_FOUND:
+      option_value = default_value;
+      AWS_LOGSTREAM_INFO(__func__,
+                         option_key << " parameter not found, setting to default value: " << default_value);
+      break;
+    case Aws::AwsError::AWS_ERR_OK:
+      AWS_LOGSTREAM_INFO(__func__, option_key << " is set to: " << option_value);
+      break;
+    default:
+      option_value = default_value;
+      AWS_LOGSTREAM_ERROR(__func__,
+                          "Error " << ret << " retrieving option " << option_key << ", setting to default value: " << default_value);
+  }
+}
+
 }  // namespace Utils
 }  // namespace CloudWatchLogs
 }  // namespace Aws

cloudwatch_logger/test/log_node_param_helper_test.cpp

@@ -319,6 +319,86 @@ TEST_F(LogNodeParamHelperFixture, TestReadIgnoreNodesSet)
     EXPECT_EQ(AwsError::AWS_ERR_OK, ReadIgnoreNodesSet(param_reader_, param));
     EXPECT_EQ(1, param.count("String1"));
 }
+/**
+ * FileManagerStrategyOptions options defined with delete_stale_data set to true.
+ * First ReadParam call fails, therefore we expect options_.delete_stale_data set true->false.
+ * Second ReadParam call suceeds, therefore we expect options_.delete_stale_data set false->true.
+ * Third ReadParam call not found, therefore we expect options_.delete_stale_data set true->false.
+ */
+TEST_F(LogNodeParamHelperFixture, Test_Delete_Stale_Data_True)
+{
+  {
+    InSequence read_param_seq;
+
+    EXPECT_CALL(*param_reader_, ReadParam(Eq(ParameterPath(kNodeParamDeleteStaleData)), A<bool &>()))
+    .WillOnce(Return(AwsError::AWS_ERR_FAILURE));
+
+    EXPECT_CALL(*param_reader_, ReadParam(Eq(ParameterPath(kNodeParamDeleteStaleData)), A<bool &>()))
+    .WillOnce(
+      DoAll(SetArgReferee<1>(true), Return(AwsError::AWS_ERR_OK))
+    );
+
+    EXPECT_CALL(*param_reader_, ReadParam(Eq(ParameterPath(kNodeParamDeleteStaleData)), A<bool &>()))
+    .WillOnce(Return(AwsError::AWS_ERR_NOT_FOUND));
+  }
+
+  Aws::FileManagement::FileManagerStrategyOptions options_{"test", "log_tests/", ".log", 1024*1024, 1024*1024, true};
+  ASSERT_TRUE(options_.delete_stale_data);
+
+  ReadOption(param_reader_, kNodeParamDeleteStaleData, Aws::FileManagement::kDefaultFileManagerStrategyOptions.delete_stale_data, options_.delete_stale_data);
+  ASSERT_FALSE(options_.delete_stale_data);
+
+  ReadOption(param_reader_, kNodeParamDeleteStaleData, Aws::FileManagement::kDefaultFileManagerStrategyOptions.delete_stale_data, options_.delete_stale_data);
+  ASSERT_TRUE(options_.delete_stale_data);
+
+  ReadOption(param_reader_, kNodeParamDeleteStaleData, Aws::FileManagement::kDefaultFileManagerStrategyOptions.delete_stale_data, options_.delete_stale_data);
+  ASSERT_FALSE(options_.delete_stale_data);
+}
+/**
+ * FileManagerStrategyOptions options defined with delete_stale_data not defined.
+ * We expect that when delete_stale_data is not defined that it will default to false.
+ * First ReadParam call suceeds, therefore we expect options_.delete_stale_data set false->true.
+ * Second ReadParam call fails, therefore we expect options_.delete_stale_data set true->false.
+ * Third ReadParam call suceeds, therefore we expect options_.delete_stale_data set false->true.
+ * Forth ReadParam call not found, therefore we expect options_.delete_stale_data set true->false.
+ */
+TEST_F(LogNodeParamHelperFixture, Test_Delete_Stale_Data_Not_Defined)
+{
+  {
+    InSequence read_param_seq;
+
+    EXPECT_CALL(*param_reader_, ReadParam(Eq(ParameterPath(kNodeParamDeleteStaleData)), A<bool &>()))
+    .WillOnce(
+      DoAll(SetArgReferee<1>(true), Return(AwsError::AWS_ERR_OK))
+    );
+
+    EXPECT_CALL(*param_reader_, ReadParam(Eq(ParameterPath(kNodeParamDeleteStaleData)), A<bool &>()))
+    .WillOnce(Return(AwsError::AWS_ERR_FAILURE));
+
+    EXPECT_CALL(*param_reader_, ReadParam(Eq(ParameterPath(kNodeParamDeleteStaleData)), A<bool &>()))
+    .WillOnce(
+      DoAll(SetArgReferee<1>(true), Return(AwsError::AWS_ERR_OK))
+    );
+
+    EXPECT_CALL(*param_reader_, ReadParam(Eq(ParameterPath(kNodeParamDeleteStaleData)), A<bool &>()))
+    .WillOnce(Return(AwsError::AWS_ERR_NOT_FOUND));
+  }
+
+  Aws::FileManagement::FileManagerStrategyOptions options_{"test", "log_tests/", ".log", 1024*1024, 1024*1024};
+  ASSERT_FALSE(options_.delete_stale_data);
+
+  ReadOption(param_reader_, kNodeParamDeleteStaleData, Aws::FileManagement::kDefaultFileManagerStrategyOptions.delete_stale_data, options_.delete_stale_data);
+  ASSERT_TRUE(options_.delete_stale_data);
+
+  ReadOption(param_reader_, kNodeParamDeleteStaleData, Aws::FileManagement::kDefaultFileManagerStrategyOptions.delete_stale_data, options_.delete_stale_data);
+  ASSERT_FALSE(options_.delete_stale_data);
+
+  ReadOption(param_reader_, kNodeParamDeleteStaleData, Aws::FileManagement::kDefaultFileManagerStrategyOptions.delete_stale_data, options_.delete_stale_data);
+  ASSERT_TRUE(options_.delete_stale_data);
+
+  ReadOption(param_reader_, kNodeParamDeleteStaleData, Aws::FileManagement::kDefaultFileManagerStrategyOptions.delete_stale_data, options_.delete_stale_data);
+  ASSERT_FALSE(options_.delete_stale_data);
+}
 
 int main(int argc, char ** argv)
 {