Update to version v2.2.2 (#306)

* Update to version v2.1.1

* Update version to v2.2.1

* Update version to v2.2.2

---------

Co-authored-by: James Ma <[email protected]>

Author: James96315

CHANGELOG.md

@@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.2.2] - 2024-08-23
+
+### Changed
+
+- Support editing the auto-generated Regular Expression of Nginx and Apache log in the Log Config. #301
+- Adjusted the error logs from info level to error level in the log-processor function. #305
+
+### Fixed
+
+- Fixed failed to retrieve instance status due to too many instances in the Instance Group details page. #298
+- Remove the redundant sign in alert dialog modal when user session expired. #303
+- Fixed the Grafana url validation failed with spaces when import a Grafana Server. #304
+
 ## [2.2.1] - 2024-07-22
 
 ### Added

source/constructs/lambda/api/app_log_ingestion/flb/flb_builder.py

@@ -331,6 +331,11 @@ def _get_flb_params(self):
             WithDecryption=True,
         )["Parameter"]["Value"]
 
+        storage_total_limit_size = ssm_cli.get_parameter(
+            Name=f"/{stack_prefix}/FLB/storage_total_limit_size",
+            WithDecryption=True,
+        )["Parameter"]["Value"]
+
         flb_params = {
             "log_level": log_level,
             "flush": flush,
@@ -342,6 +347,7 @@ def _get_flb_params(self):
             "store_dir_limit_size": store_dir_limit_size,
             "storage_type": storage_type,
             "storage_pause_on_chunks_overlimit": storage_pause_on_chunks_overlimit,
+            "storage_total_limit_size": storage_total_limit_size,
         }
         return flb_params
 

source/constructs/lambda/api/app_log_ingestion/flb/flb_template/pipeline.conf

@@ -178,6 +178,7 @@
 {% endif %}
 {{placeholder}}    Retry_Limit                 {{ssm_params.retry_limit}}
 {{placeholder}}    Role_arn                    {{item.role_arn}}
+{{placeholder}}    storage.total_limit_size    {{ssm_params.storage_total_limit_size}}
 {{placeholder}}
 {% elif item.output_name=='MSK' %}
 {{placeholder}}[OUTPUT]
@@ -194,6 +195,7 @@
 {{placeholder}}    rdkafka.compression.type    snappy
 {{placeholder}}    rdkafka.security.protocol   plaintext
 {{placeholder}}    Retry_Limit                 {{ssm_params.retry_limit}}
+{{placeholder}}    storage.total_limit_size    {{ssm_params.storage_total_limit_size}}
 {{placeholder}}
 {% elif item.output_name=='S3' %}
 {{placeholder}}[OUTPUT]
@@ -204,7 +206,9 @@
 {{placeholder}}    region                      {{item.region_name}}
 {{placeholder}}    total_file_size             {{item.s3.max_file_size}}M
 {{placeholder}}    upload_timeout              {{item.s3.upload_timeout}}s
-{{placeholder}}    store_dir_limit_size        {{ssm_params.store_dir_limit_size}}
+{{placeholder}}    #store_dir_limit_size        {{ssm_params.store_dir_limit_size}}
+{{placeholder}}    #FluentBit version 2.0 and above, please use the store_dir_limit_size parameter. Don't use storage.total_limit_size
+{{placeholder}}    storage.total_limit_size    {{ssm_params.store_dir_limit_size}}
 {{placeholder}}    use_put_object              true
 {% if item.s3.compression_type | lower == "gzip" %}
 {{placeholder}}    s3_key_format               /{{item.s3.prefix}}/%Y-%m-%d-%H-%M-%S-$UUID.gz
@@ -235,7 +239,7 @@
 {{placeholder}}    AWS_Auth                    On
 {{placeholder}}    TLS                         On
 {{placeholder}}    Suppress_Type_Name          On
-{{placeholder}}    Buffer_Size                 20M
+{{placeholder}}    Buffer_Size                 50M
 {{placeholder}}    #When enabled, generate _id for outgoing records. This prevents duplicate records when retrying.
 {{placeholder}}    Generate_ID                 On
 {{placeholder}}    Logstash_Format             Off
@@ -244,6 +248,7 @@
 {{placeholder}}    Time_Key_Nanos              Off
 {{placeholder}}    Write_Operation             create
 {{placeholder}}    AWS_Role_ARN                {{item.role_arn}}
+{{placeholder}}    storage.total_limit_size    {{ssm_params.storage_total_limit_size}}
 {{placeholder}}
 {% endif %}
 {% endwith %}

source/constructs/lambda/api/app_log_ingestion/test/test_flb.py

@@ -76,6 +76,12 @@ def mock_ssm_context():
             Type="String",
         )
 
+        ssm_client.put_parameter(
+            Name=f"/{stack_prefix}/FLB/storage_total_limit_size",
+            Value="off",
+            Type="String",
+        )
+
         yield
 
 

source/constructs/lambda/pipeline/log-processor/idx/idx_svc.py

@@ -91,10 +91,13 @@ def run_func_with_retry(
                 break
             logger.error("%s failed: %s", func_name, response.text)
             if response.status_code == 403 or response.status_code == 409:
-                logger.info(
-                    "Please add access to OpenSearch for this Lambda and rerun this"
-                )
+                logger.info("Please add access to OpenSearch for this Lambda")
                 if response.status_code == 403:
+                    logger.error(
+                        "the last response code is %d, the last response content is %s",
+                        response.status_code,
+                        response.content,
+                    )
                     self.map_backend_role()
                 raise APIException(
                     ErrorCode.UNKNOWN_ERROR,
@@ -107,7 +110,7 @@ def run_func_with_retry(
                     func_name,
                     retry,
                 )
-                logger.info(
+                logger.error(
                     "the last response code is %d, the last response content is %s",
                     response.status_code,
                     response.content,

source/constructs/lambda/pipeline/log-processor/idx/opensearch_client.py

@@ -120,9 +120,9 @@ def create_ism_policy(
             response = requests.put(
                 url, auth=self._awsauth, json=policy_doc, timeout=30
             )
-            logger.info("--> update ism policy response code %d", response.status_code)
+            logger.error("--> update ism policy response code %d", response.status_code)
             return response
-        logger.info(
+        logger.error(
             "the last response code is %d, the last response content is %s",
             response.status_code,
             response.content,
@@ -348,6 +348,12 @@ def add_master_role(self, role_arn: str):
                 status_code = resp["ResponseMetadata"]["HTTPStatusCode"]
                 logger.info("Response status: %d", status_code)
                 if status_code not in (200, 201):
+                    logger.error(
+                        "Add backend role %s to domain %s, response status: %d",
+                        role_arn,
+                        domain_name,
+                        status_code,
+                    )
                     raise APIException(
                         ErrorCode.UNKNOWN_ERROR,
                         "Failed to add backend role {role_arn} to domain {domain_name}",

source/constructs/lambda/pipeline/log-processor/lambda_function.py

@@ -71,7 +71,9 @@ def handle_sqs_retries(record):
             "This message has exceeded the maximum number of retries, verify that you can connect to OpenSearch or that the data type does not match the field type defined for the index",
         )
     else:
-        raise Exception(f"Error processing SQS message: {record}")
+        raise Exception(
+            f"Error processing SQS message: {record}, Lambda function has been called {approximate_receive_count} times, the message will be re-consumed and then retried!"
+        )
 
 
 def change_sqs_message_visibility(event_record):

source/constructs/lib/api/app-log-ingestion-stack.ts

@@ -122,11 +122,6 @@ export class AppLogIngestionStack extends Construct {
           description:
             'Download Fluent-Bit config file and reboot the Fluent-Bit',
           parameters: {
-            ARCHITECTURE: {
-              type: "String",
-              default: "",
-              description: "(Required) Machine Architecture"
-            },
             INSTANCEID: {
               type: 'String',
               default: '',
@@ -146,7 +141,7 @@ export class AppLogIngestionStack extends Construct {
               name: "updateFluentBitVersion",
               inputs: {
                 runCommand: [
-                  `[ -e /opt/fluent-bit/bin/fluent-bit ] && [ -z \"$(/opt/fluent-bit/bin/fluent-bit -V | grep '${FluentBitVersion}')\" ] && curl -o /opt/fluent-bit{{ARCHITECTURE}}.tar.gz 'https://${flb_s3_addr}/clo/${process.env.VERSION}/aws-for-fluent-bit/fluent-bit{{ARCHITECTURE}}.tar.gz' && tar xzvf /opt/fluent-bit{{ARCHITECTURE}}.tar.gz -C /opt/ --exclude=fluent-bit/etc; echo 0`
+                  `ARCHITECTURE=''; if [ \"$(uname -m)\" = \"aarch64\" ]; then ARCHITECTURE='-arm64'; fi; [ -e /opt/fluent-bit/bin/fluent-bit ] && [ -z \"$(/opt/fluent-bit/bin/fluent-bit -V | grep '${FluentBitVersion}')\" ] && curl -o /opt/fluent-bit$ARCHITECTURE.tar.gz https://${flb_s3_addr}/clo/${process.env.VERSION}/aws-for-fluent-bit/fluent-bit$ARCHITECTURE.tar.gz && tar xzvf /opt/fluent-bit$ARCHITECTURE.tar.gz -C /opt/ --exclude=fluent-bit/etc; echo 0`
                 ]
               },
             },

source/constructs/lib/api/fluent-bit-config-stack.ts

@@ -67,7 +67,7 @@ export class FluentBitConfigStack extends Construct {
         });
         new ssm.StringParameter(this, "FlbBufferSizeParameter", { //NOSONAR
             parameterName: `/${props.stackPrefix}/FLB/buffer_size`,
-            description: "Set the buffer size for HTTP client when reading responses from Kubernetes API server. A value of 0 results in no limit, and the buffer will expand as-needed.",
+            description: "Set the buffer size for HTTP client when reading responses from Kubernetes API server and the buffer size to read data in INPUT plugin. A value of 0 results in no limit, and the buffer will expand as-needed.",
             stringValue: "0",
         });
         new ssm.StringParameter(this, "FlbRetryLimitParameter", { //NOSONAR
@@ -90,6 +90,11 @@ export class FluentBitConfigStack extends Construct {
             description: "This parameter is to specifies if file storage is to be paused when reaching the chunk limit. Default is off",
             stringValue: "off",
         });
+        new ssm.StringParameter(this, "FlbStorageTotalLimitSizeParameter", { //NOSONAR
+            parameterName: `/${props.stackPrefix}/FLB/storage_total_limit_size`,
+            description: "This parameter is to limit the maximum number of Chunks in the filesystem for the current output logical destination. Default is 500M",
+            stringValue: "500M",
+        });
 
 
 

source/constructs/lib/api/instance-stack.ts

@@ -374,7 +374,7 @@ export class InstanceStack extends Construct {
         runtime: lambda.Runtime.PYTHON_3_11,
         handler: 'lambda_function.lambda_handler',
         timeout: Duration.minutes(5),
-        memorySize: 4096,
+        memorySize: 1024,
         logFormat: "JSON",
         applicationLogLevel: "INFO",
         systemLogLevel: "WARN",