Enhance AWS S3 integration dashboard (elastic#4641)

Author: kaiyan-sheng

packages/aws/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.28.0"
+  changes:
+    - description: Enhance S3 integration dashboard
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/4641
 - version: "1.27.3"
   changes:
     - description: Support multiple forwarded IPs in cloudfront integration

packages/aws/data_stream/s3_request/agent/stream/stream.yml.hbs

@@ -37,19 +37,24 @@ metrics:
 - namespace: AWS/S3
   statistic: ["Average"]
   name:
+  - SelectScannedBytes
+  - SelectReturnedBytes
+  - BytesDownloaded
+  - BytesUploaded
+  - 4xxErrors
+  - 5xxErrors
+  - FirstByteLatency
+  - TotalRequestLatency
+- namespace: AWS/S3
+  statistic: ["Sum"]
+  name:
   - AllRequests
   - GetRequests
   - PutRequests
   - DeleteRequests
   - HeadRequests
   - PostRequests
   - SelectRequests
-  - SelectScannedBytes
-  - SelectReturnedBytes
   - ListRequests
   - BytesDownloaded
   - BytesUploaded
-  - 4xxErrors
-  - 5xxErrors
-  - FirstByteLatency
-  - TotalRequestLatency

packages/aws/data_stream/s3_request/elasticsearch/ingest_pipeline/default.yml

@@ -3,31 +3,31 @@ description: "Pipeline for S3 request metrics"
 
 processors:
   - rename:
-      field: aws.s3.metrics.AllRequests.avg
+      field: aws.s3.metrics.AllRequests.sum
       target_field: aws.s3_request.requests.total
       ignore_missing: true
   - rename:
-      field: aws.s3.metrics.GetRequests.avg
+      field: aws.s3.metrics.GetRequests.sum
       target_field: aws.s3_request.requests.get
       ignore_missing: true
   - rename:
-      field: aws.s3.metrics.PutRequests.avg
+      field: aws.s3.metrics.PutRequests.sum
       target_field: aws.s3_request.requests.put
       ignore_missing: true
   - rename:
-      field: aws.s3.metrics.DeleteRequests.avg
+      field: aws.s3.metrics.DeleteRequests.sum
       target_field: aws.s3_request.requests.delete
       ignore_missing: true
   - rename:
-      field: aws.s3.metrics.HeadRequests.avg
+      field: aws.s3.metrics.HeadRequests.sum
       target_field: aws.s3_request.requests.head
       ignore_missing: true
   - rename:
-      field: aws.s3.metrics.PostRequests.avg
+      field: aws.s3.metrics.PostRequests.sum
       target_field: aws.s3_request.requests.post
       ignore_missing: true
   - rename:
-      field: aws.s3.metrics.SelectRequests.avg
+      field: aws.s3.metrics.SelectRequests.sum
       target_field: aws.s3_request.requests.select
       ignore_missing: true
   - rename:
@@ -39,7 +39,7 @@ processors:
       target_field: aws.s3_request.requests.select_returned.bytes
       ignore_missing: true
   - rename:
-      field: aws.s3.metrics.ListRequests.avg
+      field: aws.s3.metrics.ListRequests.sum
       target_field: aws.s3_request.requests.list
       ignore_missing: true
   - rename:
@@ -50,6 +50,14 @@ processors:
       field: aws.s3.metrics.BytesUploaded.avg
       target_field: aws.s3_request.uploaded.bytes
       ignore_missing: true
+  - rename:
+      field: aws.s3.metrics.BytesDownloaded.sum
+      target_field: aws.s3_request.downloaded.bytes_per_period
+      ignore_missing: true
+  - rename:
+      field: aws.s3.metrics.BytesUploaded.sum
+      target_field: aws.s3_request.uploaded.bytes_per_period
+      ignore_missing: true
   - rename:
       field: aws.s3.metrics.4xxErrors.avg
       target_field: aws.s3_request.errors.4xx

packages/aws/data_stream/s3_request/fields/fields.yml

@@ -63,11 +63,21 @@
           format: bytes
           description: |
             The number bytes downloaded for requests made to an Amazon S3 bucket, where the response includes a body.
+        - name: downloaded.bytes_per_period
+          type: long
+          format: bytes
+          description: |
+            The number bytes per period downloaded for requests made to an Amazon S3 bucket, where the response includes a body.
         - name: uploaded.bytes
           type: long
           format: bytes
           description: |
             The number bytes uploaded that contain a request body, made to an Amazon S3 bucket.
+        - name: uploaded.bytes_per_period
+          type: long
+          format: bytes
+          description: |
+            The number bytes per period uploaded that contain a request body, made to an Amazon S3 bucket.
         - name: errors.4xx
           type: long
           description: |

packages/aws/docs/s3.md

@@ -505,6 +505,7 @@ An example event for `s3_request` looks as following:
 | aws.dimensions.StorageType | This dimension filters the data that you have stored in a bucket by types of storage. | keyword |
 | aws.s3.bucket.name | Name of a S3 bucket. | keyword |
 | aws.s3_request.downloaded.bytes | The number bytes downloaded for requests made to an Amazon S3 bucket, where the response includes a body. | long |
+| aws.s3_request.downloaded.bytes_per_period | The number bytes per period downloaded for requests made to an Amazon S3 bucket, where the response includes a body. | long |
 | aws.s3_request.errors.4xx | The number of HTTP 4xx client error status code requests made to an Amazon S3 bucket with a value of either 0 or 1. | long |
 | aws.s3_request.errors.5xx | The number of HTTP 5xx server error status code requests made to an Amazon S3 bucket with a value of either 0 or 1. | long |
 | aws.s3_request.latency.first_byte.ms | The per-request time from the complete request being received by an Amazon S3 bucket to when the response starts to be returned. | long |
@@ -520,6 +521,7 @@ An example event for `s3_request` looks as following:
 | aws.s3_request.requests.select_scanned.bytes | The number of bytes of data scanned with Amazon S3 SELECT Object Content requests in an Amazon S3 bucket. | long |
 | aws.s3_request.requests.total | The total number of HTTP requests made to an Amazon S3 bucket, regardless of type. | long |
 | aws.s3_request.uploaded.bytes | The number bytes uploaded that contain a request body, made to an Amazon S3 bucket. | long |
+| aws.s3_request.uploaded.bytes_per_period | The number bytes per period uploaded that contain a request body, made to an Amazon S3 bucket. | long |
 | aws.tags.\* | Tag key value pairs from aws resources. | object |
 | cloud | Fields related to the cloud or infrastructure the events are coming from. | group |
 | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword |