Add new AWS Security Hub Findings Full Posture data stream and update misconfig transform to use it

.github/CODEOWNERS

@@ -58,6 +58,7 @@
 /packages/aws/data_stream/s3_storage_lens @elastic/obs-infraobs-integrations
 /packages/aws/data_stream/s3access @elastic/obs-ds-hosted-services
 /packages/aws/data_stream/securityhub_findings @elastic/security-service-integrations
+/packages/aws/data_stream/securityhub_findings_full_posture @elastic/security-service-integrations
 /packages/aws/data_stream/securityhub_insights @elastic/security-service-integrations
 /packages/aws/data_stream/sns @elastic/obs-infraobs-integrations
 /packages/aws/data_stream/sqs @elastic/obs-infraobs-integrations

packages/aws/_dev/build/docs/securityhub.md

@@ -22,6 +22,7 @@ The [AWS Security Hub](https://docs.aws.amazon.com/securityhub/) integration col
 
   1. For the current integration package, it is recommended to have interval in hours.
   2. For the current integration package, it is compulsory to add Secret Access Key and Access Key ID.
+  3. Findings Full Posture data stream request all the hsitorical findings every 24 hours
 
 ## Logs
 
@@ -37,6 +38,18 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur
 
 {{fields "securityhub_findings"}}
 
+### Findings Full Posture
+
+This is the [`securityhub_findings_full_posture`](https://docs.aws.amazon.com/securityhub/1.0/APIReference/API_GetFindings.html#API_GetFindings_ResponseElements) data stream.
+
+{{event "securityhub_findings_full_posture"}}
+
+**ECS Field Reference**
+
+Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields.
+
+{{fields "securityhub_findings_full_posture"}}
+
 ### Insights
 
 This is the [`securityhub_insights`](https://docs.aws.amazon.com/securityhub/1.0/APIReference/API_GetInsights.html#API_GetInsights_ResponseElements) data stream.

packages/aws/changelog.yml

@@ -1,19 +1,24 @@
 # newer versions go on top
+- version: "3.0.0"
+  changes:
+    - description: Add new Security Hub Findings Full Posture data stream. If you rely on Findings > Misconfigurations view, enable this new data stream.
+      type: breaking-change
+      link: tba
 - version: "2.45.1"
   changes:
     - description: Fix handling of SQS worker count configuration.
       type: bugfix
       link: https://github.com/elastic/integrations/pull/13350
 - version: "2.45.0"
   changes:
-  - description: Update default data_stream.dataset to aws.cloudwatch_logs for cloudwatch_logs data stream.
-    type: breaking-change
-    link: https://github.com/elastic/integrations/pull/13370
+    - description: Update default data_stream.dataset to aws.cloudwatch_logs for cloudwatch_logs data stream.
+      type: breaking-change
+      link: https://github.com/elastic/integrations/pull/13370
 - version: "2.44.0"
   changes:
-  - description: Add `actor.entity.id` and `target.entity.id`
-    type: enhancement
-    link: https://github.com/elastic/integrations/pull/12685
+    - description: Add `actor.entity.id` and `target.entity.id`
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12685
 - version: "2.43.0"
   changes:
     - description: Set `event.type` and `event.action` fields in vpcflow logs.
@@ -36,7 +41,7 @@
       link: https://github.com/elastic/integrations/pull/12755
 - version: "2.40.0"
   changes:
-    - description: Add support for Kibana `9.0.0` 
+    - description: Add support for Kibana `9.0.0`
       type: enhancement
       link: https://github.com/elastic/integrations/pull/12637
 - version: "2.39.0"

packages/aws/data_stream/cloudtrail/fields/fields.yml

@@ -200,7 +200,6 @@
       fields:
         - name: id
           type: keyword
-
 - name: actor
   type: group
   fields:

packages/aws/data_stream/cloudtrail/manifest.yml

@@ -178,18 +178,17 @@ streams:
         show_user: false
         description: >
           Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details.
-      
+
       - name: actor_target_mapping
         required: true
         show_user: true
         title: Actor and Target Entity Mapping
         description: >
           Maps actor and target entity identifiers relative to an event into designated fields (`actor.entity.id` for the acting entity and `target.entity.id` for the affected entity/entities). All identifiers, regardless of role, are captured in the `related.entity` field.This introduces additional ingest pipeline processors for parsing, which may introduce performance overhead in certain cases.
+
         type: bool
         multi: false
         default: true
-
-
   - input: httpjson
     title: AWS CloudTrail Logs via Splunk Enterprise REST API
     description: Collect AWS CloudTrail logs via Splunk Enterprise REST API
@@ -298,17 +297,16 @@ streams:
         type: bool
         multi: false
         default: false
-
       - name: actor_target_mapping
         required: true
         show_user: true
         title: Actor and Target Entity Mapping
         description: >
           Maps actor and target entity identifiers relative to an event into designated fields (`actor.entity.id` for the acting entity and `target.entity.id` for the affected entity/entities). All identifiers, regardless of role, are captured in the `related.entity` field. This introduces additional ingest pipeline processors for parsing, which may introduce performance overhead in certain cases.
+
         type: bool
         multi: false
         default: true
-
   - input: aws-cloudwatch
     template_path: aws-cloudwatch.yml.hbs
     title: AWS CloudTrail Logs
@@ -435,13 +433,13 @@ streams:
         type: bool
         multi: false
         default: false
-
       - name: actor_target_mapping
         required: true
         show_user: true
         title: Actor and Target Entity Mapping
         description: >
           Maps actor and target entity identifiers relative to an event into designated fields (`actor.entity.id` for the acting entity and `target.entity.id` for the affected entity/entities). All identifiers, regardless of role, are captured in the `related.entity` field. This introduces additional ingest pipeline processors for parsing, which may introduce performance overhead in certain cases.
+
         type: bool
         multi: false
         default: true

packages/aws/data_stream/elb_logs/fields/fields.yml

@@ -136,38 +136,35 @@
       type: date
       description: >
         The time recorded at the beginning of the TLS connection.
-    
+
     - name: leaf_client_cert_not_after
       type: date
       description: >
         The time recorded at the start of the validity period of the leaf client certificate.
-    
+
     - name: leaf_client_cert_not_before
       type: date
       description: >
         The time recorded at the end of the validity period of the leaf client certificate.
-  
+
     - name: leaf_client_cert_serial_number
       type: keyword
       description: >
         The serial number of the leaf client certificate.
 
     - name: leaf_client_cert_subject
       type: keyword
-      description: >
-        The subject name of the leaf client certificate. 
-
+      description: "The subject name of the leaf client certificate. \n"
     - name: tls_error_code
       type: keyword
       description: >
         The reason recorded when the load balancer fails to establish a connection, stored as a code in the connection log.
 
     - name: tls_handshake_latency
       type: long
-      description: >
-        The total time in seconds, with a millisecond precision, elapsed while establishing a successful handshake.        
-
+      description: "The total time in seconds, with a millisecond precision, elapsed while establishing a successful handshake.        \n"
     - name: tls_verify_status
       type: keyword
       description: >
         The status of the connection request. This value is Success if the connection is established successfully. On an unsuccessful connection the value is Failed.
+

packages/aws/data_stream/securityhub_findings_full_posture/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,17 @@
+version: '2.3'
+services:
+  securityhub_full_posture:
+    image: docker.elastic.co/observability/stream:v0.15.0
+    hostname: securityhub.xxxx.amazonaws.cn
+    ports:
+      - 443
+    volumes:
+      - ./files:/files:ro
+    environment:
+      PORT: "443"
+    command:
+      - http-server
+      - --addr=:443
+      - --config=/files/config.yml
+      - --tls-cert=/files/certificate.crt
+      - --tls-key=/files/private.key

packages/aws/data_stream/securityhub_findings_full_posture/_dev/deploy/docker/files/certificate.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDUjCCAjoCCQDQ1VVKJuqgWjANBgkqhkiG9w0BAQsFADBrMQswCQYDVQQGEwJY
+WDEVMBMGA1UEBwwMRGVmYXVsdCBDaXR5MRwwGgYDVQQKDBNEZWZhdWx0IENvbXBh
+bnkgTHRkMScwJQYDVQQDDB5zZWN1cml0eWh1Yi54eHh4LmFtYXpvbmF3cy5jb20w
+HhcNMjIwNzA2MDg1MTUwWhcNMjMwNzA2MDg1MTUwWjBrMQswCQYDVQQGEwJYWDEV
+MBMGA1UEBwwMRGVmYXVsdCBDaXR5MRwwGgYDVQQKDBNEZWZhdWx0IENvbXBhbnkg
+THRkMScwJQYDVQQDDB5zZWN1cml0eWh1Yi54eHh4LmFtYXpvbmF3cy5jb20wggEi
+MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDhyLkZGxIdXMUb8UuD16U67hGi
+/W7SvhtHLkQGbHTWAD7+AAg5ybbBFa2LTf3G5lprgJ/nUAl5N2i7CnSOlRxm6yKU
+VeyXPzQ8327sb7Y1pm07hU2Y+unKXcCjQi4lgF9GUXgRFYGxzIiwbG52XgZNJ4Cq
+TWXAlRi8J4nJbSPty3R6wt2+bxIGf9/v6VoBpj0Ltal7aM9/YTGYkc+PprcoK6+x
+o5IzXha4iedNLjVRl7MLkP57BmDTTJpdO8OraddWjm1/I3kG5Lyu19A9URMg47vW
+L7IOtOZzfDNyCYbFwqNMHk62AVpTOYqL/icNlX+EpUxX4kyVhd4W0Y9xBs5HAgMB
+AAEwDQYJKoZIhvcNAQELBQADggEBAFA+VI+UgD2ldDLkfoCG+BNtasm9dyJvuer+
+9+R8IyMDL0O8ppLSpKny7MbTLFKymIkTFJzCKf3+q5cL/y4W5YRPsm3tYD8wzBfN
+o+sG2e1UlmMtv0vU4dsmoeHqYFyuxuDlgtH0FynCYgh+Xo6s6zPpNi48QsLebIf9
+Bp0lgklIyHpVhMTwUua5P0t00ecKvkCNf51x/apqyRYBdoAvrwQ9IRVPmvu/iQCR
+3AMQH0dhaDjS3aVzKyRrhu+jjEAFRV5yVr64LTkQAWzMb6yz1KaQa0OjXNV1wX4F
+/k5zhqX0C0HAvDkSKXqwtUXl8jKyvP3Ogwddzg17932lVJe/3jc=
+-----END CERTIFICATE-----

packages/aws/data_stream/securityhub_findings_full_posture/_dev/deploy/docker/files/config.yml

@@ -0,0 +1,7 @@
+rules:
+  - path: /findings
+    methods: ["POST"]
+    responses:
+      - status_code: 200
+        body: |
+          {"Findings":[{"Action":{"ActionType":"PORT_PROBE","PortProbeAction":{"PortProbeDetails":[{"LocalPortDetails":{"Port":80,"PortName":"HTTP"},"LocalIpDetails":{"IpAddressV4":"1.128.0.0"},"RemoteIpDetails":{"Country":{"CountryName":"Example Country"},"City":{"CityName":"Example City"},"GeoLocation":{"Lon":0,"Lat":0},"Organization":{"AsnOrg":"ExampleASO","Org":"ExampleOrg","Isp":"ExampleISP","Asn":64496}}}],"Blocked":false}},"AwsAccountId":"111111111111","CompanyName":"AWS","Compliance":{"RelatedRequirements":["Req1","Req2"],"Status":"PASSED","StatusReasons":[{"ReasonCode":"CLOUDWATCH_ALARMS_NOT_PRESENT","Description":"CloudWatch alarms do not exist in the account"}]},"Confidence":42,"CreatedAt":"2017-03-22T13:22:13.933Z","Criticality":99,"Description":"The version of openssl found on instance i-abcd1234 is known to contain a vulnerability.","FindingProviderFields":{"Confidence":42,"Criticality":99,"RelatedFindings":[{"ProductArn":"arn:aws:securityhub:us-west-2::product/aws/guardduty","Id":"123e4567-e89b-12d3-a456-426655440000"}],"Severity":{"Label":"MEDIUM","Original":"MEDIUM"},"Types":["Software and Configuration Checks/Vulnerabilities/CVE"]},"FirstObservedAt":"2017-03-22T13:22:13.933Z","GeneratorId":"acme-vuln-9ab348","Id":"us-west-2/111111111111/98aebb2207407c87f51e89943f12b1ef","LastObservedAt":"2017-03-23T13:22:13.933Z","Malware":[{"Name":"Stringler","Type":"COIN_MINER","Path":"/usr/sbin/stringler","State":"OBSERVED"}],"Network":{"Direction":"IN","OpenPortRange":{"Begin":443,"End":443},"Protocol":"TCP","SourceIpV4":"1.128.0.0","SourceIpV6":"2a02:cf40::","SourcePort":"42","SourceDomain":"example1.com","SourceMac":"00:0d:83:b1:c0:8e","DestinationIpV4":"1.128.0.0","DestinationIpV6":"2a02:cf40::","DestinationPort":"80","DestinationDomain":"example2.com"},"NetworkPath":[{"ComponentId":"abc-01a234bc56d8901ee","ComponentType":"AWS::EC2::InternetGateway","Egress":{"Destination":{"Address":["1.128.0.0/24"],"PortRanges":[{"Begin":443,"End":443}]},"Protocol":"TCP","Source":{"Address":["175.16.199.1/24"]}},"Ingress":{"Destination":{"Address":["175.16.199.1/24"],"PortRanges":[{"Begin":443,"End":443}]},"Protocol":"TCP","Source":{"Address":["175.16.199.1/24"]}}}],"Note":{"Text":"Don't forget to check under the mat.","UpdatedBy":"jsmith","UpdatedAt":"2018-08-31T00:15:09Z"},"PatchSummary":{"Id":"pb-123456789098","InstalledCount":"100","MissingCount":"100","FailedCount":"0","InstalledOtherCount":"1023","InstalledRejectedCount":"0","InstalledPendingReboot":"0","OperationStartTime":"2018-09-27T23:37:31Z","OperationEndTime":"2018-09-27T23:39:31Z","RebootOption":"RebootIfNeeded","Operation":"Install"},"Process":{"Name":"syslogd","Path":"/usr/sbin/syslogd","Pid":12345,"ParentPid":56789,"LaunchedAt":"2018-09-27T22:37:31Z","TerminatedAt":"2018-09-27T23:37:31Z"},"ProductArn":"arn:aws:securityhub:us-east-1:111111111111:product/111111111111/default","ProductFields":{"generico/secure-pro/Count":"6","Service_Name":"cloudtrail.amazonaws.com","aws/inspector/AssessmentTemplateName":"My daily CVE assessment","aws/inspector/AssessmentTargetName":"My prod env","aws/inspector/RulesPackageName":"Common Vulnerabilities and Exposures"},"ProductName":"Security Hub","RecordState":"ACTIVE","Region":"us-east-1","RelatedFindings":[{"ProductArn":"arn:aws:securityhub:us-west-2::product/aws/guardduty","Id":"123e4567-e89b-12d3-a456-426655440000"},{"ProductArn":"arn:aws:securityhub:us-west-2::product/aws/guardduty","Id":"AcmeNerfHerder-111111111111-x189dx7824"}],"Remediation":{"Recommendation":{"Text":"Run sudo yum update and cross your fingers and toes.","Url":"http://myfp.com/recommendations/dangerous_things_and_how_to_fix_them.html"}},"Resources":[{"Type":"AwsEc2Instance","Id":"i-cafebabe","Partition":"aws","Region":"us-west-2","Tags":{"billingCode":"Lotus-1-2-3","needsPatching":"true"},"Details":{"IamInstanceProfileArn":"arn:aws:iam::123456789012:role/IamInstanceProfileArn","ImageId":"ami-79fd7eee","IpV4Addresses":["175.16.199.1"],"IpV6Addresses":["2a02:cf40::"],"KeyName":"testkey","LaunchedAt":"2018-09-29T01:25:54Z","MetadataOptions":{"HttpEndpoint":"enabled","HttpProtocolIpv6":"enabled","HttpPutResponseHopLimit":1,"HttpTokens":"optional","InstanceMetadataTags":"disabled"},"NetworkInterfaces":[{"NetworkInterfaceId":"eni-e5aa89a3"}],"SubnetId":"PublicSubnet","Type":"i3.xlarge","VirtualizationType":"hvm","VpcId":"TestVPCIpv6"}}],"Sample":true,"SchemaVersion":"2018-10-08","Severity":{"Label":"CRITICAL","Original":"8.3"},"SourceUrl":"http://threatintelweekly.org/backdoors/8888","ThreatIntelIndicators":[{"Type":"IPV4_ADDRESS","Value":"175.16.199.1","Category":"BACKDOOR","LastObservedAt":"2018-09-27T23:37:31Z","Source":"Threat Intel Weekly","SourceUrl":"http://threatintelweekly.org/backdoors/8888"}],"Threats":[{"FilePaths":[{"FileName":"b.txt","FilePath":"/tmp/b.txt","Hash":"sha256","ResourceId":"arn:aws:ec2:us-west-2:123456789012:volume/vol-032f3bdd89aee112f"}],"ItemCount":3,"Name":"Iot.linux.mirai.vwisi","Severity":"HIGH"}],"Title":"EC2.20 Both VPN tunnels for an AWS Site-to-Site VPN connection should be up","Types":["Software and Configuration Checks/Vulnerabilities/CVE"],"UpdatedAt":"2018-08-31T00:15:09Z","UserDefinedFields":{"reviewedByCio":"true","comeBackToLater":"Check this again on Monday"},"VerificationState":"UNKNOWN","Vulnerabilities":[{"Cvss":[{"BaseScore":4.7,"BaseVector":"AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:N/A:N","Version":"V3"},{"BaseScore":4.7,"BaseVector":"AV:L/AC:M/Au:N/C:C/I:N/A:N","Version":"V2"}],"Id":"CVE-2020-12345","ReferenceUrls":["http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-12418","http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-17563"],"RelatedVulnerabilities":["CVE-2020-12345"],"Vendor":{"Name":"Alas","Url":"https://alas.aws.amazon.com/ALAS-2020-1337.html","VendorCreatedAt":"2020-01-16T00:01:43Z","VendorSeverity":"Medium","VendorUpdatedAt":"2020-01-16T00:01:43Z"},"VulnerablePackages":[{"Architecture":"x86_64","Epoch":"1","Name":"openssl","Release":"16.amzn2.0.3","Version":"1.0.2k"}]}],"Workflow":{"Status":"NEW"},"WorkflowState":"NEW"}]}

packages/aws/data_stream/securityhub_findings_full_posture/_dev/deploy/docker/files/private.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDhyLkZGxIdXMUb
+8UuD16U67hGi/W7SvhtHLkQGbHTWAD7+AAg5ybbBFa2LTf3G5lprgJ/nUAl5N2i7
+CnSOlRxm6yKUVeyXPzQ8327sb7Y1pm07hU2Y+unKXcCjQi4lgF9GUXgRFYGxzIiw
+bG52XgZNJ4CqTWXAlRi8J4nJbSPty3R6wt2+bxIGf9/v6VoBpj0Ltal7aM9/YTGY
+kc+PprcoK6+xo5IzXha4iedNLjVRl7MLkP57BmDTTJpdO8OraddWjm1/I3kG5Lyu
+19A9URMg47vWL7IOtOZzfDNyCYbFwqNMHk62AVpTOYqL/icNlX+EpUxX4kyVhd4W
+0Y9xBs5HAgMBAAECggEAF21MR16XspQ9n3iZ7UQi0MqC6faB2TwAeJJEXKZEOTAt
+WQ2HzPcxDzfAmgOtoUWlfCIMdWPIl9s38rBTB7hRChy7qciAk/Dq6qYETGQK8+Yg
+z1w1gPoH6AdyRX5Ia3u2ZwVs/9jLbDdct3GIxJ9c6ASBRSpitGjD+EHh+hRo9fNE
+bnTNCS9roukGIyXbRDJMplAoCLNI+HVjTkjWPq4mff6EeYuTCjPKoJVzsrp8Ecai
+Rf9a444KeUFlE4rcNmFtHJiVohJiPpIF85DUb8RBfVr8xSdoG6QHxaTcjbk3nPd2
+/x+NSY5O5PkEXbQpsBZmEo1Aba1qjeRg2pCNsP9tgQKBgQDxNBtroNv7uWMeQMKf
+fj4FtyvFfgfBt4fdUZblW60sWbRu2PnrwDyFxGGX+KKVFrKauS2R8SfSvX230kGl
+vbKXSxo10XjmmY0Kaulet7z9awjK+yTcj3HKqVpjCdZK0KO1FXwZ45hwM7ewB6KI
+xukbZPORJwbwIjBYAGt0mfSaTQKBgQDvondtX11L0qjDoqcW5a6o2cdkj1MjBfP+
+AKZqOKDNNeHG3hT/YWfcFUis/UXMV7TBG4NQuIRGu5xZn3WbxgynHx3/QiVKG90/
+m56hsAStcVHTVcPcAh48jgYF60u60jgUhBcyrAZpsskul+oY/v16Eutx5QqjGjnc
+3bmFZe/s4wKBgB2SeOYqM65aHVfhMrthO/NxcLFm8UaD3Ol6jliSc9njKacJfSK1
+T/ZKjHiYaD6FKOKlX3vsKCjDSL2XzqqmZlX8RDti8kK7grpLP094kXg0fkB8qBlO
+kPH673UDCL3ldJzIBI4cBF2FSbkQRpIkaQINz3r1YPliB7FSY9pI4d9lAoGAWGyz
+8vjonUz7l00SqQFR5N6PlAzLGbZdpVGqFrIUrASA7ngOeXoA8BYufh7rPY7zlPpJ
+B2U+8jbSZ8POiw+Wpah20jUfO2xyxMDw1Sr1Xubc0cXpAusJK0Eg+dgsVqCxruhb
+Awi1SRV+5SGLcXPOJtiKZrmkpjDMPzLV/WJzGQ8CgYAbcMtnLshdYVNXfutWgSm2
+TqYfGm/L+njAFXfSnIxotIw0jQVt/uB0okcNAHKTn1elCxC0v0BZDsSUhxToUGk+
+x1wfip3SVhR5sYg8HBYbDCkTKZerleeW5PzcFFf+BY4DxR+8yWNEA1PrAejKyXk5
+Id0GFdKT0A2niGndkyL7/A==
+-----END PRIVATE KEY-----

packages/aws/data_stream/securityhub_findings_full_posture/_dev/test/pipeline/test-common-config.yml

@@ -0,0 +1,6 @@
+fields:
+  tags:
+    - preserve_original_event
+    - preserve_duplicate_custom_fields
+dynamic_fields:
+  "@timestamp": ".*"