updated Github Secret Scanning fingerprint with resolved_at (#3802)

kcreddy · web-flow · commit fb2eef1f07ff · 2022-07-25T14:07:03.000+05:30
* updated fingerprint with resolved_at

* update readme

* update version

* update if conditions and readme

* update readme
diff --git a/packages/github/_dev/build/docs/README.md b/packages/github/_dev/build/docs/README.md
@@ -21,7 +21,8 @@ To use this integration, you must be an organization owner, and you must use an
 
 The Code Scanning lets you retrieve all security vulnerabilities and coding errors from a repository setup using Github Advanced Security Code Scanning feature. See [About code scanning](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-code-scanning) for more details.
 
-To use this integration, you must use an access token with the `security_events` scope for private repos or `public_repo` scope for public repos.
+To use this integration, GitHub Apps must have the `security_events` read permission. 
+Or use a personal access token with the `security_events` scope for private repos or `public_repo` scope for public repos. See [List code scanning alerts](https://docs.github.com/en/enterprise-cloud@latest/rest/code-scanning#list-code-scanning-alerts-for-a-repository)
 
 {{fields "code_scanning"}}
 
@@ -32,7 +33,8 @@ To use this integration, you must use an access token with the `security_events`
 
 The Github Secret Scanning lets you retrieve secret scanning for advanced security alerts from a repository setup using Github Advanced Security Secret Scanning feature. See [About Secret scanning](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/about-secret-scanning) for more details.
 
-To use this integration, you must be an administrator for the repository or for the organization that owns the repository, and you must use a personal access token with the `repo` scope or `security_events` scope. For public repositories, you may instead use the public_repo scope.
+To use this integration, GitHub Apps must have the `secret_scanning_alerts` read permission. 
+Or you must be an administrator for the repository or for the organization that owns the repository, and you must use a personal access token with the `repo` scope or `security_events` scope. For public repositories, you may instead use the `public_repo` scope. See [List secret scanning alerts](https://docs.github.com/en/enterprise-cloud@latest/rest/secret-scanning#list-secret-scanning-alerts-for-a-repository)
 
 {{fields "secret_scanning"}}
 
diff --git a/packages/github/changelog.yml b/packages/github/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.2.2"
+  changes:
+    - description: Update Github Secret Scanning fingerprint with resolved_at
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/3802
 - version: "1.2.1"
   changes:
     - description: Update package name and description to align with standard wording
diff --git a/packages/github/data_stream/secret_scanning/_dev/test/pipeline/test-ghas-secret-scanning-json.log-expected.json b/packages/github/data_stream/secret_scanning/_dev/test/pipeline/test-ghas-secret-scanning-json.log-expected.json
@@ -1,13 +1,13 @@
 {
     "expected": [
         {
-            "@timestamp": "2020-11-06T18:48:51.000Z",
+            "@timestamp": "2020-11-07T02:47:13.000Z",
             "ecs": {
                 "version": "8.3.0"
             },
             "event": {
                 "action": "secret_scanning",
-                "created": "2020-11-06T18:48:51.000Z",
+                "created": "2020-11-06T18:48:51Z",
                 "original": "{\"number\":2,\"created_at\":\"2020-11-06T18:48:51Z\",\"url\":\"https://api.github.com/repos/owner/private-repo/secret-scanning/alerts/2\",\"html_url\":\"https://github.com/owner/private-repo/security/secret-scanning/2\",\"locations_url\":\"https://api.github.com/repos/owner/private-repo/secret-scanning/alerts/2/locations\",\"state\":\"resolved\",\"resolution\":\"false_positive\",\"resolved_at\":\"2020-11-07T02:47:13Z\",\"resolved_by\":{\"login\":\"monalisa\",\"id\":2,\"node_id\":\"MDQ6VXNlcjI=\",\"avatar_url\":\"https://alambic.github.com/avatars/u/2?\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/monalisa\",\"html_url\":\"https://github.com/monalisa\",\"followers_url\":\"https://api.github.com/users/monalisa/followers\",\"following_url\":\"https://api.github.com/users/monalisa/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/monalisa/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/monalisa/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/monalisa/subscriptions\",\"organizations_url\":\"https://api.github.com/users/monalisa/orgs\",\"repos_url\":\"https://api.github.com/users/monalisa/repos\",\"events_url\":\"https://api.github.com/users/monalisa/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/monalisa/received_events\",\"type\":\"User\",\"site_admin\":true},\"secret_type\":\"adafruit_io_key\",\"secret_type_display_name\":\"Adafruit IO Key\",\"secret\":\"aio_XXXXXXXXXXXXXXXXXXXXXXXXXXXX\",\"push_protection_bypassed_by\":{\"login\":\"monalisa\",\"id\":2,\"node_id\":\"MDQ6VXNlcjI=\",\"avatar_url\":\"https://alambic.github.com/avatars/u/2?\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/monalisa\",\"html_url\":\"https://github.com/monalisa\",\"followers_url\":\"https://api.github.com/users/monalisa/followers\",\"following_url\":\"https://api.github.com/users/monalisa/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/monalisa/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/monalisa/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/monalisa/subscriptions\",\"organizations_url\":\"https://api.github.com/users/monalisa/orgs\",\"repos_url\":\"https://api.github.com/users/monalisa/repos\",\"events_url\":\"https://api.github.com/users/monalisa/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/monalisa/received_events\",\"type\":\"User\",\"site_admin\":true},\"push_protection_bypassed\":true,\"push_protection_bypassed_at\":\"2020-11-06T21:48:51Z\"}"
             },
             "github": {
@@ -56,7 +56,7 @@
             },
             "event": {
                 "action": "secret_scanning",
-                "created": "2020-11-06T18:18:30.000Z",
+                "created": "2020-11-06T18:18:30Z",
                 "original": "{\"number\":1,\"created_at\":\"2020-11-06T18:18:30Z\",\"url\":\"https://api.github.com/repos/owner/repo/secret-scanning/alerts/1\",\"html_url\":\"https://github.com/owner/repo/security/secret-scanning/1\",\"locations_url\":\"https://api.github.com/repos/owner/private-repo/secret-scanning/alerts/1/locations\",\"state\":\"open\",\"resolution\":null,\"resolved_at\":null,\"resolved_by\":null,\"secret_type\":\"mailchimp_api_key\",\"secret_type_display_name\":\"Mailchimp API Key\",\"secret\":\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX-us2\",\"push_protection_bypassed_by\":null,\"push_protection_bypassed\":false,\"push_protection_bypassed_at\":null  }"
             },
             "github": {
@@ -84,7 +84,7 @@
             },
             "event": {
                 "action": "secret_scanning",
-                "created": "2022-07-07T12:56:24.000Z",
+                "created": "2022-07-07T12:56:24Z",
                 "original": "{\"number\":7,\"created_at\":\"2022-07-07T12:56:24Z\",\"updated_at\":\"2022-07-07T12:56:24Z\",\"url\":\"https://api.github.com/repos/kcreddy-org/dummy-pub-repo/secret-scanning/alerts/7\",\"html_url\":\"https://github.com/kcreddy-org/dummy-pub-repo/security/secret-scanning/7\",\"locations_url\":\"https://api.github.com/repos/kcreddy-org/dummy-pub-repo/secret-scanning/alerts/7/locations\",\"state\":\"open\",\"secret_type\":\"npm_access_token\",\"secret_type_display_name\":\"npm Access Token\",\"secret\":\"npm_A7WfAVLMKkzhcGGxyCH8kQiKgTJhtU1DsGCG\",\"resolution\":null,\"resolved_by\":null,\"resolved_at\":null,\"push_protection_bypassed\":true,\"push_protection_bypassed_by\":{\"login\":\"kcreddy\",\"id\":11301409,\"node_id\":\"MDQ6VXNlcjExMzAxNDA5\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/11301409?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/kcreddy\",\"html_url\":\"https://github.com/kcreddy\",\"followers_url\":\"https://api.github.com/users/kcreddy/followers\",\"following_url\":\"https://api.github.com/users/kcreddy/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/kcreddy/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/kcreddy/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/kcreddy/subscriptions\",\"organizations_url\":\"https://api.github.com/users/kcreddy/orgs\",\"repos_url\":\"https://api.github.com/users/kcreddy/repos\",\"events_url\":\"https://api.github.com/users/kcreddy/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/kcreddy/received_events\",\"type\":\"User\",\"site_admin\":false},\"push_protection_bypassed_at\":\"2022-07-07T12:55:53Z\"  }"
             },
             "github": {
@@ -123,7 +123,7 @@
             },
             "event": {
                 "action": "secret_scanning",
-                "created": "2022-07-07T12:54:02.000Z",
+                "created": "2022-07-07T12:54:02Z",
                 "original": "{\"number\":6,\"created_at\":\"2022-07-07T12:54:02Z\",\"updated_at\":\"2022-07-07T12:54:02Z\",\"url\":\"https://api.github.com/repos/kcreddy-org/dummy-pub-repo/secret-scanning/alerts/6\",\"html_url\":\"https://github.com/kcreddy-org/dummy-pub-repo/security/secret-scanning/6\",\"locations_url\":\"https://api.github.com/repos/kcreddy-org/dummy-pub-repo/secret-scanning/alerts/6/locations\",\"state\":\"open\",\"secret_type\":\"atlassian_api_token\",\"secret_type_display_name\":\"Atlassian API Token\",\"secret\":\"DobuHe3ygkLnhf0efFG05A81\",\"resolution\":null,\"resolved_by\":null,\"resolved_at\":null,\"push_protection_bypassed\":false,\"push_protection_bypassed_by\":null,\"push_protection_bypassed_at\":null  }"
             },
             "github": {
@@ -152,7 +152,7 @@
             },
             "event": {
                 "action": "secret_scanning",
-                "created": "2022-07-07T12:48:57.000Z",
+                "created": "2022-07-07T12:48:57Z",
                 "original": "{\"number\":5,\"created_at\":\"2022-07-07T12:48:57Z\",\"updated_at\":\"2022-07-07T12:48:57Z\",\"url\":\"https://api.github.com/repos/kcreddy-org/dummy-pub-repo/secret-scanning/alerts/5\",\"html_url\":\"https://github.com/kcreddy-org/dummy-pub-repo/security/secret-scanning/5\",\"locations_url\":\"https://api.github.com/repos/kcreddy-org/dummy-pub-repo/secret-scanning/alerts/5/locations\",\"state\":\"open\",\"secret_type\":\"atlassian_api_token\",\"secret_type_display_name\":\"Atlassian API Token\",\"secret\":\"SlHw1Z8v4PaQHIudLweh178G\",\"resolution\":null,\"resolved_by\":null,\"resolved_at\":null,\"push_protection_bypassed\":false,\"push_protection_bypassed_by\":null,\"push_protection_bypassed_at\":null  }"
             },
             "github": {
@@ -181,7 +181,7 @@
             },
             "event": {
                 "action": "secret_scanning",
-                "created": "2022-07-07T10:52:40.000Z",
+                "created": "2022-07-07T10:52:40Z",
                 "original": "{\"number\":4,\"created_at\":\"2022-07-07T10:52:40Z\",\"updated_at\":\"2022-07-07T10:52:40Z\",\"url\":\"https://api.github.com/repos/kcreddy-org/dummy-pub-repo/secret-scanning/alerts/4\",\"html_url\":\"https://github.com/kcreddy-org/dummy-pub-repo/security/secret-scanning/4\",\"locations_url\":\"https://api.github.com/repos/kcreddy-org/dummy-pub-repo/secret-scanning/alerts/4/locations\",\"state\":\"open\",\"secret_type\":\"atlassian_api_token\",\"secret_type_display_name\":\"Atlassian API Token\",\"secret\":\"W7PwnhKGwHMzwc3nHukPDAG6\",\"resolution\":null,\"resolved_by\":null,\"resolved_at\":null,\"push_protection_bypassed\":false,\"push_protection_bypassed_by\":null,\"push_protection_bypassed_at\":null  }"
             },
             "github": {
@@ -204,13 +204,13 @@
             ]
         },
         {
-            "@timestamp": "2022-07-07T10:52:40.000Z",
+            "@timestamp": "2022-07-07T12:45:43.000Z",
             "ecs": {
                 "version": "8.3.0"
             },
             "event": {
                 "action": "secret_scanning",
-                "created": "2022-07-07T10:52:40.000Z",
+                "created": "2022-07-07T10:52:40Z",
                 "original": "{\"number\":3,\"created_at\":\"2022-07-07T10:52:40Z\",\"updated_at\":\"2022-07-07T10:52:40Z\",\"url\":\"https://api.github.com/repos/kcreddy-org/dummy-pub-repo/secret-scanning/alerts/3\",\"html_url\":\"https://github.com/kcreddy-org/dummy-pub-repo/security/secret-scanning/3\",\"locations_url\":\"https://api.github.com/repos/kcreddy-org/dummy-pub-repo/secret-scanning/alerts/3/locations\",\"state\":\"resolved\",\"secret_type\":\"custom_pattern_2\",\"secret_type_display_name\":\"custom_pattern_2\",\"secret\":\"FAHf9g\",\"resolution\":\"wont_fix\",\"resolved_by\":{\"login\":\"kcreddy\",\"id\":11301409,\"node_id\":\"MDQ6VXNlcjExMzAxNDA5\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/11301409?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/kcreddy\",\"html_url\":\"https://github.com/kcreddy\",\"followers_url\":\"https://api.github.com/users/kcreddy/followers\",\"following_url\":\"https://api.github.com/users/kcreddy/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/kcreddy/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/kcreddy/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/kcreddy/subscriptions\",\"organizations_url\":\"https://api.github.com/users/kcreddy/orgs\",\"repos_url\":\"https://api.github.com/users/kcreddy/repos\",\"events_url\":\"https://api.github.com/users/kcreddy/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/kcreddy/received_events\",\"type\":\"User\",\"site_admin\":false},\"resolved_at\":\"2022-07-07T12:45:43Z\",\"push_protection_bypassed\":false,\"push_protection_bypassed_by\":null,\"push_protection_bypassed_at\":null  }"
             },
             "github": {
@@ -250,7 +250,7 @@
             },
             "event": {
                 "action": "secret_scanning",
-                "created": "2022-07-07T09:47:47.000Z",
+                "created": "2022-07-07T09:47:47Z",
                 "original": "{\"number\":2,\"created_at\":\"2022-07-07T09:47:47Z\",\"updated_at\":\"2022-07-07T09:47:51Z\",\"url\":\"https://api.github.com/repos/kcreddy-org/dummy-pub-repo/secret-scanning/alerts/2\",\"html_url\":\"https://github.com/kcreddy-org/dummy-pub-repo/security/secret-scanning/2\",\"locations_url\":\"https://api.github.com/repos/kcreddy-org/dummy-pub-repo/secret-scanning/alerts/2/locations\",\"state\":\"open\",\"secret_type\":\"custom_pattern_1\",\"secret_type_display_name\":\"custom_pattern_1\",\"secret\":\"custom_54fH8\",\"resolution\":null,\"resolved_by\":null,\"resolved_at\":null,\"push_protection_bypassed\":false,\"push_protection_bypassed_by\":null,\"push_protection_bypassed_at\":null  }"
             },
             "github": {
@@ -273,13 +273,13 @@
             ]
         },
         {
-            "@timestamp": "2022-07-07T09:23:23.000Z",
+            "@timestamp": "2022-07-07T10:13:56.000Z",
             "ecs": {
                 "version": "8.3.0"
             },
             "event": {
                 "action": "secret_scanning",
-                "created": "2022-07-07T09:23:23.000Z",
+                "created": "2022-07-07T09:23:23Z",
                 "original": "{\"number\":1,\"created_at\":\"2022-07-07T09:23:23Z\",\"updated_at\":\"2022-07-07T09:23:23Z\",\"url\":\"https://api.github.com/repos/kcreddy-org/dummy-pub-repo/secret-scanning/alerts/1\",\"html_url\":\"https://github.com/kcreddy-org/dummy-pub-repo/security/secret-scanning/1\",\"locations_url\":\"https://api.github.com/repos/kcreddy-org/dummy-pub-repo/secret-scanning/alerts/1/locations\",\"state\":\"resolved\",\"secret_type\":\"npm_access_token\",\"secret_type_display_name\":\"npm Access Token\",\"secret\":\"npm_2aZQ3QzGXlPbEgMMduZS1k0M1C0wNu3oqNbk\",\"resolution\":\"revoked\",\"resolved_by\":{\"login\":\"kcreddy\",\"id\":11301409,\"node_id\":\"MDQ6VXNlcjExMzAxNDA5\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/11301409?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/kcreddy\",\"html_url\":\"https://github.com/kcreddy\",\"followers_url\":\"https://api.github.com/users/kcreddy/followers\",\"following_url\":\"https://api.github.com/users/kcreddy/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/kcreddy/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/kcreddy/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/kcreddy/subscriptions\",\"organizations_url\":\"https://api.github.com/users/kcreddy/orgs\",\"repos_url\":\"https://api.github.com/users/kcreddy/repos\",\"events_url\":\"https://api.github.com/users/kcreddy/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/kcreddy/received_events\",\"type\":\"User\",\"site_admin\":false},\"resolved_at\":\"2022-07-07T10:13:56Z\",\"push_protection_bypassed\":false,\"push_protection_bypassed_by\":null,\"push_protection_bypassed_at\":null  }"
             },
             "github": {
diff --git a/packages/github/data_stream/secret_scanning/elasticsearch/ingest_pipeline/default.yml b/packages/github/data_stream/secret_scanning/elasticsearch/ingest_pipeline/default.yml
@@ -18,29 +18,35 @@ processors:
     fields: 
         - github.secret_scanning.number
         - github.secret_scanning.updated_at
+        - github.secret_scanning.resolved_at
     target_field: "_id"
     ignore_missing: true
+- set:
+    copy_from: github.secret_scanning.created_at
+    field: event.created
+    if: ctx.github.secret_scanning?.created_at != null
 - date:
     field: github.secret_scanning.created_at
     formats:
     - ISO8601
     timezone: UTC
-    target_field: "event.created"
-    if: ctx.github.secret_scanning?.created_at != null
+    target_field: "@timestamp"
+    if: ctx.github.secret_scanning?.created_at != null && ctx.github.secret_scanning?.updated_at == null && ctx.github.secret_scanning?.resolved_at == null
 - date:
-    field: github.secret_scanning.created_at
+    field: github.secret_scanning.updated_at
     formats:
     - ISO8601
     timezone: UTC
     target_field: "@timestamp"
-    if: ctx.github.secret_scanning?.created_at != null
+    if: ctx.github.secret_scanning?.updated_at != null && ctx.github.secret_scanning?.resolved_at == null
 - date:
-    field: github.secret_scanning.updated_at
+    field: github.secret_scanning.resolved_at
     formats:
     - ISO8601
     timezone: UTC
     target_field: "@timestamp"
-    if: ctx.github.secret_scanning?.updated_at != null
+    if: ctx.github.secret_scanning?.resolved_at != null
+
 - rename: 
     target_field: _temp.resolved_by
     field: github.secret_scanning.resolved_by
diff --git a/packages/github/data_stream/secret_scanning/sample_event.json b/packages/github/data_stream/secret_scanning/sample_event.json
@@ -1,8 +1,8 @@
 {
     "@timestamp": "2022-06-30T18:07:27.000Z",
     "agent": {
-        "ephemeral_id": "183ffdee-46fb-4023-90ba-a5af85bf6db2",
-        "id": "84b3a3da-c733-473b-8c02-cd9e4c7d1d8e",
+        "ephemeral_id": "49c616b3-b36b-4732-98a7-fc09eadb244f",
+        "id": "49202dc3-9434-459b-9a0c-a6ec637ef4e9",
         "name": "docker-fleet-agent",
         "type": "filebeat",
         "version": "8.3.0"
@@ -16,16 +16,16 @@
         "version": "8.3.0"
     },
     "elastic_agent": {
-        "id": "84b3a3da-c733-473b-8c02-cd9e4c7d1d8e",
+        "id": "49202dc3-9434-459b-9a0c-a6ec637ef4e9",
         "snapshot": false,
         "version": "8.3.0"
     },
     "event": {
         "action": "secret_scanning",
         "agent_id_status": "verified",
-        "created": "2022-06-30T18:07:27.000Z",
+        "created": "2022-06-30T18:07:27Z",
         "dataset": "github.secret_scanning",
-        "ingested": "2022-07-08T11:55:25Z",
+        "ingested": "2022-07-22T17:01:02Z",
         "original": "{\"created_at\":\"2022-06-30T18:07:27Z\",\"html_url\":\"https://github.com/sample_owner/sample_repo/security/secret-scanning/3\",\"number\":3,\"push_protection_bypassed\":true,\"push_protection_bypassed_by\":{\"html_url\":\"https://github.com/sample_owner\",\"login\":\"sample_owner\",\"type\":\"User\",\"url\":\"https://api.github.com/users/sample_owner\"},\"resolution\":\"revoked\",\"resolved_by\":{\"login\":\"sample_owner\",\"type\":\"User\",\"url\":\"https://api.github.com/users/sample_owner\"},\"secret\":\"npm_2vYJ3QzGXoGbEgMYduYS1k2M4D0wDu2opJbl\",\"secret_type\":\"npm_access_token\",\"secret_type_display_name\":\"npm Access Token\",\"state\":\"open\",\"url\":\"https://api.github.com/repos/sample_owner/sample_repo/secret-scanning/alerts/3\"}"
     },
     "github": {
diff --git a/packages/github/docs/README.md b/packages/github/docs/README.md
diff --git a/packages/github/manifest.yml b/packages/github/manifest.yml
