Fix pipeline when user.name and postgresql.log.database with brackets (#3812)

Author: kaiyan-sheng

packages/postgresql/changelog.yml

@@ -1,3 +1,8 @@
+- version: "1.4.1"
+  changes:
+    - description: Fix pipeline when user.name and postgresql.log.database with brackets
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/3812
 - version: "1.4.0"
   changes:
     - description: Add support for AWS postgresql standard log format

packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-aws.log

@@ -9,4 +9,5 @@
 	       d.datctype as "Ctype",
 	       pg_catalog.array_to_string(d.datacl, E'\n') AS "Access privileges"
 	FROM pg_catalog.pg_database d
-	ORDER BY 1;
+	ORDER BY 1;
+2013-11-05 16:45:14 UTC:172.23.160.69(59176):[unknown]@[unknown]:[9002]:FATAL:  unsupported frontend protocol 1234.5679: server supports 2.0 to 3.0

packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-aws.log-expected.json

@@ -9,7 +9,7 @@
                 "category": [
                     "database"
                 ],
-                "ingested": "2022-07-18T21:14:43.737829400Z",
+                "ingested": "2022-07-22T16:21:07.906562100Z",
                 "kind": "event",
                 "original": "2022-06-16 17:58:40 UTC::@:[7770]:LOG:  checkpoint starting: time",
                 "timezone": "UTC",
@@ -42,7 +42,7 @@
                 "category": [
                     "database"
                 ],
-                "ingested": "2022-07-18T21:14:43.737839600Z",
+                "ingested": "2022-07-22T16:21:07.906572300Z",
                 "kind": "event",
                 "original": "2019-03-10 03:54:59 UTC:10.0.0.123(52834):postgres@tstdb:[20175]:ERROR: column \"wrong\" does not exist at character 8",
                 "timezone": "UTC",
@@ -86,7 +86,7 @@
                 "category": [
                     "database"
                 ],
-                "ingested": "2022-07-18T21:14:43.737847400Z",
+                "ingested": "2022-07-22T16:21:07.906578400Z",
                 "kind": "event",
                 "original": "2022-06-20 22:19:26 UTC:172.23.160.54(33998):accounters@found:[20452]:LOG:  statement: INSERT INTO events (user_id, user_principal_id, organization_id, region, cluster_id, invoice_id, trial_id, type, unique_token, data) VALUES (NULL, NULL, NULL, 'aws-eu-west-1', NULL, NULL, NULL, 'accounting', NULL, '{\\\"hostname\\\": \\\"44058ed0088d\\\", \\\"pid\\\": 41441}') RETURNING events.event_id",
                 "timezone": "UTC",
@@ -131,7 +131,7 @@
                 "category": [
                     "database"
                 ],
-                "ingested": "2022-07-18T21:14:43.737855100Z",
+                "ingested": "2022-07-22T16:21:07.906585200Z",
                 "kind": "event",
                 "original": "2013-11-05 16:51:10 UTC:[local]:master@postgres:[9193]:LOG:  statement: SELECT c.oid::pg_catalog.regclass FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid=i.inhparent AND i.inhrelid = '1255' ORDER BY inhseqno;",
                 "timezone": "UTC",
@@ -175,7 +175,7 @@
                 "category": [
                     "database"
                 ],
-                "ingested": "2022-07-18T21:14:43.737862700Z",
+                "ingested": "2022-07-22T16:21:07.906592500Z",
                 "kind": "event",
                 "original": "2013-11-05 16:45:14 UTC:[local]:master@postgres:[8839]:LOG:  statement: SELECT d.datname as \"Name\",\n\t       pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t       pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t       d.datcollate as \"Collate\",\n\t       d.datctype as \"Ctype\",\n\t       pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;",
                 "timezone": "UTC",
@@ -209,6 +209,50 @@
             "user": {
                 "name": "master"
             }
+        },
+        {
+            "@timestamp": "2013-11-05T16:45:14.000Z",
+            "ecs": {
+                "version": "8.0.0"
+            },
+            "event": {
+                "category": [
+                    "database"
+                ],
+                "ingested": "2022-07-22T16:21:07.906599400Z",
+                "kind": "event",
+                "original": "2013-11-05 16:45:14 UTC:172.23.160.69(59176):[unknown]@[unknown]:[9002]:FATAL:  unsupported frontend protocol 1234.5679: server supports 2.0 to 3.0",
+                "timezone": "UTC",
+                "type": [
+                    "info"
+                ]
+            },
+            "log": {
+                "level": "FATAL"
+            },
+            "message": "unsupported frontend protocol 1234.5679: server supports 2.0 to 3.0",
+            "postgresql": {
+                "log": {
+                    "client_addr": "172.23.160.69",
+                    "client_port": 59176,
+                    "database": "unknown",
+                    "timestamp": "2013-11-05 16:45:14 UTC"
+                }
+            },
+            "process": {
+                "pid": 9002
+            },
+            "related": {
+                "user": [
+                    "unknown"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "user": {
+                "name": "unknown"
+            }
         }
     ]
 }

packages/postgresql/data_stream/log/elasticsearch/ingest_pipeline/pipeline-aws-log.yml

@@ -5,7 +5,7 @@ processors:
       field: raw_message
       ignore_missing: true
       patterns:
-        - '(%{DATA:postgresql.log.client_addr}\(%{NUMBER:postgresql.log.client_port:int}\)|\[%{DATA:postgresql.log.client_addr}\])?:%{USERNAME:user.name}?@%{POSTGRESQL_DB_NAME:postgresql.log.database}?:(\[%{NUMBER:process.pid:long}\])?:%{WORD:log.level}: (%{POSTGRESQL_QUERY_STEP}: %{GREEDYDATA:postgresql.log.query}| %{GREEDYDATA:message}|%{GREEDYDATA:message})'
+        - '(%{DATA:postgresql.log.client_addr}\(%{NUMBER:postgresql.log.client_port:int}\)|\[%{DATA:postgresql.log.client_addr}\])?:(%{USERNAME:user.name}?@%{POSTGRESQL_DB_NAME:postgresql.log.database}?|\[%{USERNAME:user.name}?\]@\[%{POSTGRESQL_DB_NAME:postgresql.log.database}?\]):(\[%{NUMBER:process.pid:long}\])?:%{WORD:log.level}: (%{POSTGRESQL_QUERY_STEP}: %{GREEDYDATA:postgresql.log.query}| %{GREEDYDATA:message}|%{GREEDYDATA:message})'
       pattern_definitions:
         GREEDYDATA: '(.|\r|\n)*'
         POSTGRESQL_DB_NAME: '[a-zA-Z0-9_]+[a-zA-Z0-9_\$]*'

packages/postgresql/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: postgresql
 title: PostgreSQL
-version: 1.4.0
+version: 1.4.1
 license: basic
 description: Collect logs and metrics from PostgreSQL servers with Elastic Agent.
 type: integration