Merge pull request #2597 from sebix/jsonb

PostgreSQL: use JSONB for extra column

Author: sebix

CHANGELOG.md

@@ -41,6 +41,7 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o
 - `.github/workflows/codespell.yml`, `debian-package.yml`, `regexploit.yml`: Upgrade to `ubuntu-latest` runners (PR#2602 by Sebastian Wagner).
 
 ### Tools
+- `intelmq.bin.intelmq_psql_initdb`: Use `JSONB` type by default, Postgres supports it since version 9 (PR#2597 by Sebastian Wagner).
 
 ### Contrib
 

NEWS.md

@@ -24,6 +24,11 @@ Please refer to the change log for a full list of changes.
 ### Libraries
 
 ### Postgres databases
+To switch to the more efficient data type `jsonb` instead of `json`, use the following SQL statement. Data is preserved. JSONB also has more query and data manipulation features than plain JSON.
+```sql
+ALTER TABLE events
+   ALTER COLUMN "extra" SET DATA TYPE jsonb;
+```
 
 
 3.4.0 Feature release (2025-03-14)

intelmq/bin/intelmq_psql_initdb.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2015 Sebastian Wagner, 2023 CERT.at GmbH
+# SPDX-FileCopyrightText: 2015-2021 nic.at GmbH, 2022 Sebastian Wagner, 2023 CERT.at GmbH, 2025 Institute for Common Good Technology
 #
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
@@ -133,7 +133,7 @@ def _generate_separated_raws_schema(fields: dict, partition_key: str) -> list:
 
 def generate(harmonization_file=HARMONIZATION_CONF_FILE, skip_events=False,
              separate_raws=False, partition_key=None, skip_or_replace=False,
-             use_jsonb=False):
+             no_jsonb=False):
     FIELDS = {}
     sql_lines = []
 
@@ -171,7 +171,7 @@ def generate(harmonization_file=HARMONIZATION_CONF_FILE, skip_events=False,
         elif value['type'] == 'UUID':
             dbtype = 'UUID'
         elif value['type'] in ('JSON', 'JSONDict'):
-            dbtype = 'jsonb' if use_jsonb else 'json'
+            dbtype = 'json' if no_jsonb else 'jsonb'
         else:
             raise ValueError('Unknown type %r.' % value['type'])
 
@@ -213,8 +213,8 @@ def main():
                         help="Path to the harmonization file")
     parser.add_argument("--skip-or-replace", default=False, action="store_true",
                         help="Add IF NOT EXISTS or REPLACE directive to created schemas")
-    parser.add_argument("--jsonb", default=False, action="store_true",
-                        help="Use JSONB type to represent dictionary fields")
+    parser.add_argument("--no-jsonb", action="store_true",
+                        help="Do not use JSONB but JSON type to represent dictionary fields")
     args = parser.parse_args()
 
     OUTPUTFILE = args.outputfile
@@ -232,7 +232,7 @@ def main():
                         separate_raws=args.separate_raws,
                         partition_key=args.partition_key,
                         skip_or_replace=args.skip_or_replace,
-                        use_jsonb=args.jsonb,
+                        no_jsonb=args.no_jsonb,
                         )
         print("INFO - Writing %s file" % OUTPUTFILE)
         fp.write(psql)

intelmq/tests/bin/initdb.sql

@@ -32,7 +32,7 @@ CREATE TABLE events (
     "event_description.text" text,
     "event_description.url" text,
     "event_hash" varchar(40),
-    "extra" json,
+    "extra" jsonb,
     "feed.accuracy" real,
     "feed.code" varchar(100),
     "feed.documentation" text,
@@ -46,7 +46,7 @@ CREATE TABLE events (
     "malware.version" text,
     "misp.attribute_uuid" varchar(36),
     "misp.event_uuid" varchar(36),
-    "output" json,
+    "output" jsonb,
     "protocol.application" varchar(100),
     "protocol.transport" varchar(11),
     "raw" text,

intelmq/tests/bin/initdb.sql.license

@@ -1,2 +1,2 @@
-SPDX-FileCopyrightText: 2016 Sebastian Wagner
+SPDX-FileCopyrightText: 2016 nic.at GmbH
 SPDX-License-Identifier: AGPL-3.0-or-later