From 4ff437df6b74bd83cedb4ca0456d88ef0b2a3887 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Fri, 11 Apr 2025 10:58:44 +0200 Subject: [PATCH 1/2] psql_initdb: use JSONB by default Postgres supports it since version 9, IntelMQ should have defaulted to it since then --- CHANGELOG.md | 1 + intelmq/bin/intelmq_psql_initdb.py | 12 ++++++------ intelmq/tests/bin/initdb.sql | 4 ++-- intelmq/tests/bin/initdb.sql.license | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0747a31073..d27bde0047 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o - `.github/workflows/codespell.yml`, `debian-package.yml`, `regexploit.yml`: Upgrade to `ubuntu-latest` runners (PR#2602 by Sebastian Wagner). ### Tools +- `intelmq.bin.intelmq_psql_initdb`: Use `JSONB` type by default, Postgres supports it since version 9 (PR#2597 by Sebastian Wagner). ### Contrib diff --git a/intelmq/bin/intelmq_psql_initdb.py b/intelmq/bin/intelmq_psql_initdb.py index 5b715a0e7a..bc56c692b3 100644 --- a/intelmq/bin/intelmq_psql_initdb.py +++ b/intelmq/bin/intelmq_psql_initdb.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2015 Sebastian Wagner, 2023 CERT.at GmbH +# SPDX-FileCopyrightText: 2015-2021 nic.at GmbH, 2022 Sebastian Wagner, 2023 CERT.at GmbH, 2025 Institute for Common Good Technology # # SPDX-License-Identifier: AGPL-3.0-or-later @@ -133,7 +133,7 @@ def _generate_separated_raws_schema(fields: dict, partition_key: str) -> list: def generate(harmonization_file=HARMONIZATION_CONF_FILE, skip_events=False, separate_raws=False, partition_key=None, skip_or_replace=False, - use_jsonb=False): + no_jsonb=False): FIELDS = {} sql_lines = [] @@ -171,7 +171,7 @@ def generate(harmonization_file=HARMONIZATION_CONF_FILE, skip_events=False, elif value['type'] == 'UUID': dbtype = 'UUID' elif value['type'] in ('JSON', 'JSONDict'): - dbtype = 'jsonb' if use_jsonb else 'json' + dbtype = 'json' if no_jsonb else 'jsonb' else: raise ValueError('Unknown type %r.' % value['type']) @@ -213,8 +213,8 @@ def main(): help="Path to the harmonization file") parser.add_argument("--skip-or-replace", default=False, action="store_true", help="Add IF NOT EXISTS or REPLACE directive to created schemas") - parser.add_argument("--jsonb", default=False, action="store_true", - help="Use JSONB type to represent dictionary fields") + parser.add_argument("--no-jsonb", action="store_true", + help="Do not use JSONB but JSON type to represent dictionary fields") args = parser.parse_args() OUTPUTFILE = args.outputfile @@ -232,7 +232,7 @@ def main(): separate_raws=args.separate_raws, partition_key=args.partition_key, skip_or_replace=args.skip_or_replace, - use_jsonb=args.jsonb, + no_jsonb=args.no_jsonb, ) print("INFO - Writing %s file" % OUTPUTFILE) fp.write(psql) diff --git a/intelmq/tests/bin/initdb.sql b/intelmq/tests/bin/initdb.sql index 5a5f839f58..8be18d2c21 100644 --- a/intelmq/tests/bin/initdb.sql +++ b/intelmq/tests/bin/initdb.sql @@ -32,7 +32,7 @@ CREATE TABLE events ( "event_description.text" text, "event_description.url" text, "event_hash" varchar(40), - "extra" json, + "extra" jsonb, "feed.accuracy" real, "feed.code" varchar(100), "feed.documentation" text, @@ -46,7 +46,7 @@ CREATE TABLE events ( "malware.version" text, "misp.attribute_uuid" varchar(36), "misp.event_uuid" varchar(36), - "output" json, + "output" jsonb, "protocol.application" varchar(100), "protocol.transport" varchar(11), "raw" text, diff --git a/intelmq/tests/bin/initdb.sql.license b/intelmq/tests/bin/initdb.sql.license index f0b62ad2d3..094f5ca9b9 100644 --- a/intelmq/tests/bin/initdb.sql.license +++ b/intelmq/tests/bin/initdb.sql.license @@ -1,2 +1,2 @@ -SPDX-FileCopyrightText: 2016 Sebastian Wagner +SPDX-FileCopyrightText: 2016 nic.at GmbH SPDX-License-Identifier: AGPL-3.0-or-later From cab6de528b591aac7e7e022c9c27751259dd789c Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Tue, 22 Apr 2025 17:49:56 +0200 Subject: [PATCH 2/2] doc: add SQL ALTER fro extra to JSONB type to NEWS --- NEWS.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/NEWS.md b/NEWS.md index 12c225784f..95a6c0cbce 100644 --- a/NEWS.md +++ b/NEWS.md @@ -24,6 +24,11 @@ Please refer to the change log for a full list of changes. ### Libraries ### Postgres databases +To switch to the more efficient data type `jsonb` instead of `json`, use the following SQL statement. Data is preserved. JSONB also has more query and data manipulation features than plain JSON. +```sql +ALTER TABLE events + ALTER COLUMN "extra" SET DATA TYPE jsonb; +``` 3.4.0 Feature release (2025-03-14)