Skip to content

Commit c609632

Browse files
TakoB222dragomirp
TakoB222
and
dragomirp
authored
Add new extensions/plugins (#312)
* Add new extensions/plugins * fixed unit charm test * fix lint test * add extension dependencies check function * fix tests * fix check extension dependencies * fix plugin enable/disable process * fix test * fix * fix lint test * fix Co-authored-by: Dragomir Penev <[email protected]> --------- Co-authored-by: Dragomir Penev <[email protected]>
1 parent 396377d commit c609632

File tree

6 files changed

+266
-6
lines changed

6 files changed

+266
-6
lines changed

‎config.yaml

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,82 @@ options:
219219
default: false
220220
type: boolean
221221
description: Enable spi extension
222+
plugin_bool_plperl_enable:
223+
default: false
224+
type: boolean
225+
description: Enable bool_plperl extension
226+
plugin_hll_enable:
227+
default: false
228+
type: boolean
229+
description: Enable hll extension
230+
plugin_hypopg_enable:
231+
default: false
232+
type: boolean
233+
description: Enable hypopg extension
234+
plugin_ip4r_enable:
235+
default: false
236+
type: boolean
237+
description: Enable ip4r extension
238+
plugin_plperl_enable:
239+
default: false
240+
type: boolean
241+
description: Enable plperl extension
242+
plugin_jsonb_plperl_enable:
243+
default: false
244+
type: boolean
245+
description: Enable jsonb_plperl extension
246+
plugin_orafce_enable:
247+
default: false
248+
type: boolean
249+
description: Enable orafce extension
250+
plugin_pg_similarity_enable:
251+
default: false
252+
type: boolean
253+
description: Enable pg_similarity extension
254+
plugin_prefix_enable:
255+
default: false
256+
type: boolean
257+
description: Enable prefix extension
258+
plugin_rdkit_enable:
259+
default: false
260+
type: boolean
261+
description: Enable rdkit extension
262+
plugin_tds_fdw_enable:
263+
default: false
264+
type: boolean
265+
description: Enable tds_fdw extension
266+
plugin_icu_ext_enable:
267+
default: false
268+
type: boolean
269+
description: Enable icu_ext extension
270+
plugin_pltcl_enable:
271+
default: false
272+
type: boolean
273+
description: Enable pltcl extension
274+
plugin_postgis_enable:
275+
default: false
276+
type: boolean
277+
description: Enable postgis extension
278+
plugin_address_standardizer_enable:
279+
default: false
280+
type: boolean
281+
description: Enable address_standardizer extension
282+
plugin_postgis_raster_enable:
283+
default: false
284+
type: boolean
285+
description: Enable postgis_raster extension
286+
plugin_address_standardizer_data_us_enable:
287+
default: false
288+
type: boolean
289+
description: Enable address_standardizer_data_us extension
290+
plugin_postgis_tiger_geocoder_enable:
291+
default: false
292+
type: boolean
293+
description: Enable postgis_tiger_geocoder extension
294+
plugin_postgis_topology_enable:
295+
default: false
296+
type: boolean
297+
description: Enable postgis_topology extension
222298
plugin_vector_enable:
223299
default: false
224300
type: boolean

‎lib/charms/postgresql_k8s/v0/postgresql.py

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@
2626
from psycopg2 import sql
2727
from psycopg2.sql import Composed
2828

29+
from collections import OrderedDict
30+
2931
# The unique Charmhub library identifier, never change it
3032
LIBID = "24ee217a54e840a598ff21a079c3e678"
3133

@@ -34,11 +36,24 @@
3436

3537
# Increment this PATCH version before using `charmcraft publish-lib` or reset
3638
# to 0 if you are raising the major API version
37-
LIBPATCH = 21
39+
LIBPATCH = 22
3840

3941
INVALID_EXTRA_USER_ROLE_BLOCKING_MESSAGE = "invalid role(s) for extra user roles"
4042

4143

44+
REQUIRED_PLUGINS = {
45+
"address_standardizer": ["postgis"],
46+
"address_standardizer_data_us": ["postgis"],
47+
"jsonb_plperl": ["plperl"],
48+
"postgis_raster": ["postgis"],
49+
"postgis_tiger_geocoder": ["postgis", "fuzzystrmatch"],
50+
"postgis_topology": ["postgis"],
51+
}
52+
DEPENDENCY_PLUGINS = set()
53+
for dependencies in REQUIRED_PLUGINS.values():
54+
DEPENDENCY_PLUGINS |= set(dependencies)
55+
56+
4257
logger = logging.getLogger(__name__)
4358

4459

@@ -289,12 +304,18 @@ def enable_disable_extensions(self, extensions: Dict[str, bool], database: str =
289304
cursor.execute("SELECT datname FROM pg_database WHERE NOT datistemplate;")
290305
databases = {database[0] for database in cursor.fetchall()}
291306

307+
orderedExtensions = OrderedDict()
308+
for plugin in DEPENDENCY_PLUGINS:
309+
orderedExtensions[plugin] = extensions.get(plugin, False)
310+
for extension, enable in extensions.items():
311+
orderedExtensions[extension] = enable
312+
292313
# Enable/disabled the extension in each database.
293314
for database in databases:
294315
with self._connect_to_database(
295316
database=database
296317
) as connection, connection.cursor() as cursor:
297-
for extension, enable in extensions.items():
318+
for extension, enable in orderedExtensions.items():
298319
cursor.execute(
299320
f"CREATE EXTENSION IF NOT EXISTS {extension};"
300321
if enable

‎src/charm.py

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
from charms.grafana_agent.v0.cos_agent import COSAgentProvider
1616
from charms.operator_libs_linux.v2 import snap
1717
from charms.postgresql_k8s.v0.postgresql import (
18+
REQUIRED_PLUGINS,
1819
PostgreSQL,
1920
PostgreSQLCreateUserError,
2021
PostgreSQLEnableDisableExtensionError,
@@ -88,6 +89,7 @@
8889
logger = logging.getLogger(__name__)
8990

9091
NO_PRIMARY_MESSAGE = "no primary in the cluster"
92+
EXTENSIONS_DEPENDENCY_MESSAGE = "Unsatisfied plugin dependencies. Please check the logs"
9193

9294
Scopes = Literal[APP_SCOPE, UNIT_SCOPE]
9395

@@ -859,8 +861,8 @@ def enable_disable_extensions(self, database: str = None) -> None:
859861
database: optional database where to enable/disable the extension.
860862
"""
861863
spi_module = ["refint", "autoinc", "insert_username", "moddatetime"]
862-
original_status = self.unit.status
863864
plugins_exception = {"uuid_ossp": '"uuid-ossp"'}
865+
original_status = self.unit.status
864866
extensions = {}
865867
# collect extensions
866868
for plugin in self.config.plugin_keys():
@@ -872,16 +874,33 @@ def enable_disable_extensions(self, database: str = None) -> None:
872874
for ext in spi_module:
873875
extensions[ext] = enable
874876
continue
875-
if extension in plugins_exception:
876-
extension = plugins_exception[extension]
877+
extension = plugins_exception.get(extension, extension)
878+
if self._check_extension_dependencies(extension, enable):
879+
self.unit.status = BlockedStatus(EXTENSIONS_DEPENDENCY_MESSAGE)
880+
return
877881
extensions[extension] = enable
882+
if self.is_blocked and self.unit.status.message == EXTENSIONS_DEPENDENCY_MESSAGE:
883+
self.unit.status = ActiveStatus()
878884
self.unit.status = WaitingStatus("Updating extensions")
879885
try:
880886
self.postgresql.enable_disable_extensions(extensions, database)
881887
except PostgreSQLEnableDisableExtensionError as e:
882888
logger.exception("failed to change plugins: %s", str(e))
883889
self.unit.status = original_status
884890

891+
def _check_extension_dependencies(self, extension: str, enable: bool) -> bool:
892+
skip = False
893+
if enable and extension in REQUIRED_PLUGINS:
894+
for ext in REQUIRED_PLUGINS[extension]:
895+
if not self.config[f"plugin_{ext}_enable"]:
896+
skip = True
897+
logger.exception(
898+
"cannot enable %s, extension required %s to be enabled before",
899+
extension,
900+
ext,
901+
)
902+
return skip
903+
885904
def _get_ips_to_remove(self) -> Set[str]:
886905
"""List the IPs that were part of the cluster but departed."""
887906
old = self.members_ips

‎src/config.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,25 @@ class CharmConfig(BaseConfigModel):
6464
plugin_tsm_system_time_enable: bool
6565
plugin_uuid_ossp_enable: bool
6666
plugin_spi_enable: bool
67+
plugin_bool_plperl_enable: bool
68+
plugin_hll_enable: bool
69+
plugin_hypopg_enable: bool
70+
plugin_ip4r_enable: bool
71+
plugin_plperl_enable: bool
72+
plugin_jsonb_plperl_enable: bool
73+
plugin_orafce_enable: bool
74+
plugin_pg_similarity_enable: bool
75+
plugin_prefix_enable: bool
76+
plugin_rdkit_enable: bool
77+
plugin_tds_fdw_enable: bool
78+
plugin_icu_ext_enable: bool
79+
plugin_pltcl_enable: bool
80+
plugin_postgis_enable: bool
81+
plugin_address_standardizer_enable: bool
82+
plugin_address_standardizer_data_us_enable: bool
83+
plugin_postgis_tiger_geocoder_enable: bool
84+
plugin_postgis_topology_enable: bool
85+
plugin_postgis_raster_enable: bool
6786
plugin_vector_enable: bool
6887
request_date_style: Optional[str]
6988
request_standard_conforming_strings: Optional[bool]

‎tests/integration/test_plugins.py

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,29 @@
5757
AUTOINC_EXTENSION_STATEMENT = "CREATE TABLE ids (id int4, idesc text);CREATE TRIGGER ids_nextid BEFORE INSERT OR UPDATE ON ids FOR EACH ROW EXECUTE PROCEDURE autoinc (id, next_id);"
5858
INSERT_USERNAME_EXTENSION_STATEMENT = "CREATE TABLE username_test (name text, username text not null);CREATE TRIGGER insert_usernames BEFORE INSERT OR UPDATE ON username_test FOR EACH ROW EXECUTE PROCEDURE insert_username (username);"
5959
MODDATETIME_EXTENSION_STATEMENT = "CREATE TABLE mdt (moddate timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL);CREATE TRIGGER mdt_moddatetime BEFORE UPDATE ON mdt FOR EACH ROW EXECUTE PROCEDURE moddatetime (moddate);"
60+
BOOL_PLPERL_EXTENSION_STATEMENT = "CREATE FUNCTION hello_bool(bool) RETURNS TEXT TRANSFORM FOR TYPE bool LANGUAGE plperl AS $$ my $with_world = shift; return sprintf('hello%s', $with_world ? ' world' : ''); $$;"
61+
HLL_EXTENSION_STATEMENT = "CREATE TABLE hll_test (users hll);"
62+
HYPOPG_EXTENSION_STATEMENT = "CREATE TABLE hypopg_test (id integer, val text); SELECT hypopg_create_index('CREATE INDEX ON hypopg_test (id)');"
63+
IP4R_EXTENSION_STATEMENT = "CREATE TABLE ip4r_test (ip ip4);"
64+
JSONB_PLPERL_EXTENSION_STATEMENT = "CREATE OR REPLACE FUNCTION jsonb_plperl_test(val jsonb) RETURNS jsonb TRANSFORM FOR TYPE jsonb LANGUAGE plperl as $$ return $_[0]; $$;"
65+
ORAFCE_EXTENSION_STATEMENT = "SELECT add_months(date '2005-05-31',1);"
66+
PG_SIMILARITY_EXTENSION_STATEMENT = "SHOW pg_similarity.levenshtein_threshold;"
67+
PLPERL_EXTENSION_STATEMENT = "CREATE OR REPLACE FUNCTION plperl_test(name text) RETURNS text AS $$ return $_SHARED{$_[0]}; $$ LANGUAGE plperl;"
68+
PREFIX_EXTENSION_STATEMENT = "SELECT '123'::prefix_range @> '123456';"
69+
RDKIT_EXTENSION_STATEMENT = "SELECT is_valid_smiles('CCC');"
70+
TDS_FDW_EXTENSION_STATEMENT = "CREATE SERVER mssql_svr FOREIGN DATA WRAPPER tds_fdw OPTIONS (servername 'tds_fdw_test', port '3306', database 'tds_fdw_test', tds_version '7.1');"
71+
ICU_EXT_EXTENSION_STATEMENT = (
72+
'CREATE COLLATION "vat-lat" (provider = icu, locale = "la-VA-u-kn-true")'
73+
)
74+
PLTCL_EXTENSION_STATEMENT = (
75+
"CREATE FUNCTION pltcl_test(integer) RETURNS integer AS $$ return $1 $$ LANGUAGE pltcl STRICT;"
76+
)
77+
POSTGIS_EXTENSION_STATEMENT = "SELECT PostGIS_Full_Version();"
78+
ADDRESS_STANDARDIZER_EXTENSION_STATEMENT = "SELECT num, street, city, zip, zipplus FROM parse_address('1 Devonshire Place, Boston, MA 02109-1234');"
79+
ADDRESS_STANDARDIZER_DATA_US_EXTENSION_STATEMENT = "SELECT house_num, name, suftype, city, country, state, unit FROM standardize_address('us_lex', 'us_gaz', 'us_rules', 'One Devonshire Place, PH 301, Boston, MA 02109');"
80+
POSTGIS_TIGER_GEOCODER_EXTENSION_STATEMENT = "SELECT * FROM standardize_address('tiger.pagc_lex', 'tiger.pagc_gaz', 'tiger.pagc_rules', 'One Devonshire Place, PH 301, Boston, MA 02109-1234');"
81+
POSTGIS_TOPOLOGY_STATEMENT = "SELECT topology.CreateTopology('nyc_topo', 26918, 0.5);"
82+
POSTGIS_RASTER_STATEMENT = "CREATE TABLE test_postgis_raster (name varchar, rast raster);"
6083
VECTOR_EXTENSION_STATEMENT = (
6184
"CREATE TABLE vector_test (id bigserial PRIMARY KEY, embedding vector(3));"
6285
)
@@ -113,6 +136,25 @@ async def test_plugins(ops_test: OpsTest) -> None:
113136
INSERT_USERNAME_EXTENSION_STATEMENT,
114137
MODDATETIME_EXTENSION_STATEMENT,
115138
],
139+
"plugin_bool_plperl_enable": BOOL_PLPERL_EXTENSION_STATEMENT,
140+
"plugin_hll_enable": HLL_EXTENSION_STATEMENT,
141+
"plugin_postgis_enable": POSTGIS_EXTENSION_STATEMENT,
142+
"plugin_hypopg_enable": HYPOPG_EXTENSION_STATEMENT,
143+
"plugin_ip4r_enable": IP4R_EXTENSION_STATEMENT,
144+
"plugin_plperl_enable": PLPERL_EXTENSION_STATEMENT,
145+
"plugin_jsonb_plperl_enable": JSONB_PLPERL_EXTENSION_STATEMENT,
146+
"plugin_orafce_enable": ORAFCE_EXTENSION_STATEMENT,
147+
"plugin_pg_similarity_enable": ORAFCE_EXTENSION_STATEMENT,
148+
"plugin_prefix_enable": PREFIX_EXTENSION_STATEMENT,
149+
"plugin_rdkit_enable": RDKIT_EXTENSION_STATEMENT,
150+
"plugin_tds_fdw_enable": TDS_FDW_EXTENSION_STATEMENT,
151+
"plugin_icu_ext_enable": ICU_EXT_EXTENSION_STATEMENT,
152+
"plugin_pltcl_enable": PLTCL_EXTENSION_STATEMENT,
153+
"plugin_address_standardizer_enable": ADDRESS_STANDARDIZER_EXTENSION_STATEMENT,
154+
"plugin_address_standardizer_data_us_enable": ADDRESS_STANDARDIZER_DATA_US_EXTENSION_STATEMENT,
155+
"plugin_postgis_tiger_geocoder_enable": POSTGIS_TIGER_GEOCODER_EXTENSION_STATEMENT,
156+
"plugin_postgis_raster_enable": POSTGIS_RASTER_STATEMENT,
157+
"plugin_postgis_topology_enable": POSTGIS_TOPOLOGY_STATEMENT,
116158
"plugin_vector_enable": VECTOR_EXTENSION_STATEMENT,
117159
}
118160

‎tests/unit/test_charm.py

Lines changed: 84 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
from parameterized import parameterized
1919
from tenacity import RetryError
2020

21-
from charm import NO_PRIMARY_MESSAGE, PostgresqlOperatorCharm
21+
from charm import EXTENSIONS_DEPENDENCY_MESSAGE, NO_PRIMARY_MESSAGE, PostgresqlOperatorCharm
2222
from cluster import RemoveRaftMemberFailedError
2323
from constants import PEER, POSTGRESQL_SNAP_NAME, SECRET_INTERNAL_LABEL, SNAP_PACKAGES
2424
from tests.helpers import patch_network_get
@@ -281,6 +281,32 @@ def test_on_config_changed(
281281
_enable_disable_extensions.assert_called_once()
282282
_set_up_relation.assert_called_once()
283283

284+
@patch("subprocess.check_output", return_value=b"C")
285+
def test_check_extension_dependencies(self, _):
286+
with patch.object(PostgresqlOperatorCharm, "postgresql", Mock()) as _:
287+
# Test when plugins dependencies exception is not caused
288+
config = {
289+
"plugin_address_standardizer_enable": False,
290+
"plugin_postgis_enable": False,
291+
"plugin_address_standardizer_data_us_enable": False,
292+
"plugin_jsonb_plperl_enable": False,
293+
"plugin_plperl_enable": False,
294+
"plugin_postgis_raster_enable": False,
295+
"plugin_postgis_tiger_geocoder_enable": False,
296+
"plugin_fuzzystrmatch_enable": False,
297+
"plugin_postgis_topology_enable": False,
298+
}
299+
self.harness.update_config(config)
300+
self.harness.charm.enable_disable_extensions()
301+
self.assertFalse(isinstance(self.harness.model.unit.status, BlockedStatus))
302+
303+
# Test when plugins dependencies exception caused
304+
config["plugin_address_standardizer_enable"] = True
305+
self.harness.update_config(config)
306+
self.harness.charm.enable_disable_extensions()
307+
self.assertTrue(isinstance(self.harness.model.unit.status, BlockedStatus))
308+
self.assertEqual(self.harness.model.unit.status.message, EXTENSIONS_DEPENDENCY_MESSAGE)
309+
284310
@patch("subprocess.check_output", return_value=b"C")
285311
def test_enable_disable_extensions(self, _):
286312
with patch.object(PostgresqlOperatorCharm, "postgresql", Mock()) as postgresql_mock:
@@ -396,6 +422,63 @@ def test_enable_disable_extensions(self, _):
396422
plugin_spi_enable:
397423
default: false
398424
type: boolean
425+
plugin_bool_plperl_enable:
426+
default: false
427+
type: boolean
428+
plugin_hll_enable:
429+
default: false
430+
type: boolean
431+
plugin_hypopg_enable:
432+
default: false
433+
type: boolean
434+
plugin_ip4r_enable:
435+
default: false
436+
type: boolean
437+
plugin_plperl_enable:
438+
default: false
439+
type: boolean
440+
plugin_jsonb_plperl_enable:
441+
default: false
442+
type: boolean
443+
plugin_orafce_enable:
444+
default: false
445+
type: boolean
446+
plugin_pg_similarity_enable:
447+
default: false
448+
type: boolean
449+
plugin_prefix_enable:
450+
default: false
451+
type: boolean
452+
plugin_rdkit_enable:
453+
default: false
454+
type: boolean
455+
plugin_tds_fdw_enable:
456+
default: false
457+
type: boolean
458+
plugin_icu_ext_enable:
459+
default: false
460+
type: boolean
461+
plugin_pltcl_enable:
462+
default: false
463+
type: boolean
464+
plugin_postgis_enable:
465+
default: false
466+
type: boolean
467+
plugin_postgis_raster_enable:
468+
default: false
469+
type: boolean
470+
plugin_address_standardizer_enable:
471+
default: false
472+
type: boolean
473+
plugin_address_standardizer_data_us_enable:
474+
default: false
475+
type: boolean
476+
plugin_postgis_tiger_geocoder_enable:
477+
default: false
478+
type: boolean
479+
plugin_postgis_topology_enable:
480+
default: false
481+
type: boolean
399482
plugin_vector_enable:
400483
default: false
401484
type: boolean

0 commit comments

Comments
 (0)