[DPE-1781] Configuration support (#239)

* Add missing parameters and add configuration support

* Use fixed snap

* Fix parameters validation

* Add validations

* Update library

* Revert changes on charm.py

* Add configuration logic and tests

* PR feedback (including the feedback from canonical/postgresql-k8s-operator#281)

* Fix TLS tests

* Increase timeout

Author: marceloneppel

config.yaml

@@ -2,6 +2,103 @@
 # See LICENSE file for licensing details.
 
 options:
+  durability_synchronous_commit:
+    description: |
+      Sets the current transactions synchronization level. This charm allows only the
+      “on”, “remote_apply” and “remote_write” values to avoid data loss if the primary
+      crashes and there are replicas.
+    type: string
+    default: "on"
+  instance_default_text_search_config:
+    description: |
+      Selects the text search configuration that is used by those variants of the text
+      search functions that do not have an explicit argument specifying it.
+      Allowed values start with “pg_catalog.” followed by a language name, like
+      “pg_catalog.english”.
+    type: string
+    default: "pg_catalog.simple"
+  instance_password_encryption:
+    description: |
+      Determines the algorithm to use to encrypt the password.
+      Allowed values are: “md5” and “scram-sha-256”.
+    type: string
+    default: "scram-sha-256"
+  logging_log_connections:
+    description: |
+      Logs each successful connection.
+    type: boolean
+    default: false
+  logging_log_disconnections:
+    description: |
+      Logs end of a session, including duration.
+    type: boolean
+    default: false
+  logging_log_lock_waits:
+    description: |
+      Logs long lock waits.
+    type: boolean
+    default: false
+  logging_log_min_duration_statement:
+    description: |
+      Sets the minimum running time (milliseconds) above which statements will be logged.
+      Allowed values are: from -1 to 2147483647 (-1 disables logging
+      statement durations).
+    type: int
+    default: -1
+  memory_maintenance_work_mem:
+    description: |
+      Sets the maximum memory (KB) to be used for maintenance operations.
+      Allowed values are: from 1024 to 2147483647.
+    type: int
+    default: 65536
+  memory_max_prepared_transactions:
+    description: |
+      Sets the maximum number of simultaneously prepared transactions.
+      Allowed values are: from 0 to 262143.
+    type: int
+    default: 0
+  memory_shared_buffers:
+    description: |
+      Sets the number of shared memory buffers (8 kB) used by the server. This charm allows
+      to set this value up to 40% of the available memory from the unit, as it is unlikely
+      that an allocation of more than that will work better than a smaller amount.
+      Allowed values are: from 16 to 1073741823.
+    type: int
+  memory_temp_buffers:
+    description: |
+      Sets the maximum number of temporary buffers (8 kB) used by each session.
+      Allowed values are: from 100 to 1073741823.
+    type: int
+    default: 1024
+  memory_work_mem:
+    description: |
+      Sets the maximum memory (KB) to be used for query workspaces.
+      Allowed values are: from 64 to 2147483647.
+    type: int
+    default: 4096
+  optimizer_constraint_exclusion:
+    description: |
+      Enables the planner to use constraints to optimize queries.
+      Allowed values are: “on”, “off” and “partition”.
+    type: string
+    default: "partition"
+  optimizer_default_statistics_target:
+    description: |
+      Sets the default statistics target. Allowed values are: from 1 to 10000.
+    type: int
+    default: 100
+  optimizer_from_collapse_limit:
+    description: |
+      Sets the FROM-list size beyond which subqueries are not collapsed.
+      Allowed values are: from 1 to 2147483647.
+    type: int
+    default: 8
+  optimizer_join_collapse_limit:
+    description: |
+      Sets the FROM-list size beyond which JOIN constructs are not flattened.
+      Allowed values are: from 1 to 2147483647.
+    type: int
+    default: 8
   plugin_citext_enable:
     default: false
     type: boolean
@@ -40,3 +137,84 @@ options:
       Amount of memory in Megabytes to limit PostgreSQL and associated process to.
       If unset, this will be decided according to the default memory limit in the selected profile.
       Only comes into effect when the `production` profile is selected.
+  request_date_style:
+    description: |
+      Sets the display format for date and time values. Allowed formats are explained
+      in https://www.postgresql.org/docs/14/runtime-config-client.html#GUC-DATESTYLE.
+    type: string
+    default: "ISO, MDY"
+  request_standard_conforming_strings:
+    description: |
+      Causes ... strings to treat backslashes literally.
+    type: boolean
+    default: true
+  request_time_zone:
+    description: |
+      Sets the time zone for displaying and interpreting time stamps.
+      Allowed values are the ones from IANA time zone data, a time zone abbreviation
+      like PST and POSIX-style time zone specifications.
+    type: string
+    default: "UTC"
+  response_bytea_output:
+    description: |
+      Sets the output format for bytes.
+      Allowed values are: “escape” and “hex”.
+    type: string
+    default: "hex"
+  response_lc_monetary:
+    description: |
+      Sets the locale for formatting monetary amounts.
+      Allowed values are the locales available in the unit.
+    type: string
+    default: "C"
+  response_lc_numeric:
+    description: |
+      Sets the locale for formatting numbers.
+      Allowed values are the locales available in the unit.
+    type: string
+    default: "C"
+  response_lc_time:
+    description: |
+      Sets the locale for formatting date and time values.
+      Allowed values are the locales available in the unit.
+    type: string
+    default: "C"
+  vacuum_autovacuum_analyze_scale_factor:
+    description: |
+      Specifies a fraction of the table size to add to autovacuum_vacuum_threshold when
+      deciding whether to trigger a VACUUM. The default, 0.1, means 10% of table size.
+      Allowed values are: from 0 to 100.
+    type: float
+    default: 0.1
+  vacuum_autovacuum_analyze_threshold:
+    description: |
+      Sets the minimum number of inserted, updated or deleted tuples needed to trigger
+      an ANALYZE in any one table. Allowed values are: from 0 to 2147483647.
+    type: int
+    default: 50
+  vacuum_autovacuum_freeze_max_age:
+    description: |
+      Maximum age (in transactions) before triggering autovacuum on a table to prevent
+      transaction ID wraparound. Allowed values are: from 100000 to 2000000000.
+    type: int
+    default: 200000000
+  vacuum_autovacuum_vacuum_cost_delay:
+    description: |
+      Sets cost delay value (milliseconds) that will be used in automatic VACUUM operations.
+      Allowed values are: from -1 to 100 (-1 tells PostgreSQL to use the regular
+      vacuum_cost_delay value).
+    type: float
+    default: 2.0
+  vacuum_autovacuum_vacuum_scale_factor:
+    description: |
+      Specifies a fraction of the table size to add to autovacuum_vacuum_threshold when
+      deciding whether to trigger a VACUUM. The default, 0.2, means 20% of table size.
+      Allowed values are: from 0 to 100.
+    type: float
+    default: 0.2
+  vacuum_vacuum_freeze_table_age:
+    description: |
+      Age (in transactions) at which VACUUM should scan whole table to freeze tuples.
+      Allowed values are: from 0 to 2000000000.
+    type: int
+    default: 150000000

lib/charms/postgresql_k8s/v0/postgresql.py

@@ -32,7 +32,7 @@
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 17
+LIBPATCH = 18
 
 INVALID_EXTRA_USER_ROLE_BLOCKING_MESSAGE = "invalid role(s) for extra user roles"
 
@@ -310,6 +310,32 @@ def enable_disable_extension(self, extension: str, enable: bool, database: str =
             if connection is not None:
                 connection.close()
 
+    def get_postgresql_text_search_configs(self) -> Set[str]:
+        """Returns the PostgreSQL available text search configs.
+
+        Returns:
+            Set of PostgreSQL text search configs.
+        """
+        with self._connect_to_database(
+            connect_to_current_host=True
+        ) as connection, connection.cursor() as cursor:
+            cursor.execute("SELECT CONCAT('pg_catalog.', cfgname) FROM pg_ts_config;")
+            text_search_configs = cursor.fetchall()
+            return {text_search_config[0] for text_search_config in text_search_configs}
+
+    def get_postgresql_timezones(self) -> Set[str]:
+        """Returns the PostgreSQL available timezones.
+
+        Returns:
+            Set of PostgreSQL timezones.
+        """
+        with self._connect_to_database(
+            connect_to_current_host=True
+        ) as connection, connection.cursor() as cursor:
+            cursor.execute("SELECT name FROM pg_timezone_names;")
+            timezones = cursor.fetchall()
+            return {timezone[0] for timezone in timezones}
+
     def get_postgresql_version(self) -> str:
         """Returns the PostgreSQL version.
 
@@ -445,12 +471,12 @@ def is_restart_pending(self) -> bool:
 
     @staticmethod
     def build_postgresql_parameters(
-        profile: str, available_memory: int, limit_memory: Optional[int] = None
-    ) -> Optional[Dict[str, str]]:
+        config_options: Dict, available_memory: int, limit_memory: Optional[int] = None
+    ) -> Optional[Dict]:
         """Builds the PostgreSQL parameters.
 
         Args:
-            profile: the profile to use.
+            config_options: charm config options containing profile and PostgreSQL parameters.
             available_memory: available memory to use in calculation in bytes.
             limit_memory: (optional) limit memory to use in calculation in bytes.
 
@@ -459,19 +485,60 @@ def build_postgresql_parameters(
         """
         if limit_memory:
             available_memory = min(available_memory, limit_memory)
+        profile = config_options["profile"]
         logger.debug(f"Building PostgreSQL parameters for {profile=} and {available_memory=}")
+        parameters = {}
+        for config, value in config_options.items():
+            # Filter config option not related to PostgreSQL parameters.
+            if not config.startswith(
+                (
+                    "durability",
+                    "instance",
+                    "logging",
+                    "memory",
+                    "optimizer",
+                    "request",
+                    "response",
+                    "vacuum",
+                )
+            ):
+                continue
+            parameter = "_".join(config.split("_")[1:])
+            if parameter in ["date_style", "time_zone"]:
+                parameter = "".join(x.capitalize() for x in parameter.split("_"))
+            parameters[parameter] = value
+        shared_buffers_max_value = int(int(available_memory * 0.4) / 10**6)
+        if parameters.get("shared_buffers", 0) > shared_buffers_max_value:
+            raise Exception(
+                f"Shared buffers config option should be at most 40% of the available memory, which is {shared_buffers_max_value}MB"
+            )
         if profile == "production":
             # Use 25% of the available memory for shared_buffers.
             # and the remaind as cache memory.
             shared_buffers = int(available_memory * 0.25)
             effective_cache_size = int(available_memory - shared_buffers)
-
-            parameters = {
-                "shared_buffers": f"{int(shared_buffers/10**6)}MB",
-                "effective_cache_size": f"{int(effective_cache_size/10**6)}MB",
-            }
-
-            return parameters
+            parameters.setdefault("shared_buffers", f"{int(shared_buffers/10**6)}MB")
+            parameters.update({"effective_cache_size": f"{int(effective_cache_size/10**6)}MB"})
         else:
             # Return default
-            return {"shared_buffers": "128MB"}
+            parameters.setdefault("shared_buffers", "128MB")
+        return parameters
+
+    def validate_date_style(self, date_style: str) -> bool:
+        """Validate a date style against PostgreSQL.
+
+        Returns:
+            Whether the date style is valid.
+        """
+        try:
+            with self._connect_to_database(
+                connect_to_current_host=True
+            ) as connection, connection.cursor() as cursor:
+                cursor.execute(
+                    sql.SQL(
+                        "SET DateStyle to {};",
+                    ).format(sql.Identifier(date_style))
+                )
+            return True
+        except psycopg2.Error:
+            return False

src/charm.py

@@ -5,7 +5,9 @@
 """Charmed Machine Operator for the PostgreSQL database."""
 import json
 import logging
+import os
 import subprocess
+import time
 from typing import Dict, List, Optional, Set
 
 from charms.data_platform_libs.v0.data_models import TypedCharmBase
@@ -1411,14 +1413,14 @@ def _is_workload_running(self) -> bool:
 
     def update_config(self, is_creating_backup: bool = False) -> bool:
         """Updates Patroni config file based on the existence of the TLS files."""
-        enable_tls = all(self.tls.get_tls_files())
+        enable_tls = self.is_tls_enabled
         limit_memory = None
         if self.config.profile_limit_memory:
             limit_memory = self.config.profile_limit_memory * 10**6
 
         # Build PostgreSQL parameters.
         pg_parameters = self.postgresql.build_postgresql_parameters(
-            self.config.profile, self.get_available_memory(), limit_memory
+            self.model.config, self.get_available_memory(), limit_memory
         )
 
         # Update and reload configuration based on TLS files availability.
@@ -1444,10 +1446,21 @@ def update_config(self, is_creating_backup: bool = False) -> bool:
             logger.debug("Early exit update_config: Patroni not started yet")
             return False
 
-        restart_postgresql = (
-            enable_tls != self.postgresql.is_tls_enabled()
-        ) or self.postgresql.is_restart_pending()
+        self._validate_config_options()
+
+        self._patroni.update_parameter_controller_by_patroni(
+            "max_connections", max(4 * os.cpu_count(), 100)
+        )
+        self._patroni.update_parameter_controller_by_patroni(
+            "max_prepared_transactions", self.config.memory_max_prepared_transactions
+        )
+
+        restart_postgresql = self.is_tls_enabled != self.postgresql.is_tls_enabled()
         self._patroni.reload_patroni_configuration()
+        # Sleep the same time as Patroni's loop_wait default value, which tells how much time
+        # Patroni will wait before checking the configuration file again to reload it.
+        time.sleep(10)
+        restart_postgresql = restart_postgresql or self.postgresql.is_restart_pending()
         self.unit_peer_data.update({"tls": "enabled" if enable_tls else ""})
 
         # Restart PostgreSQL if TLS configuration has changed
@@ -1473,6 +1486,28 @@ def update_config(self, is_creating_backup: bool = False) -> bool:
 
         return True
 
+    def _validate_config_options(self) -> None:
+        """Validates specific config options that need access to the database or to the TLS status."""
+        if (
+            self.config.instance_default_text_search_config is not None
+            and self.config.instance_default_text_search_config
+            not in self.postgresql.get_postgresql_text_search_configs()
+        ):
+            raise Exception(
+                "instance_default_text_search_config config option has an invalid value"
+            )
+
+        if self.config.request_date_style is not None and not self.postgresql.validate_date_style(
+            self.config.request_date_style
+        ):
+            raise Exception("request_date_style config option has an invalid value")
+
+        if (
+            self.config.request_time_zone is not None
+            and self.config.request_time_zone not in self.postgresql.get_postgresql_timezones()
+        ):
+            raise Exception("request_time_zone config option has an invalid value")
+
     def _update_relation_endpoints(self) -> None:
         """Updates endpoints and read-only endpoint in all relations."""
         self.postgresql_client_relation.update_endpoints()