Rename [database].ssl* options to [database].tls* to support pymongo 4 (#6250)

Author: cognifloyd

CHANGELOG.rst

@@ -6,6 +6,21 @@ in development
 
 Python 3.6 is no longer supported; Stackstorm requires at least Python 3.8.
 
+Several st2.conf database options have been renamed or deprecated. Most of the options will continue to work using their old name.
+However, if you use `[database].ssl_keyfile` and/or `[database].ssl_certfile`, you MUST migrate to `[database].tls_certificate_key_file`.
+This new option expects the key and certificate in the same file. Use something like the following to create that file from your old files:
+
+```
+cat path/to/ssl_keyfile path/to/ssl_certfile > path/to/tls_certificate_key_file
+```
+
+Other options that were renamed under `[database]` are (more details available in `st2.conf.sample`):
+
+* `ssl` -> `tls`
+* `ssl_cert_reqs` -> `tls_allow_invalid_certificates` (opt type change: string -> boolean)
+* `ssl_ca_certs` -> `tls_ca_file`
+* `ssl_match_hostnames` -> `tls_allow_invalid_hostnames` (meaning is inverted: the new option is the opposite of the old)
+
 Fixed
 ~~~~~
 * Fixed #6021 and #5327 by adding max_page_size to api_opts and added limit and offset to list_values() methods of
@@ -31,6 +46,11 @@ Changed
 * Updated unit tests to use redis for coordination instead of the NoOp driver. This will hopefully make CI more stable. #6245
   Contributed by @FileMagic, @guzzijones, and @cognifloyd
 
+* Renamed `[database].ssl*` options to support pymongo 4, which we have to update to support newer MongoDB servers.
+  Please see the note above about migrating to the newer options, especially if you use `[database].ssl_keyfile`
+  and/or `[database].ssl_certfile`, as those options are ignored in StackStorm 3.9.0. #6250
+  Contributed by @cognifloyd
+
 Added
 ~~~~~
 * Continue introducing `pants <https://www.pantsbuild.org/docs>`_ to improve DX (Developer Experience)

conf/st2.conf.sample

@@ -19,7 +19,7 @@ logging = /etc/st2/logging.actionrunner.conf
 # List of pip options to be passed to "pip install" command when installing pack dependencies into pack virtual environment.
 pip_opts =  # comma separated list allowed here.
 # Python binary which will be used by Python actions.
-python_binary = /usr/bin/python
+python_binary = /usr/bin/python3
 # Default log level to use for Python runner actions. Can be overriden on invocation basis using "log_level" runner parameter.
 python_runner_log_level = DEBUG
 # Time interval between subsequent queries to check running executions.
@@ -106,11 +106,13 @@ index_url = https://index.stackstorm.org/v1/index.json # comma separated list al
 pack_group = st2packs
 # Paths which will be searched for integration packs.
 packs_base_paths = None
-# Paths which will be searched for runners. NOTE: This option has been deprecated and it's unused since StackStorm v3.0.0
+# Paths which will be searched for runners.
+# DEPRECATED FOR REMOVAL since 3.0.0: Option unused since StackStorm v3.0.0
 runners_base_paths = None
 # Path to the directory which contains system packs.
 system_packs_base_path = /opt/stackstorm/packs
-# Path to the directory which contains system runners. NOTE: This option has been deprecated and it's unused since StackStorm v3.0.0
+# Path to the directory which contains system runners.
+# DEPRECATED FOR REMOVAL since 3.0.0: Option unused since StackStorm v3.0.0
 system_runners_base_path = /opt/stackstorm/runners
 
 [coordination]
@@ -142,19 +144,33 @@ host = 127.0.0.1
 password = None
 # port of db server
 port = 27017
-# Create the connection to mongodb using SSL
-ssl = False
-# ca_certs file contains a set of concatenated CA certificates, which are used to validate certificates passed from MongoDB.
-ssl_ca_certs = None
 # Specifies whether a certificate is required from the other side of the connection, and whether it will be validated if provided
+# DEPRECATED FOR REMOVAL since 3.9.0: Use tls_allow_invalid_certificates with the following: The 'optional' and 'required' values are equivalent to tls_allow_invalid_certificates=False. The 'none' value is equivalent to tls_allow_invalid_certificates=True. This option is a needlessly more complex version of tls_allow_invalid_certificates.
 # Valid values: none, optional, required
 ssl_cert_reqs = None
 # Certificate file used to identify the localconnection
+# DEPRECATED FOR REMOVAL since 3.9.0: Use tls_certificate_key_file with a path to a file containing the concatenation of the files from ssl_keyfile and ssl_certfile. This option is ignored by pymongo.
 ssl_certfile = None
 # Private keyfile used to identify the local connection against MongoDB.
+# DEPRECATED FOR REMOVAL since 3.9.0: Use tls_certificate_key_file with a path to a file containing the concatenation of the files from ssl_keyfile and ssl_certfile. This option is ignored by pymongo.
 ssl_keyfile = None
 # If True and `ssl_cert_reqs` is not None, enables hostname verification
+# DEPRECATED FOR REMOVAL since 3.9.0: Use tls_allow_invalid_hostnames with the opposite value from this option.
 ssl_match_hostname = True
+# Create the connection to mongodb using TLS.
+# This option has a deprecated alias: ssl
+tls = False
+# Specifies whether MongoDB is allowed to pass an invalid certificate. This defaults to False to have security by default. Only temporarily set to True if you need to debug the connection.
+tls_allow_invalid_certificates = False
+# If True and `tlsAllowInvalidCertificates` is True, disables hostname verification. This defaults to False to have security by default. Only temporarily set to True if you need to debug the connection.
+tls_allow_invalid_hostnames = False
+# ca_certs file contains a set of concatenated CA certificates, which are used to validate certificates passed from MongoDB.
+# This option has a deprecated alias: ssl_ca_certs
+tls_ca_file = None
+# Client certificate used to identify the local connection against MongoDB. The certificate file must contain one or both of private key and certificate. Supplying separate files for private key (ssl_keyfile) and certificate (ssl_certfile) is no longer supported. If encrypted, pass the password or passphrase in tls_certificate_key_file_password.
+tls_certificate_key_file = None
+# The password or passphrase to decrypt the file in tls_certificate_key_file. Only set this if tls_certificate_key_file is encrypted.
+tls_certificate_key_file_password = None
 # username for db login
 username = None
 # Compression level when compressors is set to zlib. Valid values are -1 to 9. Defaults to 6.

contrib/packs/actions/pack_mgmt/unload.py

@@ -59,11 +59,12 @@ def initialize(self):
             cfg.CONF.database.port,
             username=username,
             password=password,
-            ssl=cfg.CONF.database.ssl,
-            ssl_keyfile=cfg.CONF.database.ssl_keyfile,
-            ssl_certfile=cfg.CONF.database.ssl_certfile,
-            ssl_cert_reqs=cfg.CONF.database.ssl_cert_reqs,
-            ssl_ca_certs=cfg.CONF.database.ssl_ca_certs,
+            tls=cfg.CONF.database.tls,
+            tls_certificate_key_file=cfg.CONF.database.tls_certificate_key_file,
+            tls_certificate_key_file_password=cfg.CONF.database.tls_certificate_key_file_password,
+            tls_allow_invalid_certificates=cfg.CONF.database.tls_allow_invalid_certificates,
+            tls_ca_file=cfg.CONF.database.tls_ca_file,
+            ssl_cert_reqs=cfg.CONF.database.ssl_cert_reqs,  # deprecated
             authentication_mechanism=cfg.CONF.database.authentication_mechanism,
             ssl_match_hostname=cfg.CONF.database.ssl_match_hostname,
         )

pants-plugins/sample_conf/rules.py

@@ -27,7 +27,7 @@
     FileContent,
     Snapshot,
 )
-from pants.engine.process import FallibleProcessResult
+from pants.engine.process import ProcessResult
 from pants.engine.rules import Get, collect_rules, rule
 from pants.engine.target import FieldSet
 from pants.util.logging import LogLevel
@@ -64,7 +64,7 @@ async def generate_sample_conf_via_fmt(
     pex = await Get(VenvPex, PexFromTargetsRequest, subsystem.pex_request())
 
     result = await Get(
-        FallibleProcessResult,
+        ProcessResult,
         VenvPexProcess(
             pex,
             description="Regenerating st2.conf.sample",

st2common/st2common/config.py

@@ -143,8 +143,10 @@ def register_opts(ignore_errors=False):
         cfg.StrOpt(
             "system_runners_base_path",
             default=system_runners_base_path,
-            help="Path to the directory which contains system runners. "
-            "NOTE: This option has been deprecated and it's unused since StackStorm v3.0.0",
+            help="Path to the directory which contains system runners.",
+            deprecated_for_removal=True,
+            deprecated_reason="Option unused since StackStorm v3.0.0",
+            deprecated_since="3.0.0",
         ),
         cfg.StrOpt(
             "packs_base_paths",
@@ -154,8 +156,10 @@ def register_opts(ignore_errors=False):
         cfg.StrOpt(
             "runners_base_paths",
             default=None,
-            help="Paths which will be searched for runners. "
-            "NOTE: This option has been deprecated and it's unused since StackStorm v3.0.0",
+            help="Paths which will be searched for runners.",
+            deprecated_for_removal=True,
+            deprecated_reason="Option unused since StackStorm v3.0.0",
+            deprecated_since="3.0.0",
         ),
         cfg.ListOpt(
             "index_url",
@@ -172,6 +176,7 @@ def register_opts(ignore_errors=False):
         cfg.StrOpt(
             "webui_base_url",
             default="https://%s" % socket.getfqdn(),
+            sample_default="https://localhost",
             help="Base https URL to access st2 Web UI. This is used to construct history URLs "
             "that are sent out when chatops is used to kick off executions.",
         )
@@ -184,7 +189,7 @@ def register_opts(ignore_errors=False):
         cfg.IntOpt("port", default=27017, help="port of db server"),
         cfg.StrOpt("db_name", default="st2", help="name of database"),
         cfg.StrOpt("username", help="username for db login"),
-        cfg.StrOpt("password", help="password for db login"),
+        cfg.StrOpt("password", help="password for db login", secret=True),
         cfg.IntOpt(
             "connection_timeout",
             default=3 * 1000,
@@ -206,35 +211,108 @@ def register_opts(ignore_errors=False):
             help="Backoff multiplier (seconds).",
         ),
         cfg.BoolOpt(
-            "ssl", default=False, help="Create the connection to mongodb using SSL"
+            "tls",
+            deprecated_name="ssl",
+            default=False,
+            help="Create the connection to mongodb using TLS.",
+        ),
+        cfg.StrOpt(
+            "tls_certificate_key_file",
+            default=None,
+            help=(
+                "Client certificate used to identify the local connection against MongoDB. "
+                "The certificate file must contain one or both of private key and certificate. "
+                "Supplying separate files for private key (ssl_keyfile) and certificate (ssl_certfile) "
+                "is no longer supported. "
+                "If encrypted, pass the password or passphrase in tls_certificate_key_file_password."
+            ),
+        ),
+        cfg.StrOpt(
+            "tls_certificate_key_file_password",
+            default=None,
+            help=(
+                "The password or passphrase to decrypt the file in tls_certificate_key_file. "
+                "Only set this if tls_certificate_key_file is encrypted."
+            ),
+            secret=True,
         ),
         cfg.StrOpt(
             "ssl_keyfile",
             default=None,
             help="Private keyfile used to identify the local connection against MongoDB.",
+            deprecated_for_removal=True,
+            deprecated_reason=(
+                "Use tls_certificate_key_file with a path to a file containing "
+                "the concatenation of the files from ssl_keyfile and ssl_certfile. "
+                "This option is ignored by pymongo."
+            ),
+            deprecated_since="3.9.0",
         ),
         cfg.StrOpt(
             "ssl_certfile",
             default=None,
             help="Certificate file used to identify the localconnection",
+            deprecated_for_removal=True,
+            deprecated_reason=(
+                "Use tls_certificate_key_file with a path to a file containing "
+                "the concatenation of the files from ssl_keyfile and ssl_certfile. "
+                "This option is ignored by pymongo. "
+            ),
+            deprecated_since="3.9.0",
+        ),
+        cfg.BoolOpt(
+            "tls_allow_invalid_certificates",
+            default=None,
+            sample_default=False,
+            help=(
+                "Specifies whether MongoDB is allowed to pass an invalid certificate. "
+                "This defaults to False to have security by default. "
+                "Only temporarily set to True if you need to debug the connection."
+            ),
         ),
         cfg.StrOpt(
             "ssl_cert_reqs",
             default=None,
             choices=["none", "optional", "required"],
-            help="Specifies whether a certificate is required from the other side of the "
-            "connection, and whether it will be validated if provided",
+            help=(
+                "Specifies whether a certificate is required from the other side of the "
+                "connection, and whether it will be validated if provided"
+            ),
+            deprecated_for_removal=True,
+            deprecated_reason=(
+                "Use tls_allow_invalid_certificates with the following: "
+                "The 'optional' and 'required' values are equivalent to tls_allow_invalid_certificates=False. "
+                "The 'none' value is equivalent to tls_allow_invalid_certificates=True. "
+                "This option is a needlessly more complex version of tls_allow_invalid_certificates."
+            ),
+            deprecated_since="3.9.0",
         ),
         cfg.StrOpt(
-            "ssl_ca_certs",
+            "tls_ca_file",
+            deprecated_name="ssl_ca_certs",
             default=None,
-            help="ca_certs file contains a set of concatenated CA certificates, which are "
-            "used to validate certificates passed from MongoDB.",
+            help=(
+                "ca_certs file contains a set of concatenated CA certificates, which are "
+                "used to validate certificates passed from MongoDB."
+            ),
+        ),
+        cfg.BoolOpt(
+            "tls_allow_invalid_hostnames",
+            default=None,
+            sample_default=False,
+            help=(
+                "If True and `tlsAllowInvalidCertificates` is True, disables hostname verification. "
+                "This defaults to False to have security by default. "
+                "Only temporarily set to True if you need to debug the connection."
+            ),
         ),
         cfg.BoolOpt(
             "ssl_match_hostname",
             default=True,
             help="If True and `ssl_cert_reqs` is not None, enables hostname verification",
+            deprecated_for_removal=True,
+            deprecated_reason="Use tls_allow_invalid_hostnames with the opposite value from this option.",
+            deprecated_since="3.9.0",
         ),
         cfg.StrOpt(
             "authentication_mechanism",
@@ -460,11 +538,13 @@ def register_opts(ignore_errors=False):
         cfg.StrOpt(
             "python_binary",
             default=default_python_bin_path,
+            sample_default="/usr/bin/python3",
             help="Python binary which will be used by Python actions.",
         ),
         cfg.StrOpt(
             "virtualenv_binary",
             default=default_virtualenv_bin_path,
+            sample_default="/usr/bin/virtualenv",
             help="Virtualenv binary which should be used to create pack virtualenvs.",
         ),
         cfg.StrOpt(

st2common/st2common/database_setup.py

@@ -36,13 +36,15 @@ def db_config():
         "db_port": cfg.CONF.database.port,
         "username": username,
         "password": password,
-        "ssl": cfg.CONF.database.ssl,
-        "ssl_keyfile": cfg.CONF.database.ssl_keyfile,
-        "ssl_certfile": cfg.CONF.database.ssl_certfile,
-        "ssl_cert_reqs": cfg.CONF.database.ssl_cert_reqs,
-        "ssl_ca_certs": cfg.CONF.database.ssl_ca_certs,
+        "tls": cfg.CONF.database.tls,
+        "tls_certificate_key_file": cfg.CONF.database.tls_certificate_key_file,
+        "tls_certificate_key_file_password": cfg.CONF.database.tls_certificate_key_file_password,
+        "tls_allow_invalid_certificates": cfg.CONF.database.tls_allow_invalid_certificates,
+        "tls_ca_file": cfg.CONF.database.tls_ca_file,
+        "tls_allow_invalid_hostnames": cfg.CONF.database.tls_allow_invalid_hostnames,
+        "ssl_cert_reqs": cfg.CONF.database.ssl_cert_reqs,  # deprecated
         "authentication_mechanism": cfg.CONF.database.authentication_mechanism,
-        "ssl_match_hostname": cfg.CONF.database.ssl_match_hostname,
+        "ssl_match_hostname": cfg.CONF.database.ssl_match_hostname,  # deprecated
     }