Merge pull request #630 from aperture-data/release-0.4.52

Release 0.4.52

Author: gsaluja9

aperturedb/CommonLibrary.py

@@ -41,6 +41,7 @@ def __create_connector(configuration: Configuration):
             password=configuration.password,
             use_ssl=configuration.use_ssl,
             ca_cert=configuration.ca_cert,
+            verify_hostname=configuration.verify_hostname,
             config=configuration)
     else:
         connector = Connector(
@@ -51,6 +52,7 @@ def __create_connector(configuration: Configuration):
             password=configuration.password,
             use_ssl=configuration.use_ssl,
             ca_cert=configuration.ca_cert,
+            verify_hostname=configuration.verify_hostname,
             config=configuration)
     logger.debug(
         f"Created connector using: {configuration}. Will connect on query.")

aperturedb/Configuration.py

@@ -7,6 +7,7 @@
 APERTUREDB_CLOUD = ".cloud.aperturedata.io"
 APERTUREDB_KEY_VERSION = 2
 APERTUREDB_OLD_KEY_VERSION = 1
+FLAG_VERIFY_HOSTNAME = 8
 FLAG_USE_COMPRESSED_HOST = 4
 FLAG_USE_REST = 2
 FLAG_USE_SSL = 1
@@ -35,15 +36,29 @@ class Configuration:
     token: str = None
     user_keys: dict = None
     ca_cert: str = None
+    verify_hostname: bool = True
+
+    def __ssl_mode(self) -> str:
+        if not self.use_ssl:
+            mode = "SSL_OFF"  # with not use_ssl, we don't use SSL
+        elif not self.verify_hostname:
+            mode = "SSL_NO_VERIFY"  # with verify_hostname=False, we don't verify the hostname
+        elif self.ca_cert:
+            # with verify_hostname=True and ca_cert is provided, we verify the hostname using the provided ca_cert
+            mode = "SSL_WITH_CA"
+        else:
+            mode = "SSL_DEFAULT"
+
+        return mode
 
     def __repr__(self) -> str:
         mode = "REST" if self.use_rest else "TCP"
         auth_mode = "token" if self.token is not None else "password"
-        return f"[{self.host}:{self.port} as {self.username} using {mode} with SSL={self.use_ssl} auth={auth_mode}]"
+        return f"[{self.host}:{self.port} as {self.username} using {mode} with SSL={self.__ssl_mode()} auth={auth_mode}]"
 
     def deflate(self) -> list:
         return self.create_aperturedb_key(self.host, self.port, self.token,
-                                          self.use_rest, self.use_ssl, self.ca_cert, self.username, self.password)
+                                          self.use_rest, self.use_ssl, self.ca_cert, self.username, self.password, self.verify_hostname)
 
     def has_user_keys(self) -> bool:
         return self.user_keys is not None
@@ -65,16 +80,20 @@ def set_user_keys(self, keys: dict) -> None:
         self.user_keys = keys
 
     @classmethod
-    def config_to_key_type(cls, compressed_host: bool,  use_rest: bool, use_ssl: bool):
-        return (FLAG_USE_COMPRESSED_HOST if compressed_host else 0) + \
-               (FLAG_USE_REST if use_rest else 0) + \
-               (FLAG_USE_SSL if use_ssl else 0)
+    def config_to_key_type(cls, compressed_host: bool,  use_rest: bool, use_ssl: bool, verify_hostname: bool):
+        return (
+            (FLAG_USE_COMPRESSED_HOST if compressed_host else 0) |
+            (FLAG_USE_REST if use_rest else 0) |
+            (FLAG_USE_SSL if use_ssl else 0) |
+            (FLAG_VERIFY_HOSTNAME if verify_hostname else 0)
+        )
 
     @classmethod
     def key_type_to_config(cls, key_type: int): \
         return [bool(key_type & FLAG_USE_COMPRESSED_HOST),
                 bool(key_type & FLAG_USE_REST),
-                bool(key_type & FLAG_USE_SSL)]
+                bool(key_type & FLAG_USE_SSL),
+                bool(key_type & FLAG_VERIFY_HOSTNAME)]
 
     @classmethod
     def config_default_port(cls, use_rest: bool, use_ssl: bool):
@@ -87,7 +106,7 @@ def config_default_port(cls, use_rest: bool, use_ssl: bool):
     def create_aperturedb_key(
             cls, host: str, port: int,  token_string: str,
             use_rest: bool, use_ssl: bool, ca_cert: str = None,
-            username: str = None, password: str = None) -> None:
+            username: str = None, password: str = None, verify_hostname: bool = True) -> None:
         compressed = False
         if token_string is not None and token_string.startswith("adbp_"):
             token_string = token_string[5:]
@@ -99,7 +118,8 @@ def create_aperturedb_key(
                 host = "{}.{}".format(m.group(1), int(m.group(2)))
                 compressed = True
 
-        key_type = cls.config_to_key_type(compressed, use_rest, use_ssl)
+        key_type = cls.config_to_key_type(
+            compressed, use_rest, use_ssl, verify_hostname)
         default_port = cls.config_default_port(use_rest, use_ssl)
         if port != default_port:
             host = f"{host}:{port}"
@@ -126,10 +146,14 @@ def reinflate(cls, encoded_str: list) -> object:
         if version not in (APERTUREDB_KEY_VERSION, APERTUREDB_OLD_KEY_VERSION):
             raise ValueError("version identifier of configuration was"
                              f"{version}, which is not supported")
-        is_compressed, use_rest, use_ssl = cls.key_type_to_config(as_list[1])
+        is_compressed, use_rest, use_ssl, verify_hostname = cls.key_type_to_config(
+            as_list[1])
         host = as_list[2]
         pem = None
 
+        if version == APERTUREDB_OLD_KEY_VERSION:
+            verify_hostname = True
+
         if version == APERTUREDB_KEY_VERSION:
             pem = as_list[3]
             list_offset = 4
@@ -160,7 +184,7 @@ def reinflate(cls, encoded_str: list) -> object:
                     f"Unable to parse compressed host: {host} Error: {e}")
 
         c = Configuration(
-            host, port, username, password, name, use_ssl, use_rest, ca_cert=pem)
+            host, port, username, password, name, use_ssl, use_rest, ca_cert=pem, verify_hostname=verify_hostname)
         if token:
             c.token = token
         return c

aperturedb/Connector.py

@@ -149,6 +149,7 @@ def __init__(self, host="localhost", port=DEFAULT_PORT,
                  shared_data=None,
                  authenticate=True,
                  use_keepalive=True,
+                 verify_hostname=True,
                  retry_interval_seconds=DEFAULT_RETRY_INTERVAL_SECONDS,
                  retry_max_attempts=DEFAULT_RETRY_MAX_ATTEMPTS,
                  config: Optional[Configuration] = None,
@@ -179,6 +180,7 @@ def __init__(self, host="localhost", port=DEFAULT_PORT,
                 port=port,
                 use_ssl=use_ssl,
                 ca_cert=ca_cert,
+                verify_hostname=verify_hostname,
                 username=user,
                 password=password,
                 name="runtime",
@@ -334,6 +336,26 @@ def _refresh_token(self):
         else:
             raise UnauthorizedException(response)
 
+    def _build_ssl_context(self):
+        """
+        Builds an SSL context for the connection.
+        There are 3 scenarios:
+        1. verify_hostname is False, we don't verify the hostname
+        2. verify_hostname is True and ca_cert is provided, we verify the hostname using the provided ca_cert
+        3. verify_hostname is True and ca_cert is not provided, we verify the hostname using the system's default CA certificates
+        """
+        self.context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+        if not self.config.verify_hostname:
+            self.context.check_hostname = False
+            self.context.verify_mode = ssl.CERT_NONE
+        elif self.config.ca_cert:
+            self.context.load_verify_locations(
+                cafile=self.config.ca_cert
+            )
+        else:
+            self.context.load_default_certs(ssl.Purpose.SERVER_AUTH)
+        return self.context
+
     def _connect(self):
         self.conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         self.conn.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
@@ -375,35 +397,28 @@ def _connect(self):
 
             if self.use_ssl:
 
-                # Server is ok with SSL, we switch over SSL.
-                self.context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
-                self.context.verify_mode = ssl.CERT_REQUIRED
-                self.context.check_hostname = True
-                if self.config.ca_cert:
-                    self.context.load_verify_locations(
-                        cafile=self.config.ca_cert
-                    )
-                else:
-                    self.context.load_default_certs(ssl.Purpose.SERVER_AUTH)
+                self.context = self._build_ssl_context()
 
                 # TODO, we need to add support for local certificates
                 # For now, we let the server send us the certificate
                 try:
-                    self.conn = self.context.wrap_socket(
-                        self.conn, server_hostname=self.host)
+                    if self.config.verify_hostname:
+                        self.conn = self.context.wrap_socket(
+                            self.conn, server_hostname=self.host)
+                    else:
+                        self.conn = self.context.wrap_socket(
+                            self.conn)
                 except ssl.SSLCertVerificationError as e:
-                    logger.exception(
-                        f"The host name must match the certificate: {self.host}")
                     logger.exception(
                         f"You can use the ca_cert parameter to specify a custom CA certificate")
                     assert False, "Certificate verification failed" + os.linesep + \
                         f"The host name must match the certificate: {self.host} " + os.linesep + \
                         f"You can use the ca_cert parameter to specify a custom CA certificate " + os.linesep + \
                         f"Refer to the documentation for more information: {SETUP_URL}" + os.linesep + \
-                        f"Alternatively, SSL can be disabled by setting use_ssl=False (not recommended)" + os.linesep + \
-                        f"{e=}"
+                        f"Alternatively, SSL can be disabled by setting verify_hostname=False or use_ssl=False (not recommended)" + \
+                        os.linesep
                 except ssl.SSLError as e:
-                    logger.error(f"Error wrapping socket: {e}")
+                    logger.exception(f"Error wrapping socket.")
                     self.conn.close()
                     self.connected = False
                     raise
@@ -417,8 +432,8 @@ def _connect(self):
                 f"The ca certificate file does not exist: {self.config.ca_cert} " + os.linesep + \
                 f"You can use the ca_cert parameter to specify a custom CA certificate " + os.linesep + \
                 f"Refer to the documentation for more information: {SETUP_URL} " + os.linesep + \
-                f"Alternatively, SSL can be disabled by setting use_ssl=False (not recommended)" + os.linesep + \
-                f"{e=}"
+                f"Alternatively, SSL can be disabled by setting verify_hostname=False or use_ssl=False (not recommended)" + \
+                os.linesep
         except BaseException as e:
             self.conn.close()
             self.connected = False

aperturedb/ConnectorRest.py

@@ -81,7 +81,7 @@ class ConnectorRest(Connector):
 
     def __init__(self, host="localhost", port=None,
                  user="", password="", token="",
-                 use_ssl=True, ca_cert=None, shared_data=None,
+                 use_ssl=True, ca_cert=None, verify_hostname=True, shared_data=None,
                  config: Optional[Configuration] = None,
                  key: Optional[str] = None):
         self.use_keepalive = False
@@ -93,6 +93,7 @@ def __init__(self, host="localhost", port=None,
             token=token,
             use_ssl=use_ssl,
             ca_cert=ca_cert,
+            verify_hostname=verify_hostname,
             shared_data=shared_data,
             config=config,
             key=key)
@@ -112,11 +113,12 @@ def __init__(self, host="localhost", port=None,
         # Since we will be making same call to the same URL, making a session
         # REF: https://requests.readthedocs.io/en/latest/user/advanced/
         self.http_session = requests.Session()
-        if self.config.ca_cert:
-            adapter = CustomHTTPAdapter(ca_cert=self.config.ca_cert)
-        else:
-            adapter = CustomHTTPAdapter(ca_cert=None)
-        self.http_session.mount('https://', adapter=adapter)
+        if self.config.verify_hostname:
+            if self.config.ca_cert:
+                adapter = CustomHTTPAdapter(ca_cert=self.config.ca_cert)
+            else:
+                adapter = CustomHTTPAdapter(ca_cert=None)
+            self.http_session.mount('https://', adapter=adapter)
 
         self.last_response   = ''
         self.last_query_time = 0
@@ -155,10 +157,11 @@ def _query(self, query, blob_array = [], try_resume=True):
         while tries < self.config.retry_max_attempts:
             tries += 1
             try:
+                # URL takes care of the scheme
                 response = self.http_session.post(self.url,
                                                   headers = headers,
                                                   files   = files,
-                                                  verify  = self.use_ssl)
+                                                  verify  = self.config.use_ssl and self.config.verify_hostname)
                 if response.status_code == 200:
                     # Parse response:
                     json_response       = json.loads(response.text)

aperturedb/__init__.py

@@ -10,7 +10,7 @@
 import signal
 import sys
 
-__version__ = "0.4.51"
+__version__ = "0.4.52"
 
 logger = logging.getLogger(__name__)
 

aperturedb/cli/configure.py

@@ -77,7 +77,8 @@ def get_configurations(file: str):
                 token=config["token"] if "token" in config else None,
                 use_rest=config["use_rest"],
                 use_ssl=config["use_ssl"],
-                ca_cert=config.get("ca_cert", None))
+                ca_cert=config.get("ca_cert", None),
+                verify_hostname=config.get("verify_hostname", True))
             if "user_keys" in config:
                 configs[c].set_user_keys(config["user_keys"])
     active = configurations["active"]
@@ -147,7 +148,10 @@ def create(
         password: Annotated[str, typer.Option(help="Password")] = "admin",
         use_rest: Annotated[bool, typer.Option(help="Use REST")] = False,
         use_ssl: Annotated[bool, typer.Option(help="Use SSL")] = True,
-        ca_cert: Annotated[str, typer.Option(help="CA certificate")] = None,
+        ca_cert: Annotated[Optional[str],
+                           typer.Option(help="CA certificate")] = "",
+        verify_hostname: Annotated[bool, typer.Option(
+            help="Verify hostname")] = True,
         interactive: Annotated[bool, typer.Option(
             help="Interactive mode")] = True,
         overwrite: Annotated[bool, typer.Option(
@@ -179,6 +183,7 @@ def check_for_overwrite(name):
     db_use_rest = use_rest
     db_use_ssl = use_ssl
     db_ca_cert = ca_cert
+    db_verify_hostname = verify_hostname
     config_path = _config_file_path(as_global)
     configs = {}
     try:
@@ -228,10 +233,15 @@ def check_for_overwrite(name):
             db_use_ssl = typer.confirm(
                 f"Use SSL [Note: ApertureDB's defaults do not allow non SSL traffic]", default=db_use_ssl)
             db_ca_cert = typer.prompt(
-                f"Enter {APP_NAME} CA certificate's (if custom CA is used)", default=db_ca_cert)
-            db_ca_cert = os.path.abspath(db_ca_cert)
-            assert os.path.exists(
-                db_ca_cert), "CA certificate file does not exist"
+                f"Enter {APP_NAME} CA certificate's path (if custom CA is used)", default=db_ca_cert)
+            if db_ca_cert != "":
+                db_ca_cert = os.path.abspath(db_ca_cert)
+                assert os.path.exists(
+                    db_ca_cert), f"CA certificate file {db_ca_cert} does not exist"
+            else:
+                db_ca_cert = None
+            db_verify_hostname = typer.confirm(
+                f"Verify hostname", default=db_verify_hostname)
 
         gen_config = Configuration(
             name=name,
@@ -241,7 +251,8 @@ def check_for_overwrite(name):
             password=db_password,
             use_ssl=db_use_ssl,
             use_rest=db_use_rest,
-            ca_cert=db_ca_cert
+            ca_cert=db_ca_cert,
+            verify_hostname=db_verify_hostname
         )
 
     assert name is not None, "Configuration name must be specified"

aperturedb/cli/keys.py

@@ -23,6 +23,11 @@ def generate_user_key(conn: Connector, user: str):
     token = u.generate_token()
     u.assign_token(user, token)
     key = Configuration.create_aperturedb_key(
-        conn.config.host, conn.config.port, token, conn.config.use_rest,
-        conn.config.use_ssl, conn.config.ca_cert)
+        host=conn.config.host,
+        port=conn.config.port,
+        token_string=token,
+        use_rest=conn.config.use_rest,
+        use_ssl=conn.config.use_ssl,
+        ca_cert=conn.config.ca_cert,
+        verify_hostname=conn.config.verify_hostname)
     return key

test/test_Key.py

@@ -36,7 +36,8 @@ def test_encode_keys(self):
             config_type = data[1]
             host = data[2]
             username = password = token = None
-            comp, rest, ssl = Configuration.key_type_to_config(config_type)
+            comp, rest, ssl, verify_hostname = Configuration.key_type_to_config(
+                config_type)
             if host.rfind(':') != -1:
                 port = int(host.split(':')[1])
                 host = host.split(':')[0]
@@ -48,7 +49,7 @@ def test_encode_keys(self):
                 username = data[3]
                 password = data[4]
             c = Configuration(host, port, username, password,
-                              "encoding test", use_rest=rest, use_ssl=ssl, token=token)
+                              "encoding test", use_rest=rest, use_ssl=ssl, token=token, verify_hostname=verify_hostname)
             deflated = c.deflate()
             assert deflated == key