Merge pull request #626 from aperture-data/release-0.4.51

Release 0.4.51

Author: gsaluja9

.devcontainer/devcontainer.json

@@ -0,0 +1,28 @@
+{
+    "name": "devcontainer",
+    "dockerComposeFile": [
+        "docker-compose.yml"
+    ],
+    "service": "devcontainer",
+    "workspaceFolder": "/aperturedb-python",
+    "customizations": {
+        "vscode": {
+            "extensions": [
+                "ms-python.python",
+                "ms-python.pylint",
+                "ms-toolsai.jupyter"
+            ]
+        }
+    },
+    "settings": {
+        "python.defaultInterpreterPath": "/opt/venv/bin/python",
+        "python.linting.enabled": true,
+        "python.linting.pylintEnabled": true,
+        "files.exclude": {
+            "**/__pycache__": true,
+            "**/*.pyc": true
+        }
+    },
+    "initializeCommand": "./initcommand.sh",
+    "postCreateCommand": "./postinstall.sh"
+}

.devcontainer/docker-compose.yml

@@ -0,0 +1,101 @@
+name: $RUNNER_NAME
+
+services:
+  ca:
+    image: nginx
+    restart: on-failure
+    command: |
+      bash -c "
+      openssl genpkey -algorithm RSA -out /ca/ca.key -aes256 -pass pass:1234
+      openssl req -x509 -new -nodes -key /ca/ca.key -sha256 -days 3650 -out /ca/ca.crt -subj \"/C=US/ST=CA/L=Los Gatos/O=ApertureData/OU=ApertureDataCA/CN=ApertureDataCA\" -passin pass:1234
+      openssl genrsa -out /cert/tls.key 4096
+      openssl req -new -key /cert/tls.key -out /ca/tcp.csr -days 3650 -subj \"/C=US/ST=NY/L=NYC/O=instance/OU=instanceDB/CN=${DB_TCP_CN:-localhost}\"
+      openssl x509 -req  -CA /ca/ca.crt -CAkey /ca/ca.key -in /ca/tcp.csr -out /cert/tcp.crt -passin pass:1234
+      openssl req -new -key /cert/tls.key -out /ca/http.csr -days 3650 -subj \"/C=US/ST=NY/L=NYC/O=instance/OU=instanceDB/CN=${DB_HTTP_CN:-localhost}\"
+      openssl x509 -req  -CA /ca/ca.crt -CAkey /ca/ca.key -in /ca/http.csr -out /cert/http.crt -passin pass:1234"
+    volumes:
+      - ./aperturedb/certificate:/cert
+      - ./ca:/ca
+
+  lenz:
+    depends_on:
+      ca:
+        condition: service_completed_successfully
+      aperturedb:
+        condition: service_started
+    image: $LENZ_REPO:$LENZ_TAG
+    ports:
+      - $GATEWAY:55556:55551
+    restart: always
+    environment:
+      LNZ_HEALTH_PORT: 58085
+      LNZ_TCP_PORT: 55551
+      LNZ_HTTP_PORT: 8080
+      LNZ_ADB_BACKENDS: '["aperturedb:55553"]'
+      LNZ_REPLICAS: 1
+      LNZ_ADB_MAX_CONCURRENCY: 48
+      LNZ_FORCE_SSL: false
+      LNZ_CERTIFICATE_PATH: /etc/lenz/certificate/tcp.crt
+      LNZ_PRIVATE_KEY_PATH: /etc/lenz/certificate/tls.key
+    volumes:
+      - ./aperturedb/certificate:/etc/lenz/certificate
+
+  aperturedb:
+    image: $ADB_REPO:$ADB_TAG
+    volumes:
+      - ./aperturedb/db_$RUNNER_NAME:/aperturedb/db
+      - ./aperturedb/logs:/aperturedb/logs
+    restart: always
+    environment:
+      ADB_KVGD_DB_SIZE: "204800"
+      ADB_LOG_PATH: "logs"
+      ADB_ENABLE_DEBUG: 1
+      ADB_MASTER_KEY: "admin"
+      ADB_PORT: 55553
+      ADB_FORCE_SSL: false
+
+  webui:
+    image: aperturedata/aperturedata-platform-web-private:latest
+    restart: always
+
+  nginx:
+    depends_on:
+      ca:
+        condition: service_completed_successfully
+    image: nginx
+    restart: always
+    ports:
+      - $GATEWAY:8087:80
+      - $GATEWAY:8443:443
+    configs:
+      - source: nginx.conf
+        target: /etc/nginx/conf.d/default.conf
+    volumes:
+      - ./aperturedb/certificate:/etc/nginx/certificate
+  devcontainer:
+    build:
+      context: ../docker/devcontainer
+    environment:
+      DB_HOST: lenz
+      DB_PORT: 55551
+      CA_CERT: /ca/ca.crt
+    command: bash -c "while true; do sleep 1000; done"
+    volumes:
+      - ../:/aperturedb-python
+      - ./ca:/ca
+configs:
+  nginx.conf:
+    content: |
+      server {
+        listen 80;
+        listen 443 ssl;
+        client_max_body_size 256m;
+        ssl_certificate /etc/nginx/certificate/http.crt;
+        ssl_certificate_key /etc/nginx/certificate/tls.key;
+        location / {
+          proxy_pass http://webui;
+        }
+        location /api/ {
+          proxy_pass http://lenz:8080;
+        }
+      }

.gitignore

@@ -154,3 +154,6 @@ docker/pytorch-gpu/aperturedata/*
 .aperturedb
 test/data/
 test/aperturedb/certificate/
+.devcontainer/aperturedb/
+.devcontainer/ca/
+test/*_ca/

aperturedb/CommonLibrary.py

@@ -40,6 +40,7 @@ def __create_connector(configuration: Configuration):
             token=configuration.token,
             password=configuration.password,
             use_ssl=configuration.use_ssl,
+            ca_cert=configuration.ca_cert,
             config=configuration)
     else:
         connector = Connector(
@@ -49,6 +50,7 @@ def __create_connector(configuration: Configuration):
             token=configuration.token,
             password=configuration.password,
             use_ssl=configuration.use_ssl,
+            ca_cert=configuration.ca_cert,
             config=configuration)
     logger.debug(
         f"Created connector using: {configuration}. Will connect on query.")
@@ -147,6 +149,7 @@ def _store_config(config: Configuration, name: str):
         password=config.password,
         use_ssl=config.use_ssl,
         use_rest=config.use_rest,
+        ca_cert=config.ca_cert,
         interactive=False,
         overwrite=True,
         active=True,

aperturedb/Configuration.py

@@ -5,7 +5,8 @@
 from base64 import b64encode, b64decode
 
 APERTUREDB_CLOUD = ".cloud.aperturedata.io"
-APERTUREDB_KEY_VERSION = 1
+APERTUREDB_KEY_VERSION = 2
+APERTUREDB_OLD_KEY_VERSION = 1
 FLAG_USE_COMPRESSED_HOST = 4
 FLAG_USE_REST = 2
 FLAG_USE_SSL = 1
@@ -33,6 +34,7 @@ class Configuration:
     retry_max_attempts: int = 3
     token: str = None
     user_keys: dict = None
+    ca_cert: str = None
 
     def __repr__(self) -> str:
         mode = "REST" if self.use_rest else "TCP"
@@ -41,7 +43,7 @@ def __repr__(self) -> str:
 
     def deflate(self) -> list:
         return self.create_aperturedb_key(self.host, self.port, self.token,
-                                          self.use_rest, self.use_ssl, self.username, self.password)
+                                          self.use_rest, self.use_ssl, self.ca_cert, self.username, self.password)
 
     def has_user_keys(self) -> bool:
         return self.user_keys is not None
@@ -82,8 +84,10 @@ def config_default_port(cls, use_rest: bool, use_ssl: bool):
             return DEFAULT_TCP_PORT
 
     @classmethod
-    def create_aperturedb_key(cls, host: str, port: int,  token_string: str,
-                              use_rest: bool, use_ssl: bool, username: str = None, password: str = None) -> None:
+    def create_aperturedb_key(
+            cls, host: str, port: int,  token_string: str,
+            use_rest: bool, use_ssl: bool, ca_cert: str = None,
+            username: str = None, password: str = None) -> None:
         compressed = False
         if token_string is not None and token_string.startswith("adbp_"):
             token_string = token_string[5:]
@@ -100,10 +104,11 @@ def create_aperturedb_key(cls, host: str, port: int,  token_string: str,
         if port != default_port:
             host = f"{host}:{port}"
         if token_string is not None:
-            key_json = [APERTUREDB_KEY_VERSION, key_type, host, token_string]
+            key_json = [APERTUREDB_KEY_VERSION,
+                        key_type, host, ca_cert, token_string]
         else:
             key_json = [APERTUREDB_KEY_VERSION,
-                        key_type, host, username, password]
+                        key_type, host, ca_cert, username, password]
         simplified = json.dumps(key_json, separators=(',', ':'))
         encoded = b64encode(simplified.encode('utf-8')).decode('utf-8')
         return encoded
@@ -118,16 +123,25 @@ def reinflate(cls, encoded_str: list) -> object:
             raise Exception(
                 "Unable to make configuration from the provided string")
         version = as_list[0]
-        if version not in (APERTUREDB_KEY_VERSION,):
+        if version not in (APERTUREDB_KEY_VERSION, APERTUREDB_OLD_KEY_VERSION):
             raise ValueError("version identifier of configuration was"
                              f"{version}, which is not supported")
         is_compressed, use_rest, use_ssl = cls.key_type_to_config(as_list[1])
         host = as_list[2]
+        pem = None
+
+        if version == APERTUREDB_KEY_VERSION:
+            pem = as_list[3]
+            list_offset = 4
+        else:
+            list_offset = 3
         token = username = password = None
-        if len(as_list) == 4:
-            token = "adbp_" + as_list[3]
-        elif len(as_list) == 5:
-            username, password = as_list[3:]
+        if len(as_list) == list_offset + 1:
+            token = "adbp_" + as_list[list_offset]
+        elif len(as_list) == list_offset + 2:
+            username, password = as_list[list_offset:]
+        else:
+            raise ValueError("Bad format for key list")
 
         port_match = re.match(".*:(\d+)$", host)
         if port_match is not None:
@@ -146,7 +160,7 @@ def reinflate(cls, encoded_str: list) -> object:
                     f"Unable to parse compressed host: {host} Error: {e}")
 
         c = Configuration(
-            host, port, username, password, name, use_ssl, use_rest)
+            host, port, username, password, name, use_ssl, use_rest, ca_cert=pem)
         if token:
             c.token = token
         return c