Move destination config into config file (#14)

* Move destination into config file

* Update config and docs

* Update tests

* Bump 0.6.0

Author: lloesche

README.md

@@ -20,35 +20,60 @@ pip install cloud2sql
 
 ## Usage
 
-The sources for `cloud2sql` are configured via a configuration file. Create your own configuration by adjusting the [config template file](./config-template.yaml).
+The sources and destinations for `cloud2sql` are configured via a configuration file. Create your own configuration by adjusting the [config template file](./config-template.yaml).
 You can safely delete the sections that are not relevant to you (e.g. if you do not use AWS, you can delete the `aws` section).
 All sections refer to cloud providers and are enabled if a configuration section is provided.
 
-After the sources are configured, you need to define the url to the database.
 The following databases are currently supported:
 
 #### SQLite
 
 ```
-sqlite:///path/to/resoto.db
+destinations:
+    sqlite:
+        database: /path/to/database.db
 ```
 
 #### PostgreSQL
 
 ```
-postgresql://user:password@host:port/dbname[?key=value&key=value...]
+destinations:
+    postgresql:
+        host: 127.0.0.1
+        port: 5432
+        user: cloud2sql
+        password: changeme
+        database: cloud2sql
+        args:
+            key: value
 ```
 
 #### MySQL
 
 ```
-mysql://username:password@host/dbname[?key=value&key=value...]
+destinations:
+    mysql:
+        host: 127.0.0.1
+        port: 3306
+        user: cloud2sql
+        password: changeme
+        database: cloud2sql
+        args:
+            key: value
 ```
 
 #### MariaDB
 
 ```
-mariadb://username:password@host/dbname[?key=value&key=value...]
+destinations:
+    mariadb:
+        host: 127.0.0.1
+        port: 3306
+        user: cloud2sql
+        password: changeme
+        database: cloud2sql
+        args:
+            key: value
 ```
 
 #### My database is not listed here
@@ -62,7 +87,7 @@ We use a minimal configuration [example](./config-example.yaml) and export the d
 The example uses our AWS default credentials and the default kubernetes config.
 
 ```bash
-cloud2sql --config config-example.yaml --db "sqlite:///resoto.db"
+cloud2sql --config config-example.yaml
 ```
 
 ## Local Development
@@ -73,4 +98,3 @@ Create a local development environment with the following command:
 make setup
 source venv/bin/activate
 ```
-

cloud2sql/__init__.py

@@ -11,4 +11,4 @@
 __author__ = "Some Engineering Inc."
 __license__ = "Apache 2.0"
 __copyright__ = "Copyright © 2022 Some Engineering Inc."
-__version__ = "0.5.0"
+__version__ = "0.6.0"

cloud2sql/__main__.py

@@ -7,7 +7,8 @@
 from sqlalchemy.engine import Engine
 
 from cloud2sql.analytics import PosthogEventSender, NoEventSender, AnalyticsEventSender
-from cloud2sql.collect_plugins import collect_from_plugins
+from cloud2sql.collect_plugins import collect_from_plugins, configure
+from cloud2sql.util import db_string_from_config
 
 # Will fail in case snowflake is not installed - which is fine.
 try:
@@ -32,11 +33,6 @@ def parse_args() -> Namespace:
         default="progress",
         help="Output to show during the process. Default: progress",
     )
-    parser.add_argument(
-        "--db",
-        help="The database url. See https://docs.sqlalchemy.org/en/20/core/engines.html.",
-        required=True,
-    )
     parser.add_argument(
         "--analytics-opt-out",
         default=False,
@@ -62,7 +58,8 @@ def main() -> None:
     try:
         setup_logger("resoto.cloud2sql", level=args.log_level, force=True)
         sender = NoEventSender() if args.analytics_opt_out else PosthogEventSender()
-        engine = create_engine(args.db)
+        config = configure(args.config)
+        engine = create_engine(db_string_from_config(config))
         collect(engine, args, sender)
     except Exception as e:
         if args.debug:  # raise exception and show complete tracelog

cloud2sql/collect_plugins.py

@@ -51,10 +51,17 @@ def collectors(raw_config: Json, feedback: CoreFeedback) -> Dict[str, BaseCollec
 
 
 def configure(path_to_config: Optional[str]) -> Json:
+    config = {}
     if path_to_config:
         with open(path_to_config) as f:
-            return yaml.safe_load(f)  # type: ignore
-    return {}
+            config = yaml.safe_load(f)  # type: ignore
+
+    if "sources" not in config:
+        raise ValueError("No sources configured")
+    if "destinations" not in config:
+        raise ValueError("No destinations configured")
+
+    return config
 
 
 def collect(
@@ -137,7 +144,8 @@ def collect_from_plugins(engine: Engine, args: Namespace, sender: AnalyticsEvent
     core_messages: Queue[Json] = mp_manager.Queue()
     feedback = CoreFeedback("cloud2sql", "collect", "collect", core_messages)
     raw_config = configure(args.config)
-    all_collectors = collectors(raw_config, feedback)
+    sources = raw_config["sources"]
+    all_collectors = collectors(sources, feedback)
     analytics = {"total": len(all_collectors), "engine": engine.dialect.name} | {name: 1 for name in all_collectors}
     end = Event()
     with ThreadPoolExecutor(max_workers=4) as executor:

cloud2sql/util.py

@@ -1,6 +1,6 @@
 from typing import Union, List, Optional, Any
 
-from resotolib.types import JsonElement
+from resotolib.types import JsonElement, Json
 
 
 def value_in_path(element: JsonElement, path_or_name: Union[List[str], str]) -> Optional[Any]:
@@ -16,3 +16,40 @@ def at_idx(current: JsonElement, idx: int) -> Optional[Any]:
             return at_idx(current[path[idx]], idx + 1)
 
     return at_idx(element, 0)
+
+
+def db_string_from_config(config: Json) -> str:
+    destinations = config.get("destinations", {})
+
+    if len(destinations) != 1:
+        raise ValueError("Exactly one destination must be configured")
+
+    db_type = list(destinations.keys())[0]
+    db_config = destinations[db_type]
+    user = db_config.get("user")
+    password = db_config.get("password")
+    host = db_config.get("host")
+    port = db_config.get("port")
+    database = db_config.get("database")
+    args = db_config.get("args", {})
+
+    db_uri = f"{db_type}://"
+
+    if user:
+        db_uri += user
+        if password:
+            db_uri += f":{password}"
+        db_uri += "@"
+
+    if host:
+        db_uri += host
+        if port:
+            db_uri += f":{port}"
+
+    if database:
+        db_uri += f"/{database}"
+
+    if len(args) > 0:
+        db_uri += "?" + "&".join([f"{k}={v}" for k, v in args.items()])
+
+    return db_uri

config-example.yaml

@@ -1,9 +1,13 @@
-aws:
-  # Take the aws default profile in ~/.aws/credentials and collect 2 regions
-  region: ["us-east-1", "eu-central-1"]
+sources:
+  aws:
+    # Take the aws default profile in ~/.aws/credentials and collect 2 regions
+    region: ["us-east-1", "eu-central-1"]
 
-k8s:
-  # Use the default kubeconfig file and collect all contexts.
-  config_files:
-    - path: "~/.kube/config"
-      all_contexts: true
+  k8s:
+    # Use the default kubeconfig file and collect all contexts.
+    config_files:
+      - path: "~/.kube/config"
+        all_contexts: true
+destinations:
+  sqlite:
+    database: resoto.db