feat: Add Readme and configuration (#3)

* feat: Add Readme and configuration

* improve readme

Author: aquamatthias

Makefile

@@ -76,14 +76,6 @@ coverage: ## check code coverage quickly with the default Python
 venv:
 	python3 -m venv venv --prompt "cloud2sql"
 	. ./venv/bin/activate && python3 -m pip install --upgrade pip
-	# region as long as version 3 is not available on pypi
-	. ./venv/bin/activate && pip install -e ../resoto/resotolib
-	. ./venv/bin/activate && pip install -e ../resoto/plugins/aws
-	. ./venv/bin/activate && pip install -e ../resoto/plugins/digitalocean
-	. ./venv/bin/activate && pip install -e ../resoto/plugins/gcp
-	. ./venv/bin/activate && pip install -e ../resoto/plugins/k8s
-	. ./venv/bin/activate && pip install -e ../resoto/plugins/example_collector
-	# end region
 	. ./venv/bin/activate && pip install -r requirements-test.txt
 	. ./venv/bin/activate && pip install -r requirements.txt
 	. ./venv/bin/activate && pip install -e .

README.md

@@ -1,2 +1,76 @@
 # cloud2sql
-Read infrastructure data from your cloud and export it to an SQL database.
+Read infrastructure data from your cloud and export it to a SQL database.
+
+
+## Installation
+
+Python 3.10 or higher is required.
+It is recommended to use a separate virtual environment for this project. You can create one with the following command:
+
+```bash
+python3 -m venv venv --prompt "cloud2sql"
+source venv/bin/activate
+```
+
+Once the virtual environment is activated, you can install cloud2sql:
+
+```bash
+pip install cloud2sql
+```
+
+## Usage
+
+The sources for `cloud2sql` are configured via a configuration file. Create your own configuration by adjusting the [config template file](./config-template.yaml).
+You can safely delete the sections that are not relevant to you (e.g. if you do not use AWS, you can delete the `aws` section).
+All sections refer to cloud providers and are enabled if a configuration section is provided.
+
+After the sources are configured, you need to define the url to the database.
+The following databases are currently supported:
+
+#### SQLite
+
+```
+sqlite:///path/to/resoto.db
+```
+
+#### PostgreSQL
+
+```
+postgresql://user:password@host:port/dbname[?key=value&key=value...]
+```
+
+#### MySQL
+
+```
+mysql://username:password@host/dbname[?key=value&key=value...]
+```
+
+#### MariaDB
+
+```
+mariadb://username:password@host/dbname[?key=value&key=value...]
+```
+
+#### My database is not listed here
+
+cloud2sql uses SQLAlchemy to connect to the database. If your database is not listed here, you can check if it is supported in [SQLAlchemy Dialects](https://docs.sqlalchemy.org/en/20/dialects/index.html).
+Install the relevant driver and use the connection string from the documentation.
+
+#### Example
+
+We use a minimal configuration [example](./config-example.yaml) and export the data to a SQLite database.
+The example uses our AWS default credentials and the default kubernetes config.
+
+```bash
+cloud2sql --config config-example.yaml --db "sqlite:///resoto.db"
+```
+
+## Local Development
+
+Create a local development environment with the following command:
+
+```bash
+make setup
+source venv/bin/activate
+```
+

cloud2sql/__init__.py

@@ -1,13 +1,14 @@
 """
 Cloud2Sql
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
-Read infrastructure data from your cloud and export it to an SQL database.
+Read infrastructure data from your cloud and export it to a SQL database.
 :copyright: © 2022 Some Engineering Inc.
 :license: Apache 2.0, see LICENSE for more details.
 """
 
 __title__ = "cloud2sql"
-__description__ = "Read infrastructure data from your cloud and export it to an SQL database."
+__description__ = "Read infrastructure data from your cloud and export it to a SQL database."
 __author__ = "Some Engineering Inc."
 __license__ = "Apache 2.0"
 __copyright__ = "Copyright © 2022 Some Engineering Inc."
+__version__ = "0.4.0"

config-example.yaml

@@ -0,0 +1,8 @@
+aws:
+  # Take the aws default profile in ~/.aws/credentials and collect 2 regions
+  region: ["us-east-1", "eu-central-1"]
+k8s:
+  # Use the default kubeconfig file and collect all contexts.
+  config_files:
+    - path: "~/.kube/config"
+      all_contexts: true

config-template.yaml

@@ -0,0 +1,97 @@
+aws:
+  # AWS Access Key ID (null to load from env - recommended)
+  access_key_id: null
+  # AWS Secret Access Key (null to load from env - recommended)
+  secret_access_key: null
+  # IAM role name to assume
+  role: null
+  # List of AWS profiles to collect
+  profiles: null
+  # List of AWS Account ID(s) to collect (null for all if scrape_org is true)
+  account: null
+  # List of AWS Regions to collect (null for all)
+  region: null
+  # Scrape the entire AWS organization
+  scrape_org: false
+  # Fork collector process instead of using threads
+  fork_process: true
+  # List of accounts to exclude when scraping the org
+  scrape_exclude_account: []
+  # Assume given role in current account
+  assume_current: false
+  # Do not scrape current account
+  do_not_scrape_current: false
+  # Account thread/process pool size
+  account_pool_size: 8
+  # Region thread pool size
+  region_pool_size: 128
+  # Number of threads available shared for all regions
+  shared_pool_size: 128
+  # Number of threads to collect a single region
+  region_resources_pool_size: 2
+  # List of AWS services to collect (default: all)
+  collect: []
+  # List of AWS services to exclude (default: none)
+  no_collect: []
+  # This value is used to look up atime and mtime for volumes and rds instances.
+  # It defines how long Resoto should look back for CloudWatch metrics.
+  # If no metric is found, now-period is used as atime and mtime. Defaults to 60 days.
+  cloudwatch_metrics_for_atime_mtime_period: '60d'
+  # Granularity of atime and mtime.
+  # Higher precision is more expensive: Resoto will fetch period * granularity data points.
+  # Defaults to 1 hour.
+  cloudwatch_metrics_for_atime_mtime_granularity: '1h'
+gcp:
+  # GCP service account file(s)
+  service_account: []
+  # GCP project(s)
+  project: []
+  # GCP services to collect (default: all)
+  collect: []
+  # GCP services to exclude (default: none)
+  no_collect: []
+  # GCP project thread/process pool size
+  project_pool_size: 8
+  # Fork collector process instead of using threads
+  fork_process: true
+digitalocean:
+  # DigitalOcean API tokens for the teams to be collected
+  api_tokens: []
+  # DigitalOcean Spaces access keys for the teams to be collected, separated by colons
+  spaces_access_keys: []
+k8s:
+  # Configure access via kubeconfig files.
+  # Structure:
+  #   - path: "/path/to/kubeconfig"
+  #     all_contexts: false
+  #     contexts: ["context1", "context2"]
+  config_files: []
+  # Alternative: configure access to k8s clusters directly in the config.
+  # Structure:
+  #   - name: 'k8s-cluster-name'
+  #     certificate_authority_data: 'CERT'
+  #     server: 'https://k8s-cluster-server.example.com'
+  #     token: 'TOKEN'
+  configs: []
+  # Objects to collect (default: all)
+  collect: []
+  # Objects to exclude (default: none)
+  no_collect: []
+  # Thread/process pool size
+  pool_size: 8
+  # Fork collector process instead of using threads
+  fork_process: false
+slack:
+  # Bot token
+  bot_token: null
+  # Include archived channels
+  include_archived: false
+  # Do not verify the Slack API server TLS certificate
+  do_not_verify_ssl: false
+onelogin:
+  # Onelogin region
+  region: 'us'
+  # Onelogin client ID
+  client_id: null
+  # Onelogin client secret
+  client_secret: null

requirements-test.txt

@@ -6,3 +6,4 @@ mypy==0.982
 tox==3.27.1
 wheel==0.38.4
 coverage==6.5.0
+resoto-plugin-example-collector==3.0.0rc3

requirements.txt

@@ -4,16 +4,18 @@
 SQLAlchemy==2.0.0b3
 PyYAML==6.0
 rich==12.6.0
-resotolib==3.0.0a0
 resotoclient==1.2.1
 
 # bundle popular db drivers
-psycopg2==2.9.5 # postgres
+psycopg2-binary==2.9.5 # postgres
 pymysql==1.0.2 # mysql + mariadb
 
+resotolib==3.0.0rc3
 # all collector plugins
-resoto-plugin-aws==3.0.0a0
-resoto-plugin-digitalocean==3.0.0a0
-resoto-plugin-gcp==3.0.0a0
-resoto-plugin-k8s==3.0.0a0
+resoto-plugin-aws==3.0.0rc3
+resoto-plugin-digitalocean==3.0.0rc3
+resoto-plugin-gcp==3.0.0rc3
+resoto-plugin-k8s==3.0.0rc3
+resoto-plugin-slack==3.0.0rc3
+resoto-plugin-onelogin==3.0.0rc3
 

setup.py

@@ -4,6 +4,8 @@
 
 from setuptools import setup, find_packages
 
+import cloud2sql
+
 with open("requirements.txt") as f:
     required = f.read().splitlines()
 
@@ -19,10 +21,10 @@
 
 
 setup(
-    name="cloud2sql",
-    version="1.0.0a1",
-    description="Read infrastructure data from your cloud and export it to an SQL database.",
-    python_requires=">=3.9",
+    name=cloud2sql.__title__,
+    version=cloud2sql.__version__,
+    description=cloud2sql.__description__,
+    python_requires=">=3.10",
     classifiers=["Programming Language :: Python :: 3"],
     entry_points={"console_scripts": ["cloud2sql=cloud2sql.__main__:main"]},
     install_requires=required,