feat: Use latest Resoto and bump version (#40)

Author: aquamatthias

Makefile

@@ -1,4 +1,4 @@
-.PHONY: clean clean-test clean-pyc clean-build clean-env docs help setup
+.PHONY: clean clean-test clean-pyc clean-build clean-env docs help setup test test-all
 .DEFAULT_GOAL := help
 .SILENT: clean clean-build clean-pyc clean-test setup
 
@@ -77,8 +77,7 @@ venv:
 	python3 -m venv venv --prompt "cloud2sql"
 	. ./venv/bin/activate && python3 -m pip install --upgrade pip
 	. ./venv/bin/activate && pip install -r requirements-test.txt
-	. ./venv/bin/activate && pip install -r requirements.txt
-	. ./venv/bin/activate && pip install -e .
+	. ./venv/bin/activate && pip install -e ".[all]"
 	. ./venv/bin/activate && mypy --install-types --non-interactive cloud2sql tests
 
 setup: clean clean-env venv

cloud2sql/__init__.py

@@ -11,4 +11,4 @@
 __author__ = "Some Engineering Inc."
 __license__ = "Apache 2.0"
 __copyright__ = "Copyright © 2022 Some Engineering Inc."
-__version__ = "0.7.2"
+__version__ = "0.8.0"

cloud2sql/arrow/writer.py

@@ -168,7 +168,6 @@ def value_to_string(v: Any) -> str:
 
 
 def write_batch_to_file(batch: ArrowBatch) -> ArrowBatch:
-
     to_normalize = collect_normalization_paths(batch.schema)
 
     for row in batch.rows:

config-template.yaml

@@ -25,9 +25,9 @@ sources:
     # Account thread/process pool size
     account_pool_size: 8
     # Region thread pool size
-    region_pool_size: 128
+    region_pool_size: 32
     # Number of threads available shared for all regions
-    shared_pool_size: 128
+    shared_pool_size: 32
     # Number of threads to collect a single region
     region_resources_pool_size: 2
     # List of AWS services to collect (default: all)
@@ -97,7 +97,7 @@ sources:
     # Onelogin client secret
     client_secret: null
 destinations:
-  # Define the destination to sync. 
+  # Define the destination to sync.
   # PostgreSQL is listed here as example - see README.md for a list of possible targets.
   posgresql:
     host: 127.0.0.1

requirements-parquet.txt

@@ -1,3 +1,3 @@
-pyarrow==10.0.1
+pyarrow==11.0.0
 google-cloud-storage==2.7.0
-boto3==1.26.10
+boto3>=1.26.61

requirements-snowflake.txt

@@ -1 +1 @@
-snowflake-sqlalchemy>=1.4.4
+snowflake-sqlalchemy==1.4.6

requirements-test.txt

@@ -1,9 +1,9 @@
 # test dependencies
-pytest==7.2.0
-black==22.10.0
+pytest==7.2.2
+black==23.1.0
 flake8>=6.0.0
-mypy>=0.991
-tox>=4.0.3
+mypy==1.1.1
+tox==4.4.6
 wheel>=0.38.4
-coverage>=6.5.0
-resoto-plugin-example-collector>=3.0.0
+coverage==7.2.1
+resoto-plugin-example-collector==3.2.5

requirements.txt

@@ -6,11 +6,11 @@ PyYAML>=6.0
 rich>=12.6.0
 resotoclient>=1.2.1
 posthog>=2.2.0
-requests>=2.28.1
+requests>=2.28.2
 
-resotolib>=3.1.0
+resotolib>=3.2.5, <3.3
 # all collector plugins
-resoto-plugin-aws>=3.1.0
-resoto-plugin-digitalocean>=3.1.0
-resoto-plugin-gcp>=3.1.0
-resoto-plugin-k8s>=3.1.0
+resoto-plugin-aws>=3.2.5, <3.3
+resoto-plugin-digitalocean>=3.2.5, <3.3
+resoto-plugin-gcp>=3.2.5, <3.3
+resoto-plugin-k8s>=3.2.5, <3.3

tests/arrow/model_test.py

@@ -34,7 +34,6 @@ def test_create_schema(model: Model) -> None:
 
 
 def test_update(parquet_writer: ArrowWriter) -> None:
-
     parquet_writer.insert_node(  # type: ignore
         {
             "type": "node",