feat: Initial

Author: hydratim

.dockerignore

@@ -0,0 +1,110 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a example
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# profiling data
+.prof

.gitignore

@@ -0,0 +1,2 @@
+venv
+__pycache__

CHANGELOG.md

@@ -0,0 +1 @@
+# Changelog

Dockerfile

@@ -0,0 +1,23 @@
+# Start from a base image
+FROM python:3.11-slim
+
+# (Optional) Set a working directory
+WORKDIR /app
+
+# Copy requirements.txt and install the Python dependencies
+COPY pyproject.toml .
+COPY poetry.lock .
+RUN pip3 install --no-cache-dir poetry
+RUN poetry install
+
+# Copy the rest of the code
+COPY plugin plugin
+COPY main.py .
+
+# (Optional) Expose any ports your app uses
+EXPOSE 7777
+
+ENTRYPOINT ["python3", "main.py"]
+
+# Specify the command to run when the container starts
+CMD ["serve", "--address", "[::]:7777", "--log-format", "json", "--log-level", "info"]

Makefile

@@ -0,0 +1,8 @@
+test:
+	pytest .
+
+fmt:
+	black .
+
+fmt-check:
+	black --check .

README.md

@@ -0,0 +1,66 @@
+# Python Source Plugin Template
+This repo contains everything you need to get started with building a new plugin.
+To get started all you need to do is change a few names, define some tables, and write an API Client to populate the tables.
+
+## Key files & classes
+ - plugin/tables/items.py
+    - Items - A boilerplate table definition
+    - ItemResolver - A boilerplate table resolver
+ - plugin/example/client.py
+  - ExampleClient - A boilerplate API Client
+ - plugin/client/client.py
+   - Spec - Defines the CloudQuery Config
+   - Client (uses: ExampleClient) - Wraps your API Client
+ - plugin/plugin.py
+   - ExamplePlugin - The plugin registration / how CloudQuery knows what tables your plugin exposes.
+
+
+
+## Getting started
+
+### Defining your tables
+The first thing you need to do is identify the tables you want to create with your plugin.
+Conventionally, CloudQuery plugins have a direct relationship between tables and API responses.
+
+For example:
+   If you had an API endpoint https://api.example.com/items/{num} and for each value of `num` it provided an object
+   ```json
+   {
+      "num": {{num}},
+      "date": "2023-10-12", 
+      "title": "A simple example"
+   }
+   ```
+   Then you would design the table class as
+   ```python
+   class Items(Table):
+       def __init__(self) -> None:
+           super().__init__(
+               name="item",
+               title="Item",
+               columns=[
+                   Column("num", pa.uint64(), primary_key=True),
+                   Column("date", pa.date64()),
+                   Column("title", pa.string()),
+               ],
+           )
+       ...
+   ```
+
+Creating one table for each endpoint that you want to capture.
+
+### API Client
+Next you'll need to define how the tables are retrieved, it's recommended to implement this as a generator, as per the example in `plugin/example/client.py`.
+
+### Spec
+Having written your API Client you will have, identified the authentication and/or operational variables needed.
+Adding these to the CloudQuery config spec can be done by editing the `Spec` `dataclass` using standard python, and adding validation where needed.
+
+### Plugin
+Finally, you need to edit the `plugin.py` file to set the plugin name and version, and add the `Tables` to the `get_tables` function. 
+
+
+## Links
+
+- [Architecture](https://www.cloudquery.io/docs/developers/architecture)
+- [Concepts](https://www.cloudquery.io/docs/developers/creating-new-plugin/python-source)

main.py

@@ -0,0 +1,13 @@
+import sys
+from cloudquery.sdk import serve
+
+from plugin import ExamplePlugin
+
+
+def main():
+    p = ExamplePlugin()
+    serve.PluginCommand(p).run(sys.argv[1:])
+
+
+if __name__ == "__main__":
+    main()

plugin/__init__.py

@@ -0,0 +1 @@
+from .plugin import ExamplePlugin

plugin/client/__init__.py

@@ -0,0 +1 @@
+from .client import Client, Spec

plugin/client/client.py

@@ -0,0 +1,33 @@
+from dataclasses import dataclass, field
+from cloudquery.sdk.scheduler import Client as ClientABC
+
+from plugin.example.client import ExampleClient
+
+DEFAULT_CONCURRENCY = 100
+DEFAULT_QUEUE_SIZE = 10000
+
+
+@dataclass
+class Spec:
+    access_token: str
+    base_url: str = field(default="https://api.example.com")
+    concurrency: int = field(default=DEFAULT_CONCURRENCY)
+    queue_size: int = field(default=DEFAULT_QUEUE_SIZE)
+
+    def validate(self):
+        pass
+        # if self.access_token is None:
+        #     raise Exception("access_token must be provided")
+
+
+class Client(ClientABC):
+    def __init__(self, spec: Spec) -> None:
+        self._spec = spec
+        self._client = ExampleClient(spec.access_token, spec.base_url)
+
+    def id(self):
+        return "example"
+
+    @property
+    def client(self) -> ExampleClient:
+        return self._client