Finish doc generation and add tests

Author: hermanschaaf

cloudquery/sdk/docs/generator.py

@@ -98,28 +98,43 @@ def _generate_markdown(self, directory: str):
             f.write(formatted_all_tables)
 
         for table in self._tables:
-            table_template = env.from_string(TABLE)
-            table_md = table_template.render(table=table)
-            formatted_table_md = self._format_markdown(table_md)
-            with open(os.path.join(directory, table.name + ".md"), "w") as f:
-                f.write(formatted_table_md)
+            self._render_table(directory, env, table)
+
+    def _render_table(self, directory: str, env: jinja2.Environment, table: Table):
+        table_template = env.from_string(TABLE)
+        table_md = table_template.render(table=table)
+        formatted_table_md = self._format_markdown(table_md)
+        with open(os.path.join(directory, table.name + ".md"), "w") as f:
+            f.write(formatted_table_md)
+        for relation in table.relations:
+            self._render_table(directory, env, relation)
 
     def _all_tables_entry(self, table: Table):
         env = jinja2.Environment()
         env.globals['indent_to_depth'] = self._indent_to_depth
         env.globals['all_tables_entry'] = self._all_tables_entry
+        env.globals['indent_table_to_depth'] = self._indent_table_to_depth
         entry_template = env.from_string(ALL_TABLES_ENTRY)
         return entry_template.render(table=table)
 
     @staticmethod
-    def _indent_to_depth(text: str, depth: int):
+    def _indent_table_to_depth(table: Table) -> str:
+        s = ""
+        t = table
+        while t.parent is not None:
+            s += "  "
+            t = t.parent
+        return s
+
+    @staticmethod
+    def _indent_to_depth(text: str, depth: int) -> str:
         indentation = depth * 4  # You can adjust the number of spaces as needed
         lines = text.split('\n')
         indented_lines = [(' ' * indentation) + line for line in lines]
         return '\n'.join(indented_lines)
 
     @staticmethod
-    def _format_markdown(text: str):
+    def _format_markdown(text: str) -> str:
         re_match_newlines = re.compile(r'\n{3,}')
         re_match_headers = re.compile(r'(#{1,6}.+)\n+')
 

cloudquery/sdk/docs/markdown_templates.py

@@ -1,13 +1,13 @@
 ALL_TABLES = """# Source Plugin: {{ plugin_name }}
 ## Tables
-{% for table in tables %}
+{%- for table in tables %}
 {{ all_tables_entry(table) }}
-{% endfor %}"""
+{%- endfor %}"""
 
-ALL_TABLES_ENTRY = """{{ "" | indent(4) }}- [{{ table.name }}]({{ table.name }}.md){% if table.is_incremental %} (Incremental){% endif %}
-{% for rel in table.relations %}
-{{ all_tables_entry(rel) | indent(4) }}
-{% endfor %}"""
+ALL_TABLES_ENTRY = """{{ indent_table_to_depth(table) }}- [{{ table.name }}]({{ table.name }}.md){% if table.is_incremental %} (Incremental){% endif %}
+{%- for rel in table.relations %}
+{{ all_tables_entry(rel) }}
+{%- endfor %}"""
 
 TABLE = """# Table: {{ table.name }}
 
@@ -21,21 +21,20 @@
 The primary key for this table is **{{ table.primary_keys[0] }}**.
 {% else %}
 The composite primary key for this table is (
-{% for index, pk in table.primary_keys %}
-    {{ "  ," if index else "" }} **{{ pk }}**
-{% endfor %}
+{%- for pk in table.primary_keys %}
+    {{- ", " if loop.index > 1 else "" }}**{{ pk }}**
+{%- endfor -%}
 ).
 {% endif %}
 
 {% if table.is_incremental %}
 It supports incremental syncs{% set ik_length = table.incremental_keys | length %}
-{% if ik_length == 1 %} based on the **{{ table.incremental_keys[0] }}** column{% else %}
- based on the (
-{% for index, pk in table.incremental_keys %}
-    {{ "  ," if index else "" }} **{{ pk }}**
-{% endfor %}
+{%- if ik_length == 1 %} based on the **{{ table.incremental_keys[0] }}** column{% else %} based on the (
+{%- for ik in table.incremental_keys %}
+    {{- ", " if loop.index > 1 else "" }}**{{ ik }}**
+{%- endfor -%}
 ) columns{% endif %}.
-{% endif %}
+{%- endif %}
 
 {% if table.relations or table.parent %}
 ## Relations

cloudquery/sdk/schema/table.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from typing import List
 
 import pyarrow as pa
@@ -8,13 +10,14 @@
 
 class Table:
     def __init__(self, name: str, columns: List[Column], title: str = "", description: str = "",
-                 relations=None, is_incremental: bool = False) -> None:
+                 parent: Table = None, relations: List[Table] = None, is_incremental: bool = False) -> None:
         self.name = name
         self.columns = columns
         self.title = title
         self.description = description
         if relations is None:
             relations = []
+        self.parent = parent
         self.relations = relations
         self.is_incremental = is_incremental
 

requirements.txt

@@ -3,4 +3,5 @@ grpcio-tools==1.56.2
 plugin-pb-python==0.0.11
 protobuf==4.23.4
 pyarrow==12.0.1
-pytest==7.4.0
+pytest==7.4.0
+Jinja2~=3.1.2

tests/docs/__init__.py

@@ -0,0 +1,9 @@
+# Source Plugin: test_plugin
+
+## Tables
+
+- [test_table](test_table.md)
+- [test_table_composite_pk](test_table_composite_pk.md) (Incremental)
+- [test_table_relations](test_table_relations.md) (Incremental)
+  - [test_table_child](test_table_child.md)
+    - [test_table_grandchild](test_table_grandchild.md)

tests/docs/snapshots/README.md

@@ -0,0 +1,12 @@
+# Table: test_table
+
+This table shows data for Test Table.
+
+The primary key for this table is **string**.
+
+## Columns
+
+| Name          | Type          |
+| ------------- | ------------- |
+|string (PK)|`string`|
+|int32|`int32`|

tests/docs/snapshots/test_table.md

@@ -0,0 +1,20 @@
+# Table: test_table_child
+
+This table shows data for Child Table.
+
+The primary key for this table is **pk1**.
+
+## Relations
+
+This table depends on [test_table_relations](test_table_relations.md).
+
+The following tables depend on test_table_child:
+
+  - [test_table_grandchild](test_table_grandchild.md)
+
+## Columns
+
+| Name          | Type          |
+| ------------- | ------------- |
+|pk1 (PK)|`string`|
+|fk1|`string`|

tests/docs/snapshots/test_table_child.md

@@ -0,0 +1,15 @@
+# Table: test_table_composite_pk
+
+This table shows data for Composite PKs.
+
+The composite primary key for this table is (**pk1**, **pk2**).
+
+It supports incremental syncs based on the (**pk1**, **pk2**) columns.
+
+## Columns
+
+| Name          | Type          |
+| ------------- | ------------- |
+|pk1 (PK) (Incremental Key)|`string`|
+|pk2 (PK) (Incremental Key)|`string`|
+|int32|`int32`|

tests/docs/snapshots/test_table_composite_pk.md

@@ -0,0 +1,15 @@
+# Table: test_table_grandchild
+
+This table shows data for Grandchild Table.
+
+The primary key for this table is **pk1**.
+
+## Relations
+
+This table depends on [test_table_child](test_table_child.md).
+
+## Columns
+
+| Name          | Type          |
+| ------------- | ------------- |
+|pk1 (PK)|`string`|