♻️ Rewrite project initialization to write files from python instead of jinja.

Author: rafalkrupinski

docs/index.md

@@ -9,7 +9,11 @@ Lapidary-render is a code generator that creates client code from an OpenAPI doc
 
 - [x] The generator code should be simple.
 
-    Lapidary-render does most of the processing in Python, only leaving the rendering to Jinja templates.
+    Generator processes data in three stages
+
+    1. Enhance and transform OpenAPI to a structure more close resembling python model structure
+    2. Convert the enhanced OpenAPI to a metamodel
+    3. Convert the metamodel to a syntax tree
 
 - [x] Client code should be simple.
 
@@ -61,7 +65,7 @@ origin
 : URL of the OpenAPI document, used when document_path is missing, or when `servers` is not defined, or the first server URL is a relative path.
 
 extra_sources
-: list of additional source roots for manually written code. The files will be interpreted as templates, but non-template files will also work.
+: list of additional source roots for manually written python files.
 
 plugins
 : list of plugin classes. See [the section on plug-ins](/plugins)

poetry.lock

@@ -11,7 +11,6 @@ version = "0.11.1"
 packages = [{ include = "lapidary", from = "src" }]
 readme = 'Readme.md'
 repository = 'https://github.com/python-lapidary/lapidary'
-include = [{ path = 'src/lapidary/render/templates' }]
 classifiers = [
     "Development Status :: 3 - Alpha",
     "Environment :: Console",
@@ -41,6 +40,7 @@ pydantic = "^2.5.2"
 python-mimeparse = ">=1.6,<3.0"
 ruamel-yaml = "^0.18.6"
 openapi-pydantic = ">=0.5.0,<0.6.0"
+tomli-w = "^1.1.0"
 
 [tool.poetry.scripts]
 lapidary = "lapidary.render:app"
@@ -55,7 +55,6 @@ pre-commit = "^4.0.1"
 [tool.ruff]
 target-version = "py312"
 extend-exclude = [
-    "src/lapidary/render/templates/",
     "tests/e2e",
 ]
 line-length = 120
@@ -82,7 +81,6 @@ ignore_missing_imports = true
 python_version = "3.12"
 packages = ['lapidary.render']
 exclude = [
-    "src/lapidary/render/templates/",
     "tests/e2e",
 ]
 

pyproject.toml

@@ -25,6 +25,8 @@ async def init_project(
     if await project_root.exists():
         raise FileExistsError
 
+    from .writer import init_project
+
     document_handler = document_handler_for(anyio.Path(), document_path)
 
     if save_document:
@@ -58,32 +60,12 @@ async def init_project(
 
     yaml = ruamel.yaml.YAML(typ='safe')
     document = yaml.load(await document_handler.load())
-    # keep ruff happy
-    config.__str__()
-    document.__str__()
-
-    # from rybak import TreeTemplate
-    # from rybak.jinja import JinjaAdapter
-    #
-    # TreeTemplate(
-    #     JinjaAdapter(
-    #         jinja2.Environment(
-    #             loader=jinja2.loaders.PackageLoader('lapidary.render', package_path='templates/init'),
-    #         )
-    #     ),
-    #     remove_suffixes=['.jinja'],
-    # ).render(
-    #     dict(
-    #         get_version=importlib.metadata.version,
-    #         config=config.model_dump(exclude_unset=True, exclude_defaults=True, exclude_none=True),
-    #         document=document,
-    #     ),
-    #     pathlib.Path(project_root),
-    # )
+
+    await init_project(project_root, config, document)
 
 
 async def render_project(project_root: anyio.Path) -> None:
-    from .writer import write_all
+    from .writer import update_project
 
     config = await load_config(project_root)
 
@@ -102,7 +84,7 @@ async def render_project(project_root: anyio.Path) -> None:
         def progress(module: python.AbstractModule) -> None:
             progressbar.update(1, module)
 
-        await write_all(
+        await update_project(
             model.modules,
             project_root / config.extra_sources[0] if config.extra_sources else None,
             project_root / 'gen',

src/lapidary/render/main.py

@@ -0,0 +1,30 @@
+from .config import Config
+
+
+def mk_pyproject_toml(
+    title: str,
+    config: Config,
+) -> dict:
+    return {
+        'build-system': {'build-backend': 'poetry.core.masonry.api', 'requires': ['poetry-core>=1.3.2']},
+        'tool': {
+            'poetry': dict(
+                name=config.package,
+                description=f'Client library for {title}',
+                version='0.1.0',
+                authors=[],
+                license='',
+                packages=[
+                    {
+                        'include': config.package,
+                        'from': 'gen',
+                    }
+                ],
+                dependencies={
+                    'python': '3.9',
+                    'lapidary': '^0.12.0',
+                },
+            ),
+            'lapidary': config.model_dump(mode='json', exclude_unset=True, exclude_defaults=True),
+        },
+    }

src/lapidary/render/pyproject.py

@@ -1,16 +1,36 @@
 import logging
 import shutil
-from collections.abc import Callable, Iterable
+from collections.abc import Callable, Iterable, Sequence
 
 import anyio
 import asyncclick
+import libcst as cst
 
+from .config import Config
 from .model import conv_cst, python
 
 logger = logging.getLogger(__name__)
 
 
-async def write_all(
+def mk_module(module: python.AbstractModule) -> cst.Module | None:
+    match module:
+        case python.SchemaModule():
+            return conv_cst.mk_schema_module(module)
+        case python.ClientModule():
+            return conv_cst.mk_client_module(module)
+        case python.SecurityModule():
+            if not module.body:
+                return None
+            return conv_cst.mk_security_module(module)
+        case python.MetadataModule():
+            return conv_cst.mk_metadata_module(module)
+        case python.EmptyModule():
+            return conv_cst.MODULE_EMPTY
+        case _:
+            raise TypeError(type(module))
+
+
+async def update_project(
     modules: Iterable[python.AbstractModule],
     src_root: anyio.Path | None,
     target_root: anyio.Path,
@@ -21,25 +41,13 @@ async def write_all(
     written: list[anyio.Path] = []
     for module in modules:
         update_progress(module)
+        cst_module = mk_module(module)
+        if not cst_module:
+            continue
         path = module.path.to_path()
         full_path = target_root / path.with_suffix('.py')
-        match module:
-            case python.SchemaModule():
-                code = conv_cst.mk_schema_module(module).code
-            case python.ClientModule():
-                code = conv_cst.mk_client_module(module).code
-            case python.SecurityModule():
-                if not module.body:
-                    continue
-                code = conv_cst.mk_security_module(module).code
-            case python.MetadataModule():
-                code = conv_cst.mk_metadata_module(module).code
-            case python.EmptyModule():
-                code = conv_cst.MODULE_EMPTY.code
-            case _:
-                raise TypeError(type(module))
         await full_path.parent.mkdir(parents=True, exist_ok=True)
-        await full_path.write_text(code)
+        await full_path.write_text(cst_module.code)
         written.append(full_path.relative_to(target_root))
 
     root_module_path = anyio.Path(root_package) / '__init__.py'
@@ -50,16 +58,13 @@ async def write_all(
     written.append(anyio.Path(root_package, 'py.typed'))
 
     if src_root:
-        with asyncclick.progressbar(length=0, label='Copying extra sources') as bar:
-            async for parent, _, files in src_root.walk():
-                bar.length += len(files)
-                target_parent = target_root / parent.relative_to(src_root)
-                await target_parent.mkdir(parents=True, exist_ok=True)
-                for file in files:
-                    rel_path = parent.relative_to(src_root) / file
-                    bar.update(1, str(rel_path))
-                    shutil.copy(parent / file, target_root / file)
-                    written.append(rel_path)
+
+        def cp(src: str, names: list[str]) -> Sequence[str]:
+            local_src = anyio.Path(src)
+            written.extend((local_src / name).relative_to(src_root) for name in names)
+            return ()
+
+        shutil.copytree(src_root, target_root, ignore=cp, dirs_exist_ok=True)
 
     with asyncclick.progressbar(length=0, label='Removing stale files') as bar:
         async for parent, dirs, files in target_root.walk(False):
@@ -74,3 +79,33 @@ async def write_all(
             if not files_ and not dirs:
                 bar.update(1, str(parent))
                 await parent.rmdir()
+
+
+GITIGNORE_LINES = (
+    '/dist/',
+    '__pycache__',
+)
+
+
+async def write_gitignore(project_root: anyio.Path):
+    async with await anyio.open_file(project_root / '.gitignore', 'w') as f:
+        await f.writelines(GITIGNORE_LINES)
+
+
+async def write_pyproject(project_root: anyio.Path, title: str, config: Config):
+    import tomli_w
+
+    from .pyproject import mk_pyproject_toml
+
+    await (project_root / 'pyproject.toml').write_text(
+        f"""# This file was generated but won't be updated automatically and may be edited manually.
+
+{tomli_w.dumps(mk_pyproject_toml(title, config))}
+"""
+    )
+
+
+async def init_project(project_root: anyio.Path, config: Config, raw_document: dict):
+    await project_root.mkdir(parents=True, exist_ok=True)
+    await write_pyproject(project_root, raw_document['info']['title'], config)
+    await write_gitignore(project_root)