Merge pull request #414 from dhellmann/doc-config

add auto-generated config reference documentation

Author: mergify[bot]

.readthedocs.yaml

@@ -11,7 +11,7 @@ sphinx:
 # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
 python:
   install:
-    - requirements: docs/requirements.txt
+    - requirements: .[docs]
     - method: pip
       path: .
 

docs/conf.py

@@ -9,29 +9,40 @@
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 
 project = "Fromager"
-copyright = "2024, Doug Hellmann"
-author = "Doug Hellmann"
+copyright = "2024, Fromager Authors"
+author = "Fromager Authors"
 release = metadata.version("fromager")
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
 
-extensions = ["sphinx_click", 
-              "myst_parser"
-        ]
+extensions = [
+    "sphinx_click",
+    "myst_parser",
+    "sphinxcontrib.autodoc_pydantic",
+]
 
 # Recognized suffixes
 source_suffix = [
-    ".rst", ".md",
+    ".rst",
+    ".md",
 ]
 
-templates_path = ['_templates']
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+templates_path = ["_templates"]
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
-language = 'English'
+language = "English"
 
 # -- Options for HTML output -------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
 
-html_theme = 'alabaster'
-html_static_path = ['_static']
+html_theme = "sphinx_rtd_theme"
+html_static_path = ["_static"]
+
+suppress_warnings = [
+    "ref",  # autodoc generates a lot of warnings for missing classes that we do not want to include
+]
+autodoc_pydantic_model_show_json = False
+autodoc_pydantic_model_show_config_summary = False
+autodoc_pydantic_model_show_validator_summary = False
+autodoc_pydantic_model_show_validator_members = False

docs/config-reference.rst

@@ -0,0 +1,29 @@
+Configuration Reference
+=======================
+
+Per-package Settings
+--------------------
+
+Settings for individual packages can be placed in the `overrides/settings/`
+directory. Files should be named using the canonicalized name of the package.
+For example `flash_attn.yaml`.
+
+.. autopydantic_model:: fromager.packagesettings.PackageSettings
+
+.. autopydantic_model:: fromager.packagesettings.BuildOptions
+
+.. autopydantic_model:: fromager.packagesettings.DownloadSource
+
+.. autopydantic_model:: fromager.packagesettings.ResolverDist
+
+.. autopydantic_model:: fromager.packagesettings.ProjectOverride
+
+Global Settings
+---------------
+
+The global changelogs can be placed in `overrides/settings.yaml`.
+
+If you prefer managing a single settings file, per-package settings can also be
+kept in this file.
+
+.. autopydantic_model:: fromager.packagesettings.SettingsFile

docs/index.rst

@@ -28,6 +28,7 @@ those special cases directly into fromager.
 
    using.md
    customization.md
+   config-reference.rst
    cli.rst
    develop.md
    files.md

docs/requirements.txt

@@ -1,4 +1,5 @@
 sphinx
 sphinx-click
 myst-parser
-
+sphinx-rtd-theme
+autodoc-pydantic

pyproject.toml

@@ -33,6 +33,7 @@ requires-python = ">=3.11"
 optional-dependencies.build = { file = ["requirements-build.txt"] }
 optional-dependencies.mypy = { file = ["requirements-mypy.txt"] }
 optional-dependencies.test = { file = ["requirements-test.txt"] }
+optional-dependencies.docs = { file = ["docs/requirements.txt"] }
 dependencies = { file = ["requirements.txt"] }
 
 [project.urls]
@@ -57,19 +58,11 @@ build_wheel = "fromager.wheels:default_build_wheel"
 branch = true
 parallel = true
 relative_files = true
-source = [
-    "fromager",
-    "tests/",
-]
+source = ["fromager", "tests/"]
 
 [tool.coverage.paths]
-source = [
-   "src/fromager",
-   ".tox/**/site-packages/fromager",
-]
-tests =[
-   "tests/",
-]
+source = ["src/fromager", ".tox/**/site-packages/fromager"]
+tests = ["tests/"]
 
 [tool.coverage.report]
 show_missing = true
@@ -129,10 +122,10 @@ mypy_path = ["src"]
 # TODO: remove excludes and silent follow
 follow_imports = "silent"
 exclude = [
-  "^src/fromager/sources\\.py$",
-  "^src/fromager/sdist\\.py$",
-  "^src/fromager/commands/build\\.py$",
-  "^src/fromager/settings\\.py$",
+    "^src/fromager/sources\\.py$",
+    "^src/fromager/sdist\\.py$",
+    "^src/fromager/commands/build\\.py$",
+    "^src/fromager/settings\\.py$",
 ]
 
 [[tool.mypy.overrides]]

src/fromager/packagesettings.py

@@ -108,7 +108,14 @@ def _validate_envkey(v: typing.Any) -> str:
 
 
 class ResolverDist(pydantic.BaseModel):
-    """Packages resolver dist"""
+    """Packages resolver dist
+
+    ::
+
+      sdist_server_url: https://pypi.org/simple/
+      include_sdists: True
+      include_wheels: False
+    """
 
     model_config = MODEL_CONFIG
 
@@ -126,6 +133,11 @@ class DownloadSource(pydantic.BaseModel):
     """Package download source
 
     Download package sources from an alternative source, e.g. GitHub release.
+
+    ::
+
+        url: https://example.com/package.tar.gz
+        destination_filename: ${dist_name}-${version}.tar.gz
     """
 
     model_config = MODEL_CONFIG
@@ -145,12 +157,19 @@ def validate_destination_filename(cls, v):
 
 
 class BuildOptions(pydantic.BaseModel):
-    """Build system options"""
+    """Build system options
+
+    ::
+
+        build_ext_parallel: False
+        cpu_cores_per_job: 1
+        memory_per_job_gb: 1.0
+    """
 
     model_config = MODEL_CONFIG
 
     build_ext_parallel: bool = False
-    """Configure build_ext[parallel] in DIST_EXTRA_CONFIG
+    """Configure `build_ext[parallel]` in `DIST_EXTRA_CONFIG`
 
     This enables parallel builds of setuptools extensions. Incompatible
     with some packages, e.g. numba 0.60.0.
@@ -160,20 +179,31 @@ class BuildOptions(pydantic.BaseModel):
     """Scale parallel jobs by available CPU cores
 
     Examples:
+
     1: as many parallel jobs as CPU logical cores
+
     2: allocate 2 cores per job
     """
 
     memory_per_job_gb: float = Field(default=1.0, ge=0.1)
     """Scale parallel jobs by available virtual memory (without swap)
 
     Examples:
+
     0.5: assume each parallel job requires 512 MB virtual memory
     """
 
 
 class ProjectOverride(pydantic.BaseModel):
-    """Override pyproject.toml settings"""
+    """Override pyproject.toml settings
+
+    ::
+
+      update_build_requires:
+        - setuptools
+      remove_build_requires:
+        - ninja
+    """
 
     model_config = MODEL_CONFIG
 
@@ -194,7 +224,16 @@ def validate_update_build_requires(cls, v: list[str]) -> list[str]:
 
 
 class VariantInfo(pydantic.BaseModel):
-    """Variant information for a package"""
+    """Variant information for a package
+
+    ::
+
+      env:
+        VAR1: "value 1"
+        VAR2: "2.0
+      wheel_server_url: https://pypi.org/simple/
+      pre_build: False
+    """
 
     model_config = MODEL_CONFIG
 
@@ -214,7 +253,8 @@ class VariantInfo(pydantic.BaseModel):
 class PackageSettings(pydantic.BaseModel):
     """Package settings
 
-    yaml::
+    ::
+
         build_dir: python
         changelog:
             "1.0.1":
@@ -611,7 +651,16 @@ def serialize(self, **kwargs) -> dict[str, typing.Any]:
 
 
 class SettingsFile(pydantic.BaseModel):
-    """Models global settings file `settings.yaml`"""
+    """Models global settings file `settings.yaml`
+
+    ::
+
+      changelog:
+        cuda:
+          - "2024-09-13: updated CUDA version"
+        rocm:
+          - "2024-09-01: updated ROCm version"
+    """
 
     model_config = MODEL_CONFIG
 

tox.ini

@@ -81,7 +81,6 @@ commands=
 [testenv:docs]
 description = sphinx docs
 basepython = python3.11
-deps =
-    -r docs/requirements.txt
+deps = .[docs]
 commands =
     sphinx-build -M html docs docs/_build -j auto --keep-going {posargs:--fail-on-warning --fresh-env -n}