ci(release): generate release notes latex from toml (#2108)

Store release notes in a TOML file and build the develop.tex file from it at distribution time. Storing release notes in a structured form allows more easily using them elsewhere, e.g. we could substitute in the nightly build release descriptions (#2059) or into the RTD site.

Add tomli and Jinja2 to the Python dependencies. I figure we might as well adopt Jinja for this since we have discussed adopting it for other tasks here as in flopy sooner or later.

Also move the contents of develop.tex into previous/v6.6.1.tex, we hadn't done this post-6.6.1 step yet. As this is one of the only remaining manual steps at release time we can automate it at some point?

By way of demonstration, the new develop.toml representing the develop branch right now becomes

[sections]
features = "NEW FUNCTIONALITY"
fixes = "BUG FIXES AND OTHER CHANGES TO EXISTING FUNCTIONALITY"
examples = "EXAMPLES"

[subsections]
basic = "BASIC FUNCTIONALITY"
internal = "INTERNAL FLOW PACKAGES"
stress = "STRESS PACKAGES"
advanced = "ADVANCED STRESS PACKAGES"
solution = "SOLUTION"
exchanges = "EXCHANGES"
parallel = "PARALLEL"

[[items]]
section = "features"
description = "The binary grid file's name may now be specified in all discretization packages with option GRB6 FILEOUT followed by a file path. If this option is not provided, the binary grid file will be named as before, identical to the discretization file name plus a '.grb' extension. Note that renaming the binary grid file may break downstream integrations which expect the default name."

Sections and subsections can be added/removed to the leading tables as necessary, then referenced from items. For now all the existing headers/subheaders which were previously in the template are included.

I considered different ways of doing this, including

- structuring the TOML exactly as the latex was/will be, with a list of items inside each nested section — but I figured a single flat list of items was clearer
- putting sections/subsections in the jinja template and removing them from the toml file, then using a "tag" system to assign an item to section/subsection

..and went with the current approach for lack of a strong sense yet which is best.

Author: wpbonelli

distribution/build_dist.py

@@ -324,7 +324,7 @@ def build_distribution(
     build_documentation(
         bin_path=output_path / "bin",
         full=full,
-        output_path=output_path / "doc",
+        out_path=output_path / "doc",
         force=force,
     )
 

distribution/build_docs.py

@@ -128,16 +128,40 @@ def build_deprecations_tex(force: bool = False):
             out, err, ret = run_py_script("mk_deprecations.py", md_path, verbose=True)
             assert not ret, out + err
 
-    # check deprecations files exist
     assert md_path.is_file()
     assert tex_path.is_file()
 
 
+def build_notes_tex(force: bool = False):
+    """Build LaTeX files for the release notes."""
+
+    build_deprecations_tex(force=force)
+
+    toml_path = RELEASE_NOTES_PATH / "develop.toml"
+    tex_path = RELEASE_NOTES_PATH / "develop.tex"
+    if tex_path.is_file() and not force:
+        print(f"{tex_path} already exists.")
+    else:
+        tex_path.unlink(missing_ok=True)
+        with set_dir(RELEASE_NOTES_PATH):
+            out, err, ret = run_py_script(
+                "mk_releasenotes.py", toml_path, tex_path, verbose=True
+            )
+            assert not ret, out + err
+
+    assert tex_path.is_file()
+
+
 @no_parallel
 def test_build_deprecations_tex():
     build_deprecations_tex(force=True)
 
 
+@no_parallel
+def test_build_notes_tex():
+    build_notes_tex(force=True)
+
+
 def build_mf6io_tex(models: Optional[list[str]] = None, force: bool = False):
     """Build LaTeX files for the MF6IO guide from DFN files."""
 
@@ -183,7 +207,7 @@ def test_build_mf6io_tex():
     build_mf6io_tex(force=True)
 
 
-def build_usage_example_tex(
+def build_usage_tex(
     workspace_path: PathLike, bin_path: PathLike, example_model_path: PathLike
 ):
     """
@@ -320,13 +344,38 @@ def test_build_pdfs_from_tex(tmp_path):
     ]
 
     build_pdfs(tex_paths, tmp_path)
-    for p in tex_paths[:-1] + bbl_paths:
-        assert p.is_file()
+
+    expected_paths = tex_paths[:-1] + bbl_paths
+    assert all(p.is_file() for p in expected_paths)
+
+
+def fetch_example_docs(
+    out_path: PathLike, force: bool = False, repo_owner: str = "MODFLOW-USGS"
+):
+    pdf_name = "mf6examples.pdf"
+    if force or not (out_path / pdf_name).is_file():
+        latest = get_release(f"{repo_owner}/modflow6-examples", "latest")
+        assets = latest["assets"]
+        asset = next(iter([a for a in assets if a["name"] == pdf_name]), None)
+        download_and_unzip(asset["browser_download_url"], out_path, verbose=True)
+
+
+def fetch_usgs_pubs(out_path: PathLike, force: bool = False):
+    for url in PUB_URLS:
+        print(f"Downloading publication: {url}")
+        try:
+            download_and_unzip(url, path=out_path, delete_zip=False)
+            assert (out_path / url.rpartition("/")[2]).is_file()
+        except HTTPError as e:
+            if "404" in str(e):
+                warn(f"Publication not found: {url}")
+            else:
+                raise
 
 
 def build_documentation(
     bin_path: PathLike,
-    output_path: PathLike,
+    out_path: PathLike,
     force: bool = False,
     full: bool = False,
     models: Optional[list[str]] = None,
@@ -337,72 +386,46 @@ def build_documentation(
     print(f"Building {'full' if full else 'minimal'} documentation")
 
     bin_path = Path(bin_path).expanduser().absolute()
-    output_path = Path(output_path).expanduser().absolute()
+    out_path = Path(out_path).expanduser().absolute()
+    pdf_path = out_path / "mf6io.pdf"
 
-    if (output_path / "mf6io.pdf").is_file() and not force:
-        print(f"{output_path / 'mf6io.pdf'} already exists")
+    if not force and pdf_path.is_file():
+        print(f"{pdf_path} already exists, nothing to do")
         return
 
-    # make sure output directory exists
-    output_path.mkdir(parents=True, exist_ok=True)
-
-    # build LaTex input/output docs from DFN files
-    build_mf6io_tex(force=force, models=models)
+    out_path.mkdir(parents=True, exist_ok=True)
 
-    # build LaTeX input/output example model docs
     with TemporaryDirectory() as temp:
-        build_usage_example_tex(
+        build_mf6io_tex(force=force, models=models)
+        build_usage_tex(
             bin_path=bin_path,
             workspace_path=Path(temp),
             example_model_path=PROJ_ROOT_PATH / ".mf6minsim",
         )
+        build_notes_tex(force=force)
 
-    # build deprecations table LaTeX
-    build_deprecations_tex(force=force)
+        if full:
+            build_benchmark_tex(out_path=out_path, force=force, repo_owner=repo_owner)
+            fetch_example_docs(out_path=out_path, force=force, repo_owner=repo_owner)
+            fetch_usgs_pubs(out_path=out_path, force=force)
+            tex_paths = TEX_PATHS["full"]
+        else:
+            tex_paths = TEX_PATHS["minimal"]
 
-    if full:
-        # build benchmarks table LaTex, running benchmarks first if necessary
-        build_benchmark_tex(output_path=output_path, force=force)
-
-        # download example docs
-        pdf_name = "mf6examples.pdf"
-        if force or not (output_path / pdf_name).is_file():
-            latest = get_release(f"{repo_owner}/modflow6-examples", "latest")
-            assets = latest["assets"]
-            asset = next(iter([a for a in assets if a["name"] == pdf_name]), None)
-            download_and_unzip(asset["browser_download_url"], output_path, verbose=True)
-
-        # download publications
-        for url in PUB_URLS:
-            print(f"Downloading publication: {url}")
-            try:
-                download_and_unzip(url, path=output_path, delete_zip=False)
-                assert (output_path / url.rpartition("/")[2]).is_file()
-            except HTTPError as e:
-                if "404" in str(e):
-                    warn(f"Publication not found: {url}")
-                else:
-                    raise
-
-        # convert LaTex to PDF
-        build_pdfs(tex_paths=TEX_PATHS["full"], output_path=output_path, force=force)
-    else:
-        # just convert LaTeX to PDF
-        build_pdfs(tex_paths=TEX_PATHS["minimal"], output_path=output_path, force=force)
+        build_pdfs(tex_paths=tex_paths, output_path=out_path, force=force)
 
     # enforce os line endings on all text files
     windows_line_endings = True
-    convert_line_endings(output_path, windows_line_endings)
+    convert_line_endings(out_path, windows_line_endings)
 
     # make sure we have expected PDFs
-    assert (output_path / "mf6io.pdf").is_file()
+    assert pdf_path.is_file()
     if full:
-        assert (output_path / "mf6io.pdf").is_file()
-        assert (output_path / "ReleaseNotes.pdf").is_file()
-        assert (output_path / "zonebudget.pdf").is_file()
-        assert (output_path / "converter_mf5to6.pdf").is_file()
-        assert (output_path / "mf6suptechinfo.pdf").is_file()
-        assert (output_path / "mf6examples.pdf").is_file()
+        assert (out_path / "ReleaseNotes.pdf").is_file()
+        assert (out_path / "zonebudget.pdf").is_file()
+        assert (out_path / "converter_mf5to6.pdf").is_file()
+        assert (out_path / "mf6suptechinfo.pdf").is_file()
+        assert (out_path / "mf6examples.pdf").is_file()
 
 
 @no_parallel
@@ -477,7 +500,7 @@ def test_build_documentation(tmp_path):
     models = args.model if args.model else DEFAULT_MODELS
     build_documentation(
         bin_path=bin_path,
-        output_path=output_path,
+        out_path=output_path,
         force=args.force,
         full=args.full,
         models=models,

doc/ReleaseNotes/develop.tex.jinja

@@ -0,0 +1,19 @@
+\subsection{Version mf(( version ))---(( date ))}
+
+([ for section_key, section_items in items|groupby("section") ])
+\textbf{\underline{(( sections[section_key] ))}}
+
+([ for subsection_key, subsection_items in section_items|groupby("subsection") ])
+([ set subsection = subsections[subsection_key] ])
+([ if subsection|length > 0 ])
+\underline{(( subsection ))}
+([ endif ])
+
+\begin{itemize}
+([ for item in subsection_items ])
+\item (( item.description ))
+([ endfor ])
+\end{itemize}
+
+([ endfor ])
+([ endfor ])

doc/ReleaseNotes/develop.toml

@@ -0,0 +1,18 @@
+[sections]
+features = "NEW FUNCTIONALITY"
+fixes = "BUG FIXES AND OTHER CHANGES TO EXISTING FUNCTIONALITY"
+examples = "EXAMPLES"
+
+[subsections]
+basic = "BASIC FUNCTIONALITY"
+internal = "INTERNAL FLOW PACKAGES"
+stress = "STRESS PACKAGES"
+advanced = "ADVANCED STRESS PACKAGES"
+solution = "SOLUTION"
+exchanges = "EXCHANGES"
+parallel = "PARALLEL"
+
+[[items]]
+section = "features"
+subsection = "basic"
+description = "The binary grid file's name may now be specified in all discretization packages with option GRB6 FILEOUT followed by a file path. If this option is not provided, the binary grid file will be named as before, identical to the discretization file name plus a ``.grb'' extension. Note that renaming the binary grid file may break downstream integrations which expect the default name."

doc/ReleaseNotes/mk_deprecations.py

@@ -68,4 +68,4 @@
         ftex.close()
         print(f"Created LaTex file {fnametex} from markdown deprecations file {fpath}")
     else:
-        warn(f"Deprecations not found: {fpath}")
+        warn(f"Deprecations file not found: {fpath}")

doc/ReleaseNotes/mk_releasenotes.py

@@ -0,0 +1,66 @@
+# This script converts the release notes TOML file
+# to a latex file, from which is later built a PDF.
+import argparse
+import datetime
+import sys
+from pathlib import Path
+from warnings import warn
+
+version_file = Path(__file__).parents[2] / "version.txt"
+version = version_file.read_text().strip()
+date = datetime.date.today().strftime("%b %d, %Y")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("toml_path")
+    parser.add_argument("tex_path")
+    args = parser.parse_args()
+    toml_path = Path(args.toml_path).expanduser().absolute()
+    tex_path = Path(args.tex_path).expanduser().absolute()
+    if not toml_path.is_file():
+        warn(f"Release notes TOML file not found: {toml_path}")
+        sys.exit(0)
+
+    tex_path.unlink(missing_ok=True)
+
+    import tomli
+    from jinja2 import Environment, FileSystemLoader
+
+    loader = FileSystemLoader(Path(__file__).parent)
+    env = Environment(
+        loader=loader,
+        trim_blocks=True,
+        lstrip_blocks=True,
+        line_statement_prefix="_",
+        keep_trailing_newline=True,
+        # since latex uses curly brackets,
+        # replace block/var start/end tags
+        block_start_string="([",
+        block_end_string="])",
+        variable_start_string="((",
+        variable_end_string="))",
+    )
+    template = env.get_template(f"{tex_path.name}.jinja")
+    with open(tex_path, "w") as tex_file:
+        with open(toml_path, "rb") as toml_file:
+            content = tomli.load(toml_file)
+            sections = content.get("sections", [])
+            subsections = content.get("subsections", [])
+            items = content.get("items", [])
+            # make sure each item has a subsection entry even if empty
+            for item in items:
+                if not item.get("subsection"):
+                    item["subsection"] = ""
+            if not any(items):
+                warn("No release notes found, aborting")
+                sys.exit(0)
+            tex_file.write(
+                template.render(
+                    sections=sections,
+                    subsections=subsections,
+                    items=items,
+                    version=version,
+                    date=date,
+                )
+            )

doc/ReleaseNotes/develop.tex renamed to doc/ReleaseNotes/previous/v6.6.1.tex

@@ -1,7 +1,6 @@
 % Use this template for starting initializing the release notes
 % after a release has just been made.
 
-%\item \currentmodflowversion
 \subsection{Version mf6.6.1---February 7, 2025}
 
 \underline{NEW FUNCTIONALITY}

environment.yml

@@ -14,6 +14,7 @@ dependencies:
   - fprettify
   - fortran-language-server
   - gitpython
+  - Jinja2>=3.1.5,<4
   - jupytext
   - matplotlib
   - meson=1.3.0
@@ -40,4 +41,5 @@ dependencies:
   - scipy
   - shapely
   - syrupy
+  - tomli>=2.2.1,<3
 

pixi.toml

@@ -13,6 +13,7 @@ flaky = "*"
 fortran-language-server = "*"
 fprettify = "*"
 gitpython = "*"
+jinja2 = ">=3.1.5,<4"
 jupytext = "*"
 matplotlib = "*"
 meson = "==1.3.0"
@@ -37,8 +38,10 @@ ruff = "*"
 scipy = "*"
 shapely = "*"
 syrupy = "*"
+tomli = ">=2.2.1,<3"
 xmipy = "*"
 
+
 [feature.rtd.dependencies]
 numpy = "*"
 bmipy = "*"