From ce443142209f4411e8349d472431a565608acbde Mon Sep 17 00:00:00 2001 From: Bettina Gier Date: Thu, 22 May 2025 18:35:48 +0200 Subject: [PATCH 01/18] draft changes to gallery and legacy recipes --- doc/sphinx/source/generate_gallery.py | 27 ++++++++++++++----- doc/sphinx/source/recipes/index.rst | 8 +++--- .../recipes/{ => legacy}/recipe_rainfarm.rst | 0 .../{ => legacy}/recipe_schlund20jgr.rst | 0 .../recipes/{ => legacy}/recipe_spei.rst | 0 doc/sphinx/source/recipes/recipe_gier20bg.rst | 1 + 6 files changed, 24 insertions(+), 12 deletions(-) rename doc/sphinx/source/recipes/{ => legacy}/recipe_rainfarm.rst (100%) rename doc/sphinx/source/recipes/{ => legacy}/recipe_schlund20jgr.rst (100%) rename doc/sphinx/source/recipes/{ => legacy}/recipe_spei.rst (100%) diff --git a/doc/sphinx/source/generate_gallery.py b/doc/sphinx/source/generate_gallery.py index 6ee72601d2..dfc992d5c9 100644 --- a/doc/sphinx/source/generate_gallery.py +++ b/doc/sphinx/source/generate_gallery.py @@ -2,6 +2,7 @@ """Create gallery with all available recipes.""" import os +from pathlib import Path RECIPE_DIR = "recipes" OUT_PATH = os.path.abspath("gallery.rst") @@ -31,10 +32,14 @@ def _get_figure_index(file_content): """Get index of figure in text.""" + offset = 0 + if ".. gallery=True" in file_content: + offset = file_content.index(".. gallery=True") + 15 + file_content = file_content[offset:] if FIGURE_STR in file_content: - return file_content.index(FIGURE_STR) + len(FIGURE_STR) + return offset + file_content.index(FIGURE_STR) + len(FIGURE_STR) if IMAGE_STR in file_content: - return file_content.index(IMAGE_STR) + len(IMAGE_STR) + return offset + file_content.index(IMAGE_STR) + len(IMAGE_STR) raise ValueError("File does not contain image") @@ -79,6 +84,17 @@ def _get_next_row(filenames, file_contents): return (row, refs) +def _find_recipes(root_dir, exclude_dirs): + return [ + path.relative_to(root_dir) + for path in Path(root_dir).rglob("recipe_*.rst") + if not any( + path.parents[0].name.startswith(exclude) + for exclude in exclude_dirs + ) + ] + + def main(): """Generate gallery for recipe plots.""" print(f"Generating gallery at {OUT_PATH}") @@ -87,11 +103,8 @@ def main(): refs = "" filenames = [] file_contents = [] - for filename in sorted(os.listdir(RECIPE_DIR)): - if not filename.startswith("recipe_"): - continue - if not filename.endswith(".rst"): - continue + fnames = _find_recipes(RECIPE_DIR, ["legacy", "figures"]) + for filename in sorted(fnames): with open(os.path.join(RECIPE_DIR, filename)) as in_file: recipe_file = in_file.read() if FIGURE_STR not in recipe_file and IMAGE_STR not in recipe_file: diff --git a/doc/sphinx/source/recipes/index.rst b/doc/sphinx/source/recipes/index.rst index c1104fb97e..8260b5dd95 100644 --- a/doc/sphinx/source/recipes/index.rst +++ b/doc/sphinx/source/recipes/index.rst @@ -107,7 +107,6 @@ IPCC recipe_ipccwg1ar6ch3 recipe_ipccwg1ar5ch9 recipe_collins13ipcc - recipe_examples Land ^^^^ @@ -165,11 +164,10 @@ require a legacy version of ESMValTool to run. .. toctree:: - :maxdepth: 1 + :maxdepth: 2 + + legacy_recipe_list - recipe_rainfarm - recipe_schlund20jgr - recipe_spei Broken recipe list diff --git a/doc/sphinx/source/recipes/recipe_rainfarm.rst b/doc/sphinx/source/recipes/legacy/recipe_rainfarm.rst similarity index 100% rename from doc/sphinx/source/recipes/recipe_rainfarm.rst rename to doc/sphinx/source/recipes/legacy/recipe_rainfarm.rst diff --git a/doc/sphinx/source/recipes/recipe_schlund20jgr.rst b/doc/sphinx/source/recipes/legacy/recipe_schlund20jgr.rst similarity index 100% rename from doc/sphinx/source/recipes/recipe_schlund20jgr.rst rename to doc/sphinx/source/recipes/legacy/recipe_schlund20jgr.rst diff --git a/doc/sphinx/source/recipes/recipe_spei.rst b/doc/sphinx/source/recipes/legacy/recipe_spei.rst similarity index 100% rename from doc/sphinx/source/recipes/recipe_spei.rst rename to doc/sphinx/source/recipes/legacy/recipe_spei.rst diff --git a/doc/sphinx/source/recipes/recipe_gier20bg.rst b/doc/sphinx/source/recipes/recipe_gier20bg.rst index b8f8fb9b8e..443fda48db 100644 --- a/doc/sphinx/source/recipes/recipe_gier20bg.rst +++ b/doc/sphinx/source/recipes/recipe_gier20bg.rst @@ -193,6 +193,7 @@ Example plots Barplot of the growth rate, averaged over all years, with standard deviation of interannual variability. +.. gallery=True .. _fig_gier20bg_5: .. figure:: /recipes/figures/gier20bg/fig05.png :align: center From dc9ce254d7b928aadec675ec64827909ab530e08 Mon Sep 17 00:00:00 2001 From: Bettina Gier Date: Fri, 23 May 2025 10:12:29 +0200 Subject: [PATCH 02/18] add legacy recipe page --- doc/sphinx/source/recipes/legacy_recipe_list.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 doc/sphinx/source/recipes/legacy_recipe_list.rst diff --git a/doc/sphinx/source/recipes/legacy_recipe_list.rst b/doc/sphinx/source/recipes/legacy_recipe_list.rst new file mode 100644 index 0000000000..dc32ed8720 --- /dev/null +++ b/doc/sphinx/source/recipes/legacy_recipe_list.rst @@ -0,0 +1,15 @@ +.. _legacy_recipe_list: + +Legacy Recipes +============== + +Recipes that have been retired and are included for +documentation purposes only. Typically, these recipes +require a legacy version of ESMValTool to run. + +.. toctree:: + :maxdepth: 1 + + legacy/recipe_rainfarm + legacy/recipe_schlund20jgr + legacy/recipe_spei From fb0f11f4fd5f6b589eb1dc2ddf4f3b4ce4856282 Mon Sep 17 00:00:00 2001 From: Bettina Gier Date: Fri, 23 May 2025 11:20:24 +0200 Subject: [PATCH 03/18] fav image in better comment style --- doc/sphinx/source/generate_gallery.py | 4 ++-- doc/sphinx/source/recipes/recipe_gier20bg.rst | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/sphinx/source/generate_gallery.py b/doc/sphinx/source/generate_gallery.py index dfc992d5c9..59634161d6 100644 --- a/doc/sphinx/source/generate_gallery.py +++ b/doc/sphinx/source/generate_gallery.py @@ -33,8 +33,8 @@ def _get_figure_index(file_content): """Get index of figure in text.""" offset = 0 - if ".. gallery=True" in file_content: - offset = file_content.index(".. gallery=True") + 15 + if "gallery=True" in file_content: + offset = file_content.index("gallery=True") + 12 file_content = file_content[offset:] if FIGURE_STR in file_content: return offset + file_content.index(FIGURE_STR) + len(FIGURE_STR) diff --git a/doc/sphinx/source/recipes/recipe_gier20bg.rst b/doc/sphinx/source/recipes/recipe_gier20bg.rst index 443fda48db..7ead4b7b58 100644 --- a/doc/sphinx/source/recipes/recipe_gier20bg.rst +++ b/doc/sphinx/source/recipes/recipe_gier20bg.rst @@ -193,7 +193,8 @@ Example plots Barplot of the growth rate, averaged over all years, with standard deviation of interannual variability. -.. gallery=True +.. + gallery=True .. _fig_gier20bg_5: .. figure:: /recipes/figures/gier20bg/fig05.png :align: center From 720e999c0405a98c8a822bc14b01df315c784ac1 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Tue, 27 May 2025 12:32:14 +0200 Subject: [PATCH 04/18] docutils parser, html layout, multiple figures per recipe --- doc/sphinx/source/generate_gallery.py | 201 ++++++++++++-------------- 1 file changed, 92 insertions(+), 109 deletions(-) diff --git a/doc/sphinx/source/generate_gallery.py b/doc/sphinx/source/generate_gallery.py index 59634161d6..7fe2301e4c 100644 --- a/doc/sphinx/source/generate_gallery.py +++ b/doc/sphinx/source/generate_gallery.py @@ -2,7 +2,12 @@ """Create gallery with all available recipes.""" import os +import html from pathlib import Path +from docutils.core import publish_doctree +from docutils import nodes + + RECIPE_DIR = "recipes" OUT_PATH = os.path.abspath("gallery.rst") @@ -16,72 +21,78 @@ "`_." "\n\n" ) -WIDTH = ":width: 90%" -FIGURE_STR = ".. figure::" -IMAGE_STR = " image:: " -TABLE_SEP = ( - "+---------------------------------------------------" - "+---------------------------------------------------+\n" +MAX_CAPTION_LENGTH = 300 + +START_GALLERY = ( + ".. raw:: html\n\n" + ' \n\n' ) -CELL_WIDTH = 50 - - -def _get_figure_index(file_content): - """Get index of figure in text.""" - offset = 0 - if "gallery=True" in file_content: - offset = file_content.index("gallery=True") + 12 - file_content = file_content[offset:] - if FIGURE_STR in file_content: - return offset + file_content.index(FIGURE_STR) + len(FIGURE_STR) - if IMAGE_STR in file_content: - return offset + file_content.index(IMAGE_STR) + len(IMAGE_STR) - raise ValueError("File does not contain image") - - -def _get_next_row(filenames, file_contents): - """Get next row.""" - figure_idx = [_get_figure_index(content) for content in file_contents] - figure_paths = [ - file_contents[idx][fig_idx:].split("\n")[0].strip() - for (idx, fig_idx) in enumerate(figure_idx) - ] - subst = [f"|{os.path.splitext(filename)[0]}|" for filename in filenames] - link = [file_contents[0].split()[1][1:-1]] - if figure_paths[1] == "": - subst[1] = "" - link.append("") - else: - link.append(file_contents[1].split()[1][1:-1]) - - # Build table - row = "" - refs = "" - row += TABLE_SEP - row += f"| {subst[0].ljust(CELL_WIDTH)}| {subst[1].ljust(CELL_WIDTH)}|\n" - row += EMPTY_TABLE - left_col = "[#]_".ljust(CELL_WIDTH) - if figure_paths[1] == "": - right_col = "".ljust(CELL_WIDTH) - else: - right_col = "[#]_".ljust(CELL_WIDTH) - row += f"| {left_col}| {right_col}|\n" - - # Build refs - for idx, path in enumerate(figure_paths): - if path == "": - continue - refs += f".. {subst[idx]} image:: {path}\n" - refs += f" {WIDTH}\n" - refs += "\n" - refs += f".. [#] :ref:`{link[idx]}`\n" - refs += "\n" - - return (row, refs) + +FIGURE_HTML = ( + '.. figure:: {uri}\n' + ' :width: 90%\n\n' + ' :ref:`{caption} <{link}>`\n\n' +) + + +def _has_gallery_marker(node): + """Wether the node is preceeded by a ``.. gallery`` comment.""" + siblings = list(node.parent) + idx = siblings.index(node) + if idx <= 0: + return False + prev_node = siblings[idx - 1] + if not isinstance(prev_node, nodes.comment): + return False + return prev_node.astext().lower().strip().startswith("gallery") + + +def _is_excluded_from_gallery(node): + """Wether the node has a no-gallery marker.""" + comments = node.traverse(nodes.comment) + for comment in comments: + if comment.astext().lower().strip() == "no-gallery": + return True + return False + + +def _get_figures_from_file(fname): + """Get marked, no or first figure from documentation page.""" + with (Path(RECIPE_DIR)/fname).open() as f: + content = f.read() + tree = publish_doctree(content) + link = content.split("\n")[0].split(" ")[1][1:-1] # get link from first line + if _is_excluded_from_gallery(tree): # ignore files with no-gallery marker + return [] + figures = tree.traverse(nodes.figure) + for fig in figures: + fig["link"] = link # add doc page link to figure + marked_figures = [f for f in figures if _has_gallery_marker(f)] + if len(marked_figures) > 0: # consider all figures with gallery marker + return marked_figures + if len(figures) > 0: # select first figure if nothing is marked + return [figures[0]] + return [] + + +def _get_data_from_figure(figure): + image = figure.traverse(nodes.image)[0] + try: + caption = figure.traverse(nodes.caption)[0].astext().strip() + except IndexError: + caption = "No caption available" + if len(caption) > MAX_CAPTION_LENGTH: + caption = caption[:MAX_CAPTION_LENGTH] + "..." + return { + "uri": html.escape(image["uri"]), + "caption": html.escape(caption.replace("\n", " ")), + "link": html.escape(figure["link"]), + } def _find_recipes(root_dir, exclude_dirs): @@ -95,56 +106,28 @@ def _find_recipes(root_dir, exclude_dirs): ] +def _generate_rst_file(fname, data): + """Generate rst file from data.""" + output = "" + output += HEADER + output += START_GALLERY + for figure_data in data: + output += FIGURE_HTML.format(**figure_data) + output += END_GALLERY + with open(fname, "w") as f: + f.write(output) + print(f"Wrote {fname}") + + def main(): """Generate gallery for recipe plots.""" print(f"Generating gallery at {OUT_PATH}") - left_col = True - table = "" - refs = "" - filenames = [] - file_contents = [] + figures = [] fnames = _find_recipes(RECIPE_DIR, ["legacy", "figures"]) for filename in sorted(fnames): - with open(os.path.join(RECIPE_DIR, filename)) as in_file: - recipe_file = in_file.read() - if FIGURE_STR not in recipe_file and IMAGE_STR not in recipe_file: - print(f"INFO: {filename} does not contain an image, skipping") - continue - if not recipe_file.startswith(".."): - print( - f"INFO: {filename} does not contain reference at top, skipping" - ) - continue - - # Get next row - if left_col: - left_col = False - filenames = [filename] - file_contents = [recipe_file] - continue - else: - left_col = True - filenames.append(filename) - file_contents.append(recipe_file) - new_row = _get_next_row(filenames, file_contents) - table += new_row[0] - refs += new_row[1] - - # Last row - if len(filenames) == 1: - filenames.append("") - file_contents.append(f"{FIGURE_STR}\n") - new_row = _get_next_row(filenames, file_contents) - table += new_row[0] - refs += new_row[1] - table += TABLE_SEP - table += "\n" - - # Write file - whole_file = HEADER + table + refs - with open(OUT_PATH, "w") as out_file: - print(whole_file, file=out_file) - print(f"Wrote {OUT_PATH}") + figures.extend(_get_figures_from_file(filename)) + data = [_get_data_from_figure(f) for f in figures] + _generate_rst_file(OUT_PATH, data) if __name__ == "__main__": From ed6b9954833ba241eccb2ad0654dbaccf70a30a0 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Tue, 27 May 2025 14:10:56 +0200 Subject: [PATCH 05/18] skip recipes without label in first line --- doc/sphinx/source/generate_gallery.py | 28 +++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/sphinx/source/generate_gallery.py b/doc/sphinx/source/generate_gallery.py index 7fe2301e4c..1d9146baee 100644 --- a/doc/sphinx/source/generate_gallery.py +++ b/doc/sphinx/source/generate_gallery.py @@ -1,13 +1,12 @@ #!/usr/bin/env python """Create gallery with all available recipes.""" -import os import html +import os from pathlib import Path -from docutils.core import publish_doctree -from docutils import nodes - +from docutils import nodes +from docutils.core import publish_doctree RECIPE_DIR = "recipes" OUT_PATH = os.path.abspath("gallery.rst") @@ -25,18 +24,15 @@ START_GALLERY = ( ".. raw:: html\n\n" - ' \n\n" FIGURE_HTML = ( - '.. figure:: {uri}\n' - ' :width: 90%\n\n' - ' :ref:`{caption} <{link}>`\n\n' + ".. figure:: {uri}\n :width: 90%\n\n :ref:`{caption} <{link}>`\n\n" ) @@ -63,10 +59,14 @@ def _is_excluded_from_gallery(node): def _get_figures_from_file(fname): """Get marked, no or first figure from documentation page.""" - with (Path(RECIPE_DIR)/fname).open() as f: + with (Path(RECIPE_DIR) / fname).open() as f: content = f.read() tree = publish_doctree(content) - link = content.split("\n")[0].split(" ")[1][1:-1] # get link from first line + try: + link = content.split("\n")[0].split(" ")[1][1:-1] + except IndexError: + print(f"No label found in first line of {fname}. Skipping") + return [] if _is_excluded_from_gallery(tree): # ignore files with no-gallery marker return [] figures = tree.traverse(nodes.figure) From d5a93b04c1a91c993b3e0104b01b1a7aa494cfa1 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Wed, 28 May 2025 11:14:03 +0200 Subject: [PATCH 06/18] improve look and feel, extra css file --- doc/sphinx/source/_static/custom.css | 31 +++++++++++++++++++++++++++ doc/sphinx/source/conf.py | 7 +++++- doc/sphinx/source/generate_gallery.py | 13 ++++------- 3 files changed, 41 insertions(+), 10 deletions(-) create mode 100644 doc/sphinx/source/_static/custom.css diff --git a/doc/sphinx/source/_static/custom.css b/doc/sphinx/source/_static/custom.css new file mode 100644 index 0000000000..5401a023fe --- /dev/null +++ b/doc/sphinx/source/_static/custom.css @@ -0,0 +1,31 @@ +.gallery { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(200px, 250px)); + gap: 1em; +} + +.gallery figure:hover { + background-color: var(--pst-color-surface); +} + +.gallery figure img { + max-height: 300px; + object-fit: cover; + overflow: hidden; +} + +.gallery figure figcaption { + color: var(--pst-color-text-muted); + line-height: 1.2; + text-decoration: none; +} + +.gallery figure figcaption a { + text-decoration: none; + color: var(--pst-color-text); +} + +.gallery figure figcaption a:hover { + text-decoration: underline; + color: var(--pst-color-link-hover); +} diff --git a/doc/sphinx/source/conf.py b/doc/sphinx/source/conf.py index de7feb4775..fb40b0b10f 100644 --- a/doc/sphinx/source/conf.py +++ b/doc/sphinx/source/conf.py @@ -197,7 +197,12 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["figures/ESMValTool-logo-2-dark.png"] +html_static_path = [ + "figures/ESMValTool-logo-2-dark.png", + "_static", +] + +html_css_files = ["custom.css"] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied diff --git a/doc/sphinx/source/generate_gallery.py b/doc/sphinx/source/generate_gallery.py index 1d9146baee..9204058f58 100644 --- a/doc/sphinx/source/generate_gallery.py +++ b/doc/sphinx/source/generate_gallery.py @@ -14,25 +14,20 @@ ".. DO NOT MODIFY! THIS PAGE IS AUTOGENERATED!\n\n" "#######\nGallery\n#######\n\n" "This section shows example plots produced by ESMValTool. For more " - "information, click on the footnote below the image. " + "information, follow the links in the figure captions." "A website displaying results produced with the latest release of " "ESMValTool for all available recipes can be accessed `here " "`_." "\n\n" ) -MAX_CAPTION_LENGTH = 300 +MAX_CAPTION_LENGTH = 100 -START_GALLERY = ( - ".. raw:: html\n\n" - '