Write index in correct form in finish_website

Non-useful entries are removed from the index before it is written, so
it only contains information the client will need.

Author: jfrost-mo

src/CSET/cset_workflow/app/finish_website/bin/finish_website.py

@@ -29,7 +29,7 @@
 from importlib.metadata import version
 from pathlib import Path
 
-from CSET._common import combine_dicts, sort_dict
+from CSET._common import sort_dict
 
 logging.basicConfig(
     level=os.getenv("LOGLEVEL", "INFO"), format="%(asctime)s %(levelname)s %(message)s"
@@ -61,42 +61,28 @@ def install_website_skeleton(www_root_link: Path, www_content: Path):
     www_root_link.symlink_to(www_content)
 
 
-def construct_index():
-    """Construct the plot index.
-
-    Index should has the form ``{"Category Name": {"recipe_id": "Plot Name"}}``
-    where ``recipe_id`` is the name of the plot's directory.
-    """
-    index = {}
-    plots_dir = Path(os.environ["CYLC_WORKFLOW_SHARE_DIR"]) / "web/plots"
-    # Loop over all diagnostics and append to index.
-    for metadata_file in plots_dir.glob("**/*/meta.json"):
-        try:
-            with open(metadata_file, "rt", encoding="UTF-8") as fp:
-                plot_metadata = json.load(fp)
-
-            category = plot_metadata["category"]
-            case_date = plot_metadata.get("case_date", "")
-            relative_url = str(metadata_file.parent.relative_to(plots_dir))
-
-            record = {
-                category: {
-                    case_date if case_date else "Aggregation": {
-                        relative_url: plot_metadata["title"].strip()
-                    }
-                }
-            }
-        except (json.JSONDecodeError, KeyError, TypeError) as err:
-            logging.error("%s is invalid, skipping.\n%s", metadata_file, err)
-            continue
-        index = combine_dicts(index, record)
-
-    # Sort index of diagnostics.
-    index = sort_dict(index)
-
-    # Write out website index.
-    with open(plots_dir / "index.json", "wt", encoding="UTF-8") as fp:
-        json.dump(index, fp, indent=2)
+def construct_index(www_content: Path):
+    """Construct the plot index."""
+    plots_dir = www_content / "plots"
+    with open(plots_dir / "index.jsonl", "wt", encoding="UTF-8") as index_fp:
+        # Loop over all diagnostics and append to index. The glob is sorted to
+        # ensure a consistent ordering.
+        for metadata_file in sorted(plots_dir.glob("**/*/meta.json")):
+            try:
+                with open(metadata_file, "rt", encoding="UTF-8") as plot_fp:
+                    plot_metadata = json.load(plot_fp)
+                plot_metadata["path"] = str(metadata_file.parent.relative_to(plots_dir))
+                # Remove keys that are not useful for the index.
+                plot_metadata.pop("description", None)
+                plot_metadata.pop("plots", None)
+                # Sort plot metadata.
+                plot_metadata = sort_dict(plot_metadata)
+                # Write metadata into website index.
+                json.dump(plot_metadata, index_fp, separators=(",", ":"))
+                index_fp.write("\n")
+            except (json.JSONDecodeError, KeyError, TypeError) as err:
+                logging.error("%s is invalid, skipping.\n%s", metadata_file, err)
+                continue
 
 
 def bust_cache(www_content: Path):
@@ -149,7 +135,7 @@ def run():
 
     install_website_skeleton(www_root_link, www_content)
     copy_rose_config(www_content)
-    construct_index()
+    construct_index(www_content)
     bust_cache(www_content)
     update_workflow_status(www_content)
 

tests/workflow_utils/test_finish_website.py

@@ -79,9 +79,8 @@ def test_write_workflow_status(tmp_path):
     assert re.search(pattern, content)
 
 
-def test_construct_index(monkeypatch, tmp_path):
+def test_construct_index(tmp_path):
     """Test putting the index together."""
-    monkeypatch.setenv("CYLC_WORKFLOW_SHARE_DIR", str(tmp_path))
     plots_dir = tmp_path / "web/plots"
     plots_dir.mkdir(parents=True)
 
@@ -99,20 +98,22 @@ def test_construct_index(monkeypatch, tmp_path):
     static_resource.touch()
 
     # Construct index.
-    finish_website.construct_index()
+    finish_website.construct_index(plots_dir.parent)
 
     # Check index.
-    index_file = plots_dir / "index.json"
+    index_file = plots_dir / "index.jsonl"
     assert index_file.is_file()
     with open(index_file, "rt", encoding="UTF-8") as fp:
-        index = json.load(fp)
-    expected = {"Category": {"20250101": {"p1": "P1", "p2": "P2"}}}
+        index = fp.read()
+    expected = (
+        '{"case_date":"20250101","category":"Category","path":"p1","title":"P1"}\n'
+        '{"case_date":"20250101","category":"Category","path":"p2","title":"P2"}\n'
+    )
     assert index == expected
 
 
-def test_construct_index_aggregation_case(monkeypatch, tmp_path):
+def test_construct_index_aggregation_case(tmp_path):
     """Construct the index from a diagnostics without a case date."""
-    monkeypatch.setenv("CYLC_WORKFLOW_SHARE_DIR", str(tmp_path))
     plots_dir = tmp_path / "web/plots"
     plots_dir.mkdir(parents=True)
 
@@ -122,20 +123,43 @@ def test_construct_index_aggregation_case(monkeypatch, tmp_path):
     plot1.write_text('{"category": "Category", "title": "P1"}')
 
     # Construct index.
-    finish_website.construct_index()
+    finish_website.construct_index(plots_dir.parent)
 
     # Check index.
-    index_file = plots_dir / "index.json"
+    index_file = plots_dir / "index.jsonl"
     assert index_file.is_file()
     with open(index_file, "rt", encoding="UTF-8") as fp:
         index = json.load(fp)
-    expected = {"Category": {"Aggregation": {"p1": "P1"}}}
+    expected = {"category": "Category", "path": "p1", "title": "P1"}
     assert index == expected
 
 
-def test_construct_index_invalid(monkeypatch, tmp_path, caplog):
+def test_construct_index_remove_keys(tmp_path):
+    """Unneeded keys are removed from the index."""
+    plots_dir = tmp_path / "web/plots"
+    plots_dir.mkdir(parents=True)
+
+    # Plot directories.
+    plot1 = plots_dir / "p1/meta.json"
+    plot1.parent.mkdir()
+    plot1.write_text(
+        '{"category": "Category", "title": "P1", "case_date": "20250101", "plots": ["a.png"], "description": "Foo"}'
+    )
+
+    # Construct index.
+    finish_website.construct_index(plots_dir.parent)
+
+    # Check index.
+    index_file = plots_dir / "index.jsonl"
+    assert index_file.is_file()
+    with open(index_file, "rt", encoding="UTF-8") as fp:
+        index = json.loads(fp.readline())
+    assert "plots" not in index
+    assert "description" not in index
+
+
+def test_construct_index_invalid(tmp_path, caplog):
     """Test constructing index when metadata is invalid."""
-    monkeypatch.setenv("CYLC_WORKFLOW_SHARE_DIR", str(tmp_path))
     plots_dir = tmp_path / "web/plots"
     plots_dir.mkdir(parents=True)
 
@@ -145,19 +169,16 @@ def test_construct_index_invalid(monkeypatch, tmp_path, caplog):
     plot.write_text('"Not JSON!"')
 
     # Construct index.
-    finish_website.construct_index()
+    finish_website.construct_index(plots_dir.parent)
 
     # Check log message.
     _, level, message = caplog.record_tuples[0]
     assert level == logging.ERROR
     assert "p1/meta.json is invalid, skipping." in message
 
-    index_file = plots_dir / "index.json"
+    index_file = plots_dir / "index.jsonl"
     assert index_file.is_file()
-    with open(index_file, "rt", encoding="UTF-8") as fp:
-        index = json.load(fp)
-    expected = {}
-    assert index == expected
+    assert index_file.stat().st_size == 0
 
 
 def test_entrypoint(monkeypatch):
@@ -181,7 +202,7 @@ def increment_counter(*args, **kwargs):
 
     monkeypatch.setattr(finish_website, "install_website_skeleton", check_args)
     monkeypatch.setattr(finish_website, "copy_rose_config", check_single_arg)
-    monkeypatch.setattr(finish_website, "construct_index", increment_counter)
+    monkeypatch.setattr(finish_website, "construct_index", check_single_arg)
     monkeypatch.setattr(finish_website, "bust_cache", check_single_arg)
     monkeypatch.setattr(finish_website, "update_workflow_status", check_single_arg)
     monkeypatch.setenv("WEB_DIR", "/var/www/cset")