Modify NUOPC config parser to use the new Lark-based infrastructure.

micaeljtoliveira · micaeljtoliveira · commit 92b2cce5dab6 · 2025-09-10T17:57:35.000+10:00
diff --git a/src/access/parsers/nuopc_config.py b/src/access/parsers/nuopc_config.py
@@ -1,10 +1,10 @@
 # Copyright 2025 ACCESS-NRI and contributors. See the top-level COPYRIGHT file for details.
 # SPDX-License-Identifier: Apache-2.0
 
-"""Utilities to handle NUOPC configuration files.
+"""Parser for NUOPC configuration files.
 
-The `nuopc.runconfig` files used by the CESM driver, and thus by the NUOPC-based ACCESS models, are a mixture of formats.
- At the top-level, one has the Resource Files as implemented in ESMF. From the ESMF documentation:
+The `nuopc.runconfig` files used by the CESM driver, and thus by the NUOPC-based ACCESS models, are a mixture of
+formats. At the top-level, one has the Resource Files as implemented in ESMF. From the ESMF documentation:
 
     A Resource File (RF) is a text file consisting of list of label-value pairs. There is a limit of 1024 characters per
     line and the Resource File can contain a maximum of 200 records. Each label should be followed by some data, the
@@ -48,7 +48,9 @@
 See https://earthsystemmodeling.org/docs/release/ESMF_8_6_0/ESMF_refdoc/node6.html#SECTION06090000000000000000 for
 further details.
 
-The CESM driver then uses tables as defined in Resource Files to store lists of key-value pairs instead of simple values:
+
+The CESM driver then uses tables as defined in Resource Files to store lists of key-value pairs instead of simple
+values:
 
     DRIVER_attributes::
      Verbosity = off
@@ -67,82 +69,62 @@
      ocn2glc_levels = 1:10:19:26:30:33:35
     ::
 
-This format of key-value pairs does not seem to be documented and, although it resembles Fortran namlists, it is not.
+This format of key-value pairs does not seem to be documented and, although it resembles Fortran namelists, it is not.
 For example, the keys are case-sensitive, which is not the case in Fortran namelists. The format used to store arrays
 of values is also not the same as in Fortran namelists.
 """
 
-from pathlib import Path
-import re
-
-from access.parsers.utils import convert_from_string, convert_to_string
-
-
-def read_nuopc_config(file_name: str) -> dict:
-    """Read a NUOPC config file and return its contents as a dictionary.
-
-    Args:
-        file_name (str): File to read.
-
-    Returns:
-        dict: Contents of file.
-    """
-    fname = Path(file_name)
-    if not fname.is_file():
-        raise FileNotFoundError(f"File not found: {fname.as_posix()}")
-
-    label_value_pattern = re.compile(r"\s*(\w+)\s*:\s*(.+)\s*")
-    table_start_pattern = re.compile(r"\s*(\w+)\s*::\s*")
-    table_end_pattern = re.compile(r"\s*::\s*")
-    assignment_pattern = re.compile(r"\s*(\w+)\s*=\s*(\S+)\s*")
-
-    config = {}
-    with open(fname, "r") as stream:
-        reading_table = False
-        label = None
-        table = None
-        for line in stream:
-            line = re.sub(r"(#).*", "", line)
-            if line.strip():
-                if reading_table:
-                    if re.match(table_end_pattern, line):
-                        config[label] = table
-                        reading_table = False
-                    else:
-                        match = re.match(assignment_pattern, line)
-                        if match:
-                            table[match.group(1)] = convert_from_string(match.group(2))
-                        else:
-                            raise ValueError(
-                                f"Line: {line} in file {file_name} is not a valid NUOPC configuration specification"
-                            )
-
-                elif re.match(table_start_pattern, line):
-                    reading_table = True
-                    match = re.match(label_value_pattern, line)
-                    label = match.group(1)
-                    table = {}
-
-                elif re.match(label_value_pattern, line):
-                    match = re.match(label_value_pattern, line)
-                    config[match.group(1)] = [convert_from_string(string) for string in match.group(2).split()]
-
-    return config
-
-
-def write_nuopc_config(config: dict, file: Path):
-    """Write a dictionary to a NUOPC config file.
-
-    Args:
-        config (dict): NUOPC configuration to write.
-        file (Path): File to write to.
-    """
-    with open(file, "w") as stream:
-        for key, item in config.items():
-            if isinstance(item, dict):
-                stream.write(key + "::\n")
-                for label, value in item.items():
-                    stream.write("  " + label + " = " + convert_to_string(value) + "\n")
-                stream.write("::\n\n")
-            else:
-                stream.write(key + ": " + " ".join(map(convert_to_string, item)) + "\n")
+from access.parsers.config import ConfigParser
+
+
+class NUOPCParser(ConfigParser):
+    """NUOPC config parser."""
+
+    @property
+    def case_sensitive_keys(self) -> bool:
+        return True
+
+    @property
+    def grammar(self) -> str:
+        return """
+?start: lines*
+
+?lines: rfile_key_value
+      | rfile_key_list
+      | rfile_key_block
+      | empty_line
+
+rfile_key_value: ws* key ":" ws* value line_end -> key_value
+rfile_key_list: ws* key ":" ws* value (ws* value)+ line_end -> key_list
+rfile_key_block: ws* key "::" line_end block "::" line_end -> key_block
+
+block: block_line*
+
+?block_line: block_key_value
+           | block_key_list
+           | empty_line
+
+block_key_value : ws* key ws* "=" ws* value line_end -> key_value
+block_key_list : ws* key ws* "=" ws* value (":"value)+ line_end -> key_list
+
+?value: logical
+      | integer
+      | float
+      | double
+      | identifier
+      | path
+
+empty_line: line_end
+line_end: (comment|ws*) NEWLINE
+
+%import config.key
+%import config.logical
+%import config.integer
+%import config.float
+%import config.double
+%import config.identifier
+%import config.path
+%import config.comment
+%import config.ws
+%import config.NEWLINE
+"""
diff --git a/tests/test_nuopc_config.py b/tests/test_nuopc_config.py
@@ -2,19 +2,26 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import pytest
-from unittest.mock import mock_open, patch
+from lark.exceptions import UnexpectedCharacters, UnexpectedEOF
 from pathlib import Path
 
-from access.parsers.nuopc_config import read_nuopc_config, write_nuopc_config
+from access.parsers.nuopc_config import NUOPCParser
 
 
-@pytest.fixture()
-def simple_nuopc_config():
+@pytest.fixture(scope="module")
+def parser():
+    """Fixture instantiating the parser."""
+    return NUOPCParser()
+
+
+@pytest.fixture(scope="module")
+def nuopc_config():
+    """Fixture returning a dict holding the parsed content of a NUOPC config file."""
     return dict(
         DRIVER_attributes={
             "Verbosity": "off",
             "cime_model": "cesm",
-            "logFilePostFix": ".log",
+            "logFilePostFix": Path(".log"),
             "pio_blocksize": -1,
             "pio_rearr_comm_enable_hs_comp2io": True,
             "pio_rearr_comm_enable_hs_io2comp": False,
@@ -23,73 +30,145 @@ def simple_nuopc_config():
             "wv_sat_transition_start": 20.0,
         },
         COMPONENTS=["atm", "ocn"],
+        TEST="On",
         ALLCOMP_attributes={
             "ATM_model": "datm",
             "GLC_model": "sglc",
             "OCN_model": "mom",
-            "ocn2glc_levels": "1:10:19:26:30:33:35",
+            "ocn2glc_levels": [1, 10, 19, 26, 30, 33, 35],
         },
     )
 
 
-@pytest.fixture()
-def simple_nuopc_config_file():
-    return """DRIVER_attributes::
+@pytest.fixture(scope="module")
+def nuopc_config_file():
+    """Fixture returning the content of a NUOPC config file."""
+    return """DRIVER_attributes:: # Comment 1
+
   Verbosity = off
-  cime_model = cesm
+  cime_model = cesm # Comment 2
+
   logFilePostFix = .log
   pio_blocksize = -1
+
+  # Comment 3
+
   pio_rearr_comm_enable_hs_comp2io = .true.
   pio_rearr_comm_enable_hs_io2comp = .false.
   reprosum_diffmax = -1.000000D-08
   wv_sat_table_spacing = 1.000000D+00
   wv_sat_transition_start = 2.000000D+01
 ::
 
-COMPONENTS: atm ocn
+TEST: On
+
+  # Comment 4
+# Comment 5
+
+COMPONENTS: atm ocn   # Comment 6
+
 ALLCOMP_attributes::
+
   ATM_model = datm
   GLC_model = sglc
   OCN_model = mom
   ocn2glc_levels = 1:10:19:26:30:33:35
+
 ::
 
 """
 
 
-@pytest.fixture()
-def invalid_nuopc_config_file():
-    return """DRIVER_attributes::
-  Verbosity: off
-  cime_model - cesm
+@pytest.fixture(scope="module")
+def modified_nuopc_config_file():
+    """Fixture returning the content of the previous NUOPC config file, but with some modifications."""
+    return """DRIVER_attributes:: # Comment 1
+
+  Verbosity = off
+  cime_model = cesm # Comment 2
+
+  logFilePostFix = .log
+  pio_blocksize = -1
+
+  # Comment 3
+
+  pio_rearr_comm_enable_hs_comp2io = .true.
+  pio_rearr_comm_enable_hs_io2comp = .false.
+  reprosum_diffmax = -1.000000D-08
+  wv_sat_table_spacing = 1.000000D+00
+  wv_sat_transition_start = 2.000000D+01
+::
+
+TEST: Off
+
+  # Comment 4
+# Comment 5
+
+COMPONENTS: atm um   # Comment 6
+
+ALLCOMP_attributes::
+
+  ATM_model = um
+  GLC_model = sglc
+  OCN_model = mom
+  ocn2glc_levels = 1:10:19:26:30:33:36
+
 ::
 
-COMPONENTS::: atm ocn
 """
 
 
-@patch("pathlib.Path.is_file", new=lambda file: True)
-def test_read_nuopc_config(simple_nuopc_config, simple_nuopc_config_file):
-    with patch("builtins.open", mock_open(read_data=simple_nuopc_config_file)) as m:
-        config = read_nuopc_config(file_name="simple_nuopc_config_file")
+def test_valid_nuopc_config(parser):
+    """Test the basic grammar constructs"""
+    assert dict(parser.parse("TEST: a")) == {"TEST": "a"}
+    assert dict(parser.parse(" TEST:a")) == {"TEST": "a"}
+    assert dict(parser.parse("TEST: a b")) == {"TEST": ["a", "b"]}
+    assert dict(parser.parse("TEST1: a\n TEST2: b")) == {"TEST1": "a", "TEST2": "b"}
+    assert dict(parser.parse("TEST1: a \nTEST2: b")) == {"TEST1": "a", "TEST2": "b"}
+    assert dict(parser.parse("TEST1: a b \n TEST2: c")) == {"TEST1": ["a", "b"], "TEST2": "c"}
+    assert dict(parser.parse("TEST::\na=1\n::")) == {"TEST": {"a": 1}}
+    assert dict(parser.parse("TEST::\na=1:2:3\n::")) == {"TEST": {"a": [1, 2, 3]}}
+
+
+def test_invalid_nuopc_config(parser):
+    """Test checking that the parser catches malformed expressions"""
+    with pytest.raises(UnexpectedCharacters):
+        parser.parse("TEST::\n cime_model - cesm")
+
+    with pytest.raises(UnexpectedCharacters):
+        parser.parse("TEST:\n cime_model = cesm\n::")
+
+    with pytest.raises(UnexpectedEOF):
+        parser.parse("TEST::\n cime_model = cesm")
+
+    with pytest.raises(UnexpectedCharacters):
+        parser.parse("TEST::\n cime_model = cesm ATM_model = datm")
+
 
-        assert config == simple_nuopc_config
+def test_nuopc_config_parse(parser, nuopc_config, nuopc_config_file):
+    """Test parsing of a file."""
+    config = parser.parse(nuopc_config_file)
+    assert dict(config) == nuopc_config
 
 
-def test_write_nuopc_config(simple_nuopc_config, simple_nuopc_config_file):
-    with patch("builtins.open", mock_open()) as m:
-        write_nuopc_config(simple_nuopc_config, Path("config_file"))
+def test_nuopc_config_roundtrip(parser, nuopc_config_file):
+    """Test round-trip parsing."""
+    config = parser.parse(nuopc_config_file)
 
-        assert simple_nuopc_config_file == "".join(call.args[0] for call in m().write.mock_calls)
+    assert str(config) == nuopc_config_file
 
 
-@patch("pathlib.Path.is_file", new=lambda file: True)
-def test_read_invalid_nuopc_config_file(invalid_nuopc_config_file):
-    with patch("builtins.open", mock_open(read_data=invalid_nuopc_config_file)) as m:
-        with pytest.raises(ValueError):
-            read_nuopc_config(file_name="invalid_nuopc_config_file")
+def test_nuopc_config_roundtrip_with_mutation(parser, nuopc_config_file, modified_nuopc_config_file):
+    """Test round-trip parsing with mutation of the config."""
+    config = parser.parse(nuopc_config_file)
 
+    # Scalar
+    config["TEST"] = "Off"
+    # List
+    config["COMPONENTS"] = ["atm", "um"]
+    # Scalar in table
+    config["ALLCOMP_attributes"]["ATM_model"] = "um"
+    # List in table
+    config["ALLCOMP_attributes"]["ocn2glc_levels"] = [1, 10, 19, 26, 30, 33, 36]
 
-def test_read_missing_nuopc_config_file():
-    with pytest.raises(FileNotFoundError):
-        read_nuopc_config(file_name="garbage")
+    assert str(config) == modified_nuopc_config_file
