Add Fortran Serialisation Framework (#175)

Provide a flexible framework for serializing Fortran execution to improve testing and performance optimizations.

Author: samkellerhals

liskov/src/icon4py/liskov/logger.py common/src/icon4py/common/logger.pyliskov/src/icon4py/liskov/logger.py renamed to common/src/icon4py/common/logger.py

@@ -15,10 +15,18 @@ The icon4py-liskov package includes the `icon_liskov` CLI tool which takes a for
 To use the `icon_liskov` tool, run the following command:
 
 ```bash
-icon_liskov <input_filepath> <output_filepath> [--profile] [--metadatagen]
+icon_liskov <input_filepath> <output_filepath> [--profile] [--metadatagen] [--ppser]
 ```
 
-Where `input_filepath` is the path to the input file to be processed, and `output_filepath` is the path to the output file. The optional `--profile` flag adds nvtx profile statements to the stencils.
+The following are descriptions of the arguments and options:
+
+- input_filepath: path to the input file to be processed.
+- output_filepath: path to the output file.
+- profile flag: adds nvtx profile statements to the stencils (optional).
+- metadatagen flag: generates a metadata header at the top of the file which includes information on icon_liskov such as the version used.
+- ppser flag: activates serialisation mode and will trigger the generation of ppser serialisation statements serialising all variables at the start and end of each stencil directive.
+
+**Note**: By default the data will be saved at the default folder location of the currently run experiment and will have a prefix of `liskov-serialisation`.
 
 ### Preprocessor directives
 

liskov/README.md

@@ -15,8 +15,8 @@
 
 import click
 
-from icon4py.liskov.logger import setup_logger
-from icon4py.liskov.pipeline import (
+from icon4py.common.logger import setup_logger
+from icon4py.liskov.pipeline.collection import (
     load_gt4py_stencils,
     parse_fortran_file,
     run_code_generation,
@@ -26,51 +26,82 @@
 logger = setup_logger(__name__)
 
 
-@click.command("icon_liskov")
+@click.group(invoke_without_command=True)
+@click.pass_context
 @click.argument(
-    "input_filepath",
+    "input_path",
     type=click.Path(
         exists=True, dir_okay=False, resolve_path=True, path_type=pathlib.Path
     ),
 )
 @click.argument(
-    "output_filepath",
+    "output_path",
     type=click.Path(dir_okay=False, resolve_path=True, path_type=pathlib.Path),
 )
+def main(ctx, input_path, output_path):
+    """Command line interface for interacting with the ICON-Liskov DSL Preprocessor.
+
+    Arguments:
+        INPUT_PATH: Path to input file containing Liskov directives.
+        OUTPUT_PATH: Path to new file to be generated.
+    """
+    if ctx.invoked_subcommand is None:
+        click.echo(
+            "Need to choose one of the following commands:\nintegrate\nserialise"
+        )
+    else:
+        ctx.ensure_object(dict)
+        ctx.obj["INPUT"] = input_path
+        ctx.obj["OUTPUT"] = output_path
+
+
+@main.command()
 @click.option(
-    "--profile", "-p", is_flag=True, help="Add nvtx profile statements to stencils."
+    "--profile",
+    "-p",
+    is_flag=True,
+    help="Add nvtx profile statements to integration code.",
 )
 @click.option(
     "--metadatagen",
     "-m",
     is_flag=True,
-    help="Add metadata header with information about program (requires git).",
+    help="Add metadata header with information about program.",
 )
-def main(
-    input_filepath: pathlib.Path,
-    output_filepath: pathlib.Path,
-    profile: bool,
-    metadatagen: bool,
-) -> None:
-    """Command line interface for interacting with the ICON-Liskov DSL Preprocessor.
-
-    Usage:
-        icon_liskov <input_filepath> <output_filepath> [-p] [-m]
+@click.pass_context
+def integrate(ctx, profile, metadatagen):
+    mode = "integration"
+    inp = ctx.obj["INPUT"]
+    out = ctx.obj["OUTPUT"]
 
-    Options:
-        -p --profile Add nvtx profile statements to stencils.
-        -m --metadatagen Add metadata header with information about program (requires git).
-
-    Arguments:
-        input_filepath Path to the input file to process.
-        output_filepath Path to the output file to generate.
-    """
-    parsed = parse_fortran_file(input_filepath, output_filepath)
-    parsed_checked = load_gt4py_stencils(parsed)
+    iface = parse_fortran_file(inp, out, mode)
+    iface_gt4py = load_gt4py_stencils(iface)
     run_code_generation(
-        parsed_checked, input_filepath, output_filepath, profile, metadatagen
+        inp,
+        out,
+        mode,
+        iface_gt4py,
+        profile=profile,
+        metadatagen=metadatagen,
     )
 
 
+@main.command()
+@click.option(
+    "--multinode",
+    is_flag=True,
+    type=bool,
+    help="Specify whether it is a multinode run. Will generate mpi rank information.",
+    default=False,
+)
+@click.pass_context
+def serialise(ctx, multinode):
+    mode = "serialisation"
+    inp = ctx.obj["INPUT"]
+    out = ctx.obj["OUTPUT"]
+    iface = parse_fortran_file(inp, out, mode)
+    run_code_generation(inp, out, mode, iface, multinode=multinode)
+
+
 if __name__ == "__main__":
     main()

liskov/src/icon4py/liskov/cli.py

@@ -11,29 +11,29 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-from dataclasses import dataclass
-from typing import Any, Callable, Optional, Protocol, Type
+from typing import Any, Optional, Protocol, Type
 
+import icon4py.liskov.parsing.parse
 import icon4py.liskov.parsing.types as ts
-from icon4py.liskov.codegen.interface import (
+from icon4py.common.logger import setup_logger
+from icon4py.liskov.codegen.integration.interface import (
     BoundsData,
-    CodeGenInput,
     DeclareData,
-    DeserialisedDirectives,
     EndCreateData,
     EndIfData,
     EndProfileData,
     EndStencilData,
     FieldAssociationData,
     ImportsData,
     InsertData,
+    IntegrationCodeInterface,
     StartCreateData,
     StartProfileData,
     StartStencilData,
     UnusedDirective,
 )
-from icon4py.liskov.common import Step
-from icon4py.liskov.logger import setup_logger
+from icon4py.liskov.codegen.shared.deserialise import Deserialiser
+from icon4py.liskov.codegen.shared.types import CodeGenInput
 from icon4py.liskov.parsing.exceptions import (
     DirectiveSyntaxError,
     MissingBoundsError,
@@ -89,13 +89,11 @@ def __call__(
         ...
 
 
-@dataclass
 class DataFactoryBase:
     directive_cls: Type[ts.ParsedDirective]
     dtype: Type[CodeGenInput]
 
 
-@dataclass
 class OptionalMultiUseDataFactory(DataFactoryBase):
     def __call__(
         self, parsed: ts.ParsedDict, **kwargs: Any
@@ -106,48 +104,38 @@ def __call__(
         else:
             deserialised = []
             for directive in extracted:
-                deserialised.append(
-                    self.dtype(
-                        startln=directive.startln, endln=directive.endln, **kwargs
-                    )
-                )
+                deserialised.append(self.dtype(startln=directive.startln, **kwargs))
             return deserialised
 
 
-@dataclass
 class RequiredSingleUseDataFactory(DataFactoryBase):
     def __call__(self, parsed: ts.ParsedDict) -> CodeGenInput:
         extracted = extract_directive(parsed["directives"], self.directive_cls)[0]
-        return self.dtype(startln=extracted.startln, endln=extracted.endln)
+        return self.dtype(startln=extracted.startln)
 
 
-@dataclass
 class EndCreateDataFactory(RequiredSingleUseDataFactory):
-    directive_cls: Type[ts.ParsedDirective] = ts.EndCreate
+    directive_cls: Type[ts.ParsedDirective] = icon4py.liskov.parsing.parse.EndCreate
     dtype: Type[EndCreateData] = EndCreateData
 
 
-@dataclass
 class ImportsDataFactory(RequiredSingleUseDataFactory):
-    directive_cls: Type[ts.ParsedDirective] = ts.Imports
+    directive_cls: Type[ts.ParsedDirective] = icon4py.liskov.parsing.parse.Imports
     dtype: Type[ImportsData] = ImportsData
 
 
-@dataclass
 class EndIfDataFactory(OptionalMultiUseDataFactory):
-    directive_cls: Type[ts.ParsedDirective] = ts.EndIf
+    directive_cls: Type[ts.ParsedDirective] = icon4py.liskov.parsing.parse.EndIf
     dtype: Type[EndIfData] = EndIfData
 
 
-@dataclass
 class EndProfileDataFactory(OptionalMultiUseDataFactory):
-    directive_cls: Type[ts.ParsedDirective] = ts.EndProfile
+    directive_cls: Type[ts.ParsedDirective] = icon4py.liskov.parsing.parse.EndProfile
     dtype: Type[EndProfileData] = EndProfileData
 
 
-@dataclass
 class StartCreateDataFactory(DataFactoryBase):
-    directive_cls: Type[ts.ParsedDirective] = ts.StartCreate
+    directive_cls: Type[ts.ParsedDirective] = icon4py.liskov.parsing.parse.StartCreate
     dtype: Type[StartCreateData] = StartCreateData
 
     def __call__(self, parsed: ts.ParsedDict) -> StartCreateData:
@@ -159,14 +147,11 @@ def __call__(self, parsed: ts.ParsedDict) -> StartCreateData:
         if named_args:
             extra_fields = named_args["extra_fields"].split(",")
 
-        return self.dtype(
-            startln=directive.startln, endln=directive.endln, extra_fields=extra_fields
-        )
+        return self.dtype(startln=directive.startln, extra_fields=extra_fields)
 
 
-@dataclass
 class DeclareDataFactory(DataFactoryBase):
-    directive_cls: Type[ts.ParsedDirective] = ts.Declare
+    directive_cls: Type[ts.ParsedDirective] = icon4py.liskov.parsing.parse.Declare
     dtype: Type[DeclareData] = DeclareData
 
     @staticmethod
@@ -185,7 +170,6 @@ def __call__(self, parsed: ts.ParsedDict) -> list[DeclareData]:
             deserialised.append(
                 self.dtype(
                     startln=directive.startln,
-                    endln=directive.endln,
                     declarations=named_args,
                     ident_type=ident_type,
                     suffix=suffix,
@@ -194,9 +178,8 @@ def __call__(self, parsed: ts.ParsedDict) -> list[DeclareData]:
         return deserialised
 
 
-@dataclass
 class StartProfileDataFactory(DataFactoryBase):
-    directive_cls: Type[ts.ParsedDirective] = ts.StartProfile
+    directive_cls: Type[ts.ParsedDirective] = icon4py.liskov.parsing.parse.StartProfile
     dtype: Type[StartProfileData] = StartProfileData
 
     def __call__(self, parsed: ts.ParsedDict) -> list[StartProfileData]:
@@ -206,18 +189,13 @@ def __call__(self, parsed: ts.ParsedDict) -> list[StartProfileData]:
             named_args = parsed["content"]["StartProfile"][i]
             stencil_name = _extract_stencil_name(named_args, directive)
             deserialised.append(
-                self.dtype(
-                    name=stencil_name,
-                    startln=directive.startln,
-                    endln=directive.endln,
-                )
+                self.dtype(name=stencil_name, startln=directive.startln)
             )
         return deserialised
 
 
-@dataclass
 class EndStencilDataFactory(DataFactoryBase):
-    directive_cls: Type[ts.ParsedDirective] = ts.EndStencil
+    directive_cls: Type[ts.ParsedDirective] = icon4py.liskov.parsing.parse.EndStencil
     dtype: Type[EndStencilData] = EndStencilData
 
     def __call__(self, parsed: ts.ParsedDict) -> list[EndStencilData]:
@@ -232,17 +210,15 @@ def __call__(self, parsed: ts.ParsedDict) -> list[EndStencilData]:
                 self.dtype(
                     name=stencil_name,
                     startln=directive.startln,
-                    endln=directive.endln,
                     noendif=noendif,
                     noprofile=noprofile,
                 )
             )
         return deserialised
 
 
-@dataclass
 class StartStencilDataFactory(DataFactoryBase):
-    directive_cls: Type[ts.ParsedDirective] = ts.StartStencil
+    directive_cls: Type[ts.ParsedDirective] = icon4py.liskov.parsing.parse.StartStencil
     dtype: Type[StartStencilData] = StartStencilData
 
     def __call__(self, parsed: ts.ParsedDict) -> list[StartStencilData]:
@@ -282,7 +258,6 @@ def __call__(self, parsed: ts.ParsedDict) -> list[StartStencilData]:
                     fields=fields_w_tolerance,
                     bounds=bounds,
                     startln=directive.startln,
-                    endln=directive.endln,
                     acc_present=acc_present,
                     mergecopy=mergecopy,
                     copies=copies,
@@ -377,9 +352,8 @@ def _update_tolerances(
         return fields
 
 
-@dataclass
 class InsertDataFactory(DataFactoryBase):
-    directive_cls: Type[ts.ParsedDirective] = ts.Insert
+    directive_cls: Type[ts.ParsedDirective] = icon4py.liskov.parsing.parse.Insert
     dtype: Type[InsertData] = InsertData
 
     def __call__(self, parsed: ts.ParsedDict) -> list[InsertData]:
@@ -388,15 +362,13 @@ def __call__(self, parsed: ts.ParsedDict) -> list[InsertData]:
         for i, directive in enumerate(extracted):
             content = parsed["content"]["Insert"][i]
             deserialised.append(
-                self.dtype(
-                    startln=directive.startln, endln=directive.endln, content=content  # type: ignore
-                )
+                self.dtype(startln=directive.startln, content=content)  # type: ignore
             )
         return deserialised
 
 
-class DirectiveDeserialiser(Step):
-    _FACTORIES: dict[str, Callable] = {
+class IntegrationCodeDeserialiser(Deserialiser):
+    _FACTORIES = {
         "StartCreate": StartCreateDataFactory(),
         "EndCreate": EndCreateDataFactory(),
         "Imports": ImportsDataFactory(),
@@ -408,27 +380,4 @@ class DirectiveDeserialiser(Step):
         "EndProfile": EndProfileDataFactory(),
         "Insert": InsertDataFactory(),
     }
-
-    def __call__(self, directives: ts.ParsedDict) -> DeserialisedDirectives:
-        """Deserialise the provided parsed directives to a DeserialisedDirectives object.
-
-        Args:
-            directives: The parsed directives to deserialise.
-
-        Returns:
-            A DeserialisedDirectives object containing the deserialised directives.
-
-        Note:
-            The method uses the `_FACTORIES` class attribute to create the appropriate
-            factory object for each directive type, and uses these objects to deserialise
-            the parsed directives. The DeserialisedDirectives class is a dataclass
-            containing the deserialised versions of the different directives.
-        """
-        logger.info("Deserialising directives ...")
-        deserialised = dict()
-
-        for key, func in self._FACTORIES.items():
-            ser = func(directives)
-            deserialised[key] = ser
-
-        return DeserialisedDirectives(**deserialised)
+    _INTERFACE_TYPE = IntegrationCodeInterface