Merge pull request #306 from fortran-lang/feat/generic-sources

fix: rewrote the source file extension REGEX

Author: gnikit

CHANGELOG.md

@@ -18,6 +18,9 @@
 
 ### Changed
 
+- Changed `--incl_suffixes` option to faithfully match the suffixes that are
+  provided in the option, without performing any type of modification.
+  ([#300](https://github.com/fortran-lang/fortls/issues/300))
 - Changed the completion signature to include the full Markdown documentation
   for the completion item.
   ([#219](https://github.com/fortran-lang/fortls/issues/219))

docs/options.rst

@@ -102,11 +102,15 @@ incl_suffixes
 .. code-block:: json
 
    {
-      "incl_suffixes": [".h", ".FYP"]
+      "incl_suffixes": [".h", ".FYP", "inc"]
    }
 
 ``fortls`` will parse only files with ``incl_suffixes`` extensions found in
-``source_dirs``. By default ``incl_suffixes`` are defined as
+``source_dirs``. Using the above example, ``fortls`` will match files by the
+``file.h`` and ``file.FYP``, but not ``file.fyp`` or ``filefyp``.
+It will also match ``file.inc`` and ``fileinc`` but not ``file.inc2``.
+
+By default, ``incl_suffixes`` are defined as
 .F .f .F03 .f03 .F05 .f05 .F08 .f08 .F18 .f18 .F77 .f77 .F90 .f90 .F95 .f95 .FOR .for .FPP .fpp.
 Additional source file extensions can be defined in ``incl_suffixes``.
 

fortls/fortls.schema.json

@@ -61,7 +61,7 @@
     },
     "incl_suffixes": {
       "title": "Incl Suffixes",
-      "description": "Consider additional file extensions to the default (default: F,F77,F90,F95,F03,F08,FOR,FPP (lower & upper casing))",
+      "description": "Consider additional file extensions to the default (default: .F, .F77, .F90, .F95, .F03, .F08, .FOR, .FPP (lower & upper casing))",
       "default": [],
       "type": "array",
       "items": {},

fortls/interface.py

@@ -126,7 +126,7 @@ def cli(name: str = "fortls") -> argparse.ArgumentParser:
         metavar="SUFFIXES",
         help=(
             "Consider additional file extensions to the default (default: "
-            "F,F77,F90,F95,F03,F08,FOR,FPP (lower & upper casing))"
+            ".F, .F77, .F90, .F95, .F03, .F08, .FOR, .FPP (lower & upper casing))"
         ),
     )
     group.add_argument(

fortls/langserver.py

@@ -61,7 +61,7 @@
     get_use_tree,
 )
 from fortls.parse_fortran import FortranFile, get_line_context
-from fortls.regex_patterns import src_file_exts
+from fortls.regex_patterns import create_src_file_exts_str
 from fortls.version import __version__
 
 # Global regexes
@@ -89,7 +89,9 @@ def __init__(self, conn, settings: dict):
 
         self.sync_type: int = 2 if self.incremental_sync else 1
         self.post_messages = []
-        self.FORTRAN_SRC_EXT_REGEX: Pattern[str] = src_file_exts(self.incl_suffixes)
+        self.FORTRAN_SRC_EXT_REGEX: Pattern[str] = create_src_file_exts_str(
+            self.incl_suffixes
+        )
         # Intrinsic (re-loaded during initialize)
         (
             self.statements,
@@ -1569,7 +1571,7 @@ def _load_config_file_dirs(self, config_dict: dict) -> None:
         self.source_dirs = set(config_dict.get("source_dirs", self.source_dirs))
         self.incl_suffixes = set(config_dict.get("incl_suffixes", self.incl_suffixes))
         # Update the source file REGEX
-        self.FORTRAN_SRC_EXT_REGEX = src_file_exts(self.incl_suffixes)
+        self.FORTRAN_SRC_EXT_REGEX = create_src_file_exts_str(self.incl_suffixes)
         self.excl_suffixes = set(config_dict.get("excl_suffixes", self.excl_suffixes))
 
     def _load_config_file_general(self, config_dict: dict) -> None:

fortls/regex_patterns.py

@@ -149,30 +149,63 @@ class FortranRegularExpressions:
     OBJBREAK: Pattern = compile(r"[\/\-(.,+*<>=$: ]", I)
 
 
-def src_file_exts(input_exts: list[str] = []) -> Pattern[str]:
-    """Create a REGEX for which file extensions the Language Server should parse
-    Default extensions are
-    F F03 F05 F08 F18 F77 F90 F95 FOR FPP f f03 f05 f08 f18 f77 f90 f95 for fpp
+# TODO: use this in the main code
+def create_src_file_exts_regex(input_exts: list[str] = []) -> Pattern[str]:
+    r"""Create a REGEX for which sources the Language Server should parse.
+
+    Default extensions are (case insensitive):
+    F F03 F05 F08 F18 F77 F90 F95 FOR FPP
 
     Parameters
     ----------
     input_exts : list[str], optional
-        Additional Fortran, by default []
+        Additional list of file extensions to parse, in Python REGEX format
+        that means special characters must be escaped
+        , by default []
+
+    Examples
+    --------
+    >>> regex = create_src_file_exts_regex([r"\.fypp", r"\.inc"])
+    >>> regex.search("test.fypp")
+    <re.Match object; span=(4, 9), match='.fypp'>
+    >>> regex.search("test.inc")
+    <re.Match object; span=(4, 8), match='.inc'>
+
+    >>> regex = create_src_file_exts_regex([r"\.inc.*"])
+    >>> regex.search("test.inc.1")
+    <re.Match object; span=(4, 10), match='.inc.1'>
+
+    Invalid regex expressions will cause the function to revert to the default
+    extensions
+
+    >>> regex = create_src_file_exts_regex(["*.inc"])
+    >>> regex.search("test.inc") is None
+    True
 
     Returns
     -------
     Pattern[str]
-        A compiled regular expression, by default
-        '.(F|F03|F05|F08|F18|F77|F90|F95|FOR|FPP|f|f03|f05|f08|f18|f77|f90|f95|for|fpp)?'
+        A compiled regular expression for matching file extensions
     """
-    EXTS = ["", "77", "90", "95", "03", "05", "08", "18", "OR", "PP"]
-    FORTRAN_FILE_EXTS = []
-    for e in EXTS:
-        FORTRAN_FILE_EXTS.extend([f"F{e}".upper(), f"f{e}".lower()])
-    # Add the custom extensions for the server to parse
-    for e in input_exts:
-        if e.startswith("."):
-            FORTRAN_FILE_EXTS.append(e.replace(".", ""))
-    # Cast into a set to ensure uniqueness of extensions & sort for consistency
-    # Create a regular expression from this
-    return compile(rf"\.({'|'.join(sorted(set(FORTRAN_FILE_EXTS)))})?$")
+    import re
+
+    DEFAULT = r"\.[fF](77|90|95|03|05|08|18|[oO][rR]|[pP]{2})?"
+    EXPRESSIONS = [DEFAULT]
+    try:
+        EXPRESSIONS.extend(input_exts)
+        # Add its expression as an OR and force they match the end of the string
+        return re.compile(rf"(({'$)|('.join(EXPRESSIONS)}$))")
+    except re.error:
+        # TODO: Add a warning to the logger
+        return re.compile(rf"({DEFAULT}$)")
+
+
+def create_src_file_exts_str(input_exts: list[str] = []) -> Pattern[str]:
+    """This is a version of create_src_file_exts_regex that takes a list
+    sanitises the list of input_exts before compiling the regex.
+    For more info see create_src_file_exts_regex
+    """
+    import re
+
+    input_exts = [re.escape(ext) for ext in input_exts]
+    return create_src_file_exts_regex(input_exts)

setup.cfg

@@ -62,7 +62,7 @@ dev =
     black
     isort
     pre-commit
-    pydantic
+    pydantic==1.9.1
 docs =
     sphinx >= 4.0.0
     sphinx-argparse

test/test_regex_patterns.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+import pytest
+
+from fortls.regex_patterns import create_src_file_exts_regex
+
+
+@pytest.mark.parametrize(
+    "input_exts, input_files, matches",
+    [
+        (
+            [],
+            [
+                "test.f",
+                "test.F",
+                "test.f90",
+                "test.F90",
+                "test.f03",
+                "test.F03",
+                "test.f18",
+                "test.F18",
+                "test.f77",
+                "test.F77",
+                "test.f95",
+                "test.F95",
+                "test.for",
+                "test.FOR",
+                "test.fpp",
+                "test.FPP",
+            ],
+            [True] * 16,
+        ),
+        ([], ["test.ff", "test.f901", "test.f90.ff"], [False, False, False]),
+        ([r"\.inc"], ["test.inc", "testinc", "test.inc2"], [True, False, False]),
+        (["inc.*"], ["test.inc", "testinc", "test.inc2"], [True, True, True]),
+    ],
+)
+def test_src_file_exts(
+    input_exts: list[str],
+    input_files: list[str],
+    matches: list[bool],
+):
+    regex = create_src_file_exts_regex(input_exts)
+    results = [bool(regex.search(file)) for file in input_files]
+    assert results == matches