chore: Add yaml formatter and format yaml files in tests/ (#2577)

Author: aahung

Makefile

@@ -20,10 +20,12 @@ integ-test:
 black:
 	black setup.py samtranslator/* tests/* integration/* bin/*.py
 	bin/json-format.py --write tests
+	bin/yaml-format.py --write tests
 
 black-check:
 	black --check setup.py samtranslator/* tests/* integration/* bin/*.py
 	bin/json-format.py --check tests
+	bin/yaml-format.py --check tests
 
 lint:
 	# mypy performs type check

bin/__init__.py

@@ -0,0 +1,121 @@
+"""Formatter base class for JSONFormatter and YamlFormatter."""
+import argparse
+import os
+import sys
+from abc import ABC, abstractmethod
+from typing import Type
+
+
+class FileFormatter(ABC):
+    check: bool
+    write: bool
+
+    scanned_file_found: int
+    unformatted_file_count: int
+
+    def __init__(self, check: bool, write: bool) -> None:
+        self.check = check
+        self.write = write
+
+        self.scanned_file_found = 0
+        self.unformatted_file_count = 0
+
+    @staticmethod
+    @abstractmethod
+    def description() -> str:
+        """Return the description of the formatter."""
+        return "JSON file formatter"
+
+    @staticmethod
+    @abstractmethod
+    def format(input_str: str) -> str:
+        """Format method to formatted file content."""
+
+    @staticmethod
+    @abstractmethod
+    def decode_exception() -> Type[Exception]:
+        """Return the exception class when the file content cannot be decoded."""
+
+    @staticmethod
+    @abstractmethod
+    def file_extension() -> str:
+        """Return file extension of files to format."""
+
+    def process_file(self, file_path: str) -> None:
+        with open(file_path, "r", encoding="utf-8") as f:
+            file_str = f.read()
+            try:
+                formatted_file_str = self.format(file_str)
+            except self.decode_exception() as error:
+                raise ValueError(f"{file_path}: Cannot decode the file content") from error
+        if file_str != formatted_file_str:
+            if self.write:
+                with open(file_path, "w", encoding="utf-8") as f:
+                    f.write(formatted_file_str)
+                print(f"reformatted {file_path}")
+            if self.check:
+                print(f"would reformat {file_path}")
+            self.unformatted_file_count += 1
+        self.scanned_file_found += 1
+
+    def process_directory(self, directory_path: str) -> None:
+        for root, dirs, files in os.walk(directory_path):
+            for file in files:
+                file_path = os.path.join(root, file)
+                _, extension = os.path.splitext(file_path)
+                if extension != self.file_extension():
+                    continue
+                self.process_file(file_path)
+
+    def output_summary(self) -> None:
+        print(f"{self.scanned_file_found} file(s) scanned.")
+        if self.write:
+            print(f"{self.unformatted_file_count} file(s) reformatted.")
+        if self.check:
+            print(f"{self.unformatted_file_count} file(s) need reformat.")
+            if self.unformatted_file_count:
+                sys.exit(-1)
+        print("\033[1mAll done! ✨ 🍰 ✨\033[0m")  # using bold font
+
+    @classmethod
+    def main(cls) -> None:
+        parser = argparse.ArgumentParser(description=cls.description())
+        parser.add_argument(
+            "paths",
+            metavar="file|dir",
+            type=str,
+            nargs="+",
+            help="file to format or directory containing files to format",
+        )
+        group = parser.add_mutually_exclusive_group()
+        group.add_argument(
+            "-c",
+            "--check",
+            action="store_true",
+            help="Check if the given files are formatted, "
+            "print a human-friendly summary message and paths to un-formatted files",
+        )
+        group.add_argument(
+            "-w",
+            "--write",
+            action="store_true",
+            help="Edit files in-place. (Beware!)",
+        )
+
+        args = parser.parse_args()
+        formatter = cls(args.check, args.write)
+
+        for path in args.paths:
+            if not os.path.exists(path):
+                raise ValueError(f"{path}: No such file or directory")
+            if os.path.isfile(path):
+                _, extension = os.path.splitext(path)
+                if extension != cls.file_extension():
+                    raise ValueError(f"{path}: Not a format-able file")
+                formatter.process_file(path)
+            elif os.path.isdir(path):
+                formatter.process_directory(path)
+            else:
+                raise ValueError(f"{path}: Unsupported path")
+
+        formatter.output_summary()

bin/_file_formatter.py

@@ -1,105 +1,36 @@
 #!/usr/bin/env python
 """JSON file formatter (without prettier)."""
-import argparse
-import json
-import os.path
+import os
 import sys
 
+my_path = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, my_path + "/..")
 
-def format_json(json_str: str) -> str:
-    """Opinionated format JSON file."""
-    obj = json.loads(json_str)
-    return json.dumps(obj, indent=2, sort_keys=True) + "\n"
-
-
-class JSONFormatter:
-    check: bool
-    write: bool
-
-    scanned_file_found: int
-    unformatted_file_count: int
-
-    def __init__(self, check: bool, write: bool) -> None:
-        self.check = check
-        self.write = write
-
-        self.scanned_file_found = 0
-        self.unformatted_file_count = 0
-
-    def process_file(self, file_path: str) -> None:
-        with open(file_path, "r", encoding="utf-8") as f:
-            json_str = f.read()
-            try:
-                formatted_json_str = format_json(json_str)
-            except json.JSONDecodeError as error:
-                raise ValueError(f"{file_path}: Invalid JSON") from error
-        if json_str != formatted_json_str:
-            if self.write:
-                with open(file_path, "w", encoding="utf-8") as f:
-                    f.write(formatted_json_str)
-                print(f"reformatted {file_path}")
-            if self.check:
-                print(f"would reformat {file_path}")
-            self.unformatted_file_count += 1
-        self.scanned_file_found += 1
-
-    def process_directory(self, directory_path: str) -> None:
-        for root, dirs, files in os.walk(directory_path):
-            for file in files:
-                file_path = os.path.join(root, file)
-                _, extension = os.path.splitext(file_path)
-                if extension != ".json":
-                    continue
-                self.process_file(file_path)
+import json
+from typing import Type
 
-    def output_summary(self):  # type: ignore[no-untyped-def]
-        print(f"{self.scanned_file_found} file(s) scanned.")
-        if self.write:
-            print(f"{self.unformatted_file_count} file(s) reformatted.")
-        if self.check:
-            print(f"{self.unformatted_file_count} file(s) need reformat.")
-            if self.unformatted_file_count:
-                sys.exit(-1)
+from bin._file_formatter import FileFormatter
 
 
-def main() -> None:
-    parser = argparse.ArgumentParser(description="JSON file formatter.")
-    parser.add_argument(
-        "paths", metavar="file|dir", type=str, nargs="+", help="JSON file or directory containing JSON files"
-    )
-    group = parser.add_mutually_exclusive_group()
-    group.add_argument(
-        "-c",
-        "--check",
-        action="store_true",
-        help="Check if the given files are formatted, "
-        "print a human-friendly summary message and paths to un-formatted files",
-    )
-    group.add_argument(
-        "-w",
-        "--write",
-        action="store_true",
-        help="Edit files in-place. (Beware!)",
-    )
+class JSONFormatter(FileFormatter):
+    @staticmethod
+    def description() -> str:
+        return "JSON file formatter"
 
-    args = parser.parse_args()
-    formatter = JSONFormatter(args.check, args.write)
+    @staticmethod
+    def format(input_str: str) -> str:
+        """Opinionated format JSON file."""
+        obj = json.loads(input_str)
+        return json.dumps(obj, indent=2, sort_keys=True) + "\n"
 
-    for path in args.paths:
-        if not os.path.exists(path):
-            raise ValueError(f"{path}: No such file or directory")
-        if os.path.isfile(path):
-            _, extension = os.path.splitext(path)
-            if extension != ".json":
-                raise ValueError(f"{path}: Not a JSON file")
-            formatter.process_file(path)
-        elif os.path.isdir(path):
-            formatter.process_directory(path)
-        else:
-            raise ValueError(f"{path}: Unsupported path")
+    @staticmethod
+    def decode_exception() -> Type[Exception]:
+        return json.JSONDecodeError
 
-    formatter.output_summary()  # type: ignore[no-untyped-call]
+    @staticmethod
+    def file_extension() -> str:
+        return ".json"
 
 
 if __name__ == "__main__":
-    main()
+    JSONFormatter.main()

bin/json-format.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+"""JSON file formatter (without prettier)."""
+import os
+import sys
+
+my_path = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, my_path + "/..")
+
+import re
+from io import StringIO
+from typing import Type
+
+# We use ruamel.yaml for parsing yaml files because it can preserve comments
+from ruamel.yaml import YAML
+from ruamel.yaml.error import YAMLError
+
+from bin._file_formatter import FileFormatter
+
+yaml = YAML()
+
+
+class YAMLFormatter(FileFormatter):
+    @staticmethod
+    def description() -> str:
+        return "YAML file formatter"
+
+    @staticmethod
+    def format(input_str: str) -> str:
+        """Opinionated format YAML file."""
+        obj = yaml.load(input_str)
+        out_stream = StringIO()
+        yaml.dump(
+            obj,
+            stream=out_stream,
+        )
+        # ruamel.yaml tends to add 2 empty lines at the bottom of the dump
+        return re.sub(r"\n+$", "\n", out_stream.getvalue())
+
+    @staticmethod
+    def decode_exception() -> Type[Exception]:
+        return YAMLError
+
+    @staticmethod
+    def file_extension() -> str:
+        return ".yaml"
+
+
+if __name__ == "__main__":
+    YAMLFormatter.main()

bin/yaml-format.py

@@ -26,6 +26,7 @@ docopt~=0.6.2
 
 # formatter
 black==20.8b1
+ruamel.yaml==0.17.21  # It can parse yaml while perserving comments
 
 # type check
 mypy==0.971

requirements/dev.txt

@@ -16,4 +16,3 @@ Resources:
       MemorySize: 1024
       Runtime: nodejs12.x
       Timeout: 3
-