Remove all usages of PyYAML, exclusively use ruamel.yaml

Author: mpsijm

bapctools/cli.py

@@ -55,7 +55,6 @@
     error,
     fatal,
     glob,
-    has_ryaml,
     inc_label,
     is_problem_directory,
     is_relative_to,
@@ -228,17 +227,14 @@ def key(self) -> tuple[int, int]:
                 verbose(f"order: {', '.join(map(lambda p: str(p.label), problems))}")
 
                 if ask_variable_bool("Update order in contest.yaml"):
-                    if has_ryaml:
-                        contest_yaml_path = Path("contest.yaml")
-                        data = read_yaml(contest_yaml_path) or {}
-                        if not isinstance(data, dict):
-                            error("could not parse contest.yaml.")
-                        else:
-                            data["order"] = "".join(p.label or p.name for p in problems)
-                            write_yaml(data, contest_yaml_path)
-                            log("Updated order")
+                    contest_yaml_path = Path("contest.yaml")
+                    data = read_yaml(contest_yaml_path) or {}
+                    if not isinstance(data, dict):
+                        error("could not parse contest.yaml.")
                     else:
-                        error("ruamel.yaml library not found. Update the order manually.")
+                        data["order"] = "".join(p.label or p.name for p in problems)
+                        write_yaml(data, contest_yaml_path)
+                        log("Updated order")
 
     # Filter problems by submissions/testcases, if given.
     if config.level == "problemset" and (config.args.submissions or config.args.testcases):

bapctools/contest.py

@@ -8,7 +8,6 @@
 from bapctools.util import (
     error,
     fatal,
-    has_ryaml,
     log,
     read_yaml,
     verbose,
@@ -48,13 +47,6 @@ def __init__(self, yaml_data: Optional[dict[object, object]]) -> None:
     def dict(self) -> dict[str, object]:
         data = {k: v for k, v in vars(self).items() if not k.startswith("_") and v is not None}
         data.update(self._yaml)
-        if not has_ryaml:
-            for key in ("duration", "scoreboard_freeze_duration"):
-                if key in data:
-                    # YAML 1.1 parses 1:00:00 as 3600. Convert it back to a string if so.
-                    # (YAML 1.2 and ruamel.yaml parse it as a string.)
-                    if isinstance(data[key], int):
-                        data[key] = str(datetime.timedelta(seconds=data[key]))
         return data
 
 

bapctools/export.py

@@ -23,14 +23,12 @@
     error,
     fatal,
     glob,
-    has_ryaml,
     has_substitute,
     inc_label,
     log,
     normalize_yaml_value,
     PrintBar,
     read_yaml,
-    require_ruamel,
     ryaml_filter,
     substitute,
     verbose,
@@ -136,7 +134,6 @@ def build_samples_zip(problems: list[Problem], output: Path, languages: list[str
     bar.log("done")
 
 
-@require_ruamel("zip", False)
 def build_problem_zip(problem: Problem, output: Path) -> bool:
     """Make DOMjudge/Kattis ZIP file for specified problem."""
 
@@ -430,7 +427,6 @@ def add_testcase(in_file: Path) -> None:
 # solutions*.{lang}.pdf
 # problem-slides*.{lang}.pdf
 # Output is <outfile>
-@require_ruamel("zip", None)
 def build_contest_zip(
     problems: list[Problem], zipfiles: list[Path], outfile: str, languages: list[str]
 ) -> None:
@@ -481,15 +477,12 @@ def add_file(file: Path) -> None:
 
 
 def update_contest_id(cid: str) -> None:
-    if has_ryaml:
-        contest_yaml_path = Path("contest.yaml")
-        data = read_yaml(contest_yaml_path)
-        assert isinstance(data, dict)
-        data["contest_id"] = cid
-        write_yaml(data, contest_yaml_path)
-        log(f"Updated contest_id to {cid}")
-    else:
-        error(f"ruamel.yaml library not found. Update the id manually to {cid}.")
+    contest_yaml_path = Path("contest.yaml")
+    data = read_yaml(contest_yaml_path)
+    assert isinstance(data, dict)
+    data["contest_id"] = cid
+    write_yaml(data, contest_yaml_path)
+    log(f"Updated contest_id to {cid}")
 
 
 def export_contest(cid: Optional[str]) -> str:
@@ -531,14 +524,11 @@ def export_contest(cid: Optional[str]) -> str:
     return new_cid
 
 
+# Update name and time limit values.
 def update_problems_yaml(problems: list[Problem], colors: Optional[list[str]] = None) -> None:
-    # Update name and time limit values.
-    if not has_ryaml:
-        log(
-            "ruamel.yaml library not found. Make sure to update the name and time limit fields manually."
-        )
-        return
-
+    # Make sure problems.yaml is formatted correctly.
+    # We cannot use the resulting `list[ProblemsYamlEntry]`, because we need to edit them.
+    # TODO #102 Perhaps there's a way that ProblemsYamlEntry can also be a ruamel.yaml CommentedMap?
     problems_yaml()
 
     log("Updating problems.yaml")

bapctools/fuzz.py

@@ -7,6 +7,7 @@
 from typing import Any, Optional
 
 from colorama import Style
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
 
 from bapctools import config, generate, parallel, problem
 from bapctools.run import Run, Submission
@@ -15,20 +16,15 @@
     eprint,
     error,
     fatal,
-    has_ryaml,
     PrintBar,
     ProgressBar,
     read_yaml,
-    require_ruamel,
     ryaml_get_or_add,
     write_yaml,
 )
 from bapctools.validate import Mode, OutputValidator
 from bapctools.verdicts import Verdict
 
-if has_ryaml:
-    from ruamel.yaml.comments import CommentedMap, CommentedSeq
-
 # STEPS:
 # 1. Find generator invocations depending on {seed}.
 # 2. Generate a testcase + .ans using the rule using a random seed.
@@ -247,7 +243,6 @@ def add_testcase(t: generate.TestcaseRule) -> None:
         # SUBMISSIONS
         self.submissions = self.problem.selected_or_accepted_submissions()
 
-    @require_ruamel("Fuzz", False)
     def run(self) -> bool:
         if len(self.testcase_rules) == 0:
             error("No invocations depending on {seed} found.")

bapctools/generate.py

@@ -11,6 +11,7 @@
 from typing import cast, Final, Literal, Optional, overload, TypeVar
 
 from colorama import Fore, Style
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
 
 from bapctools import config, parallel, program, run, validate, visualize
 from bapctools.problem import Problem
@@ -26,7 +27,6 @@
     fatal,
     get_basedirs,
     glob,
-    has_ryaml,
     hash_file_content,
     hash_string,
     is_relative_to,
@@ -35,7 +35,6 @@
     PrintBar,
     ProgressBar,
     read_yaml,
-    require_ruamel,
     ryaml_get_or_add,
     shorten_path,
     substitute,
@@ -44,10 +43,6 @@
 )
 from bapctools.verdicts import Verdict
 
-if has_ryaml:
-    import ruamel.yaml
-
-
 YAML_TYPE = Optional[str | dict[object, object]]
 
 INCLUSIVE_RANGE_REGEX = re.compile(r"^(-?\d+)\.\.=(-?\d+)$")
@@ -721,7 +716,7 @@ def get(key: str, default: T) -> T:
 
         def write(self) -> None:
             data = {k: v for k, v in vars(self).items() if not k.startswith("_")}
-            write_yaml(data, self._path, allow_yamllib=True)
+            write_yaml(data, self._path)
 
     def link(
         t,
@@ -2244,7 +2239,6 @@ def update_gitignore_file(self) -> None:
 
     # add all testcases specified as copy keys in the generators.yaml
     # can handle files and complete directories
-    @require_ruamel("generate --upgrade", False)
     def add(self, to_add: Sequence[Path]) -> bool:
         if self.n_parse_error > 0:
             return False
@@ -2268,12 +2262,12 @@ def add(self, to_add: Sequence[Path]) -> bool:
         generators_yaml = self.problem.path / "generators" / "generators.yaml"
         data = read_yaml(generators_yaml)
         if data is None:
-            data = ruamel.yaml.comments.CommentedMap()
-        assert isinstance(data, ruamel.yaml.comments.CommentedMap)
+            data = CommentedMap()
+        assert isinstance(data, CommentedMap)
 
         parent = ryaml_get_or_add(data, "data")
         parent = ryaml_get_or_add(parent, "secret")
-        entry = ryaml_get_or_add(parent, "data", ruamel.yaml.comments.CommentedSeq)
+        entry = ryaml_get_or_add(parent, "data", CommentedSeq)
 
         bar = ProgressBar("Adding", items=in_files)
         for in_file in sorted(in_files, key=lambda x: x.name):
@@ -2283,12 +2277,10 @@ def add(self, to_add: Sequence[Path]) -> bool:
             elif in_file in known:
                 bar.log("already found in generators.yaml. Skipping.")
             else:
-                entry.append(ruamel.yaml.comments.CommentedMap())
+                entry.append(CommentedMap())
                 path_in_gen = in_file.relative_to("generators")
                 name = path_in_gen.with_suffix("").as_posix().replace("/", "_")
-                new = ruamel.yaml.comments.CommentedMap(
-                    {"copy": path_in_gen.with_suffix("").as_posix()}
-                )
+                new = CommentedMap({"copy": path_in_gen.with_suffix("").as_posix()})
                 new.fa.set_flow_style()
                 entry[-1][str(name)] = new
                 bar.log("added to generators.yaml.")
@@ -2302,7 +2294,6 @@ def add(self, to_add: Sequence[Path]) -> bool:
         return True
 
     # reorder all testcases in the given directories
-    @require_ruamel("generate --reorder", False)
     def reorder(self) -> bool:
         if self.n_parse_error > 0:
             return False

bapctools/problem.py

@@ -8,6 +8,8 @@
 from typing import Final, Literal, Optional, overload, TYPE_CHECKING
 
 from colorama import Fore, Style
+from ruamel.yaml.comments import CommentedMap
+from ruamel.yaml.scanner import ScannerError
 
 from bapctools import (
     check_testing_tool,
@@ -30,7 +32,6 @@
     fatal,
     generate_problem_uuid,
     glob,
-    has_ryaml,
     hash_file_content,
     is_relative_to,
     is_uuid,
@@ -50,9 +51,6 @@
 if TYPE_CHECKING:  # Prevent circular import: https://stackoverflow.com/a/39757388
     from bapctools.program import Program
 
-if has_ryaml:
-    import ruamel.yaml
-
 
 class Person:
     def __init__(self, yaml_data: str | dict[object, object], parent_path: str):
@@ -517,14 +515,10 @@ def _determine_statement_languages(self) -> list[str]:
     def _read_settings(self) -> None:
         # parse problem.yaml
         yaml_path = self.path / "problem.yaml"
-        if has_ryaml:
-            try:
-                yaml_data = read_yaml(yaml_path)
-            except ruamel.yaml.scanner.ScannerError:
-                fatal(f"Make sure {self.name}/problem.yaml does not contain any more {{% ... %}}.")
-        else:
-            yaml_data = read_yaml(yaml_path)
-        yaml_data = yaml_data or {}
+        try:
+            yaml_data = read_yaml(yaml_path) or {}
+        except ScannerError:
+            fatal(f"Make sure {self.name}/problem.yaml does not contain any more {{% ... %}}.")
 
         if not isinstance(yaml_data, dict):
             fatal(f"{self.name}/problem.yaml is illformed.")
@@ -1599,19 +1593,16 @@ def get_slowest(result: verdicts.Verdicts) -> tuple[str, float]:
         eprint()
 
         if config.args.write:
-            if not has_ryaml:
-                warn("ruamel.yaml library not found. Please update the time limit fields manually.")
+            yaml_path = problem.path / "problem.yaml"
+            problem_yaml = read_yaml(yaml_path)
+            if problem_yaml is None:
+                problem_yaml = CommentedMap()
+            if not isinstance(problem_yaml, CommentedMap):
+                warn("could not parse problem.yaml")
             else:
-                yaml_path = problem.path / "problem.yaml"
-                problem_yaml = read_yaml(yaml_path)
-                if problem_yaml is None:
-                    problem_yaml = ruamel.yaml.comments.CommentedMap()
-                if not isinstance(problem_yaml, ruamel.yaml.comments.CommentedMap):
-                    warn("could not parse problem.yaml")
-                else:
-                    limits = ryaml_get_or_add(problem_yaml, "limits")
-                    limits["time_limit"] = problem.limits.time_limit
-                    write_yaml(problem_yaml, problem.path / "problem.yaml")
+                limits = ryaml_get_or_add(problem_yaml, "limits")
+                limits["time_limit"] = problem.limits.time_limit
+                write_yaml(problem_yaml, problem.path / "problem.yaml")
 
         submission, testcase, duration = run_all(
             lambda vs: vs == [verdicts.Verdict.TIME_LIMIT_EXCEEDED], min

bapctools/program.py

@@ -480,11 +480,7 @@ def _compile(self, bar: ProgressBar) -> bool:
             bar.error("Failed", data)
             return False
 
-        write_yaml(
-            {"hash": self.hash, "command": self.compile_command},
-            meta_path,
-            allow_yamllib=True,
-        )
+        write_yaml({"hash": self.hash, "command": self.compile_command}, meta_path)
         return True
 
     # Return True on success, False on failure.