Extracted the code that loads the data from disk to a separate module

Author: GarkGarcia

test/test_general_yaml_sanity.py

@@ -1,10 +1,9 @@
 # -*- coding: utf-8 -*-
 
-from mathics_scanner.generate.build_tables import DEFAULT_DATA_DIR
-import yaml
+from util import yaml_data
 import unicodedata
 
-def check_attr_is_invertible(yaml_data: dict, attr: str):
+def check_attr_is_invertible(attr: str):
     for v in yaml_data.values():
         if attr in v:
             attr_v = v[attr]
@@ -17,12 +16,12 @@ def check_attr_is_invertible(yaml_data: dict, attr: str):
             ), f"{attr_vs} all have the same {attr} field set to {attr_v}"
 
 
-def check_has_attr(yaml_data: dict, attr: str):
+def check_has_attr(attr: str):
     for k, v in yaml_data.items():
         assert attr in v, f"{k} has no {attr} attribute"
 
 
-def check_wl_unicode_name(yaml_data: dict):
+def check_wl_unicode_name():
     for k, v in yaml_data.items():
         wl = v["wl-unicode"]
 
@@ -49,7 +48,7 @@ def check_wl_unicode_name(yaml_data: dict):
         ), f"{k} has wl-unicode-name set to {real_name} but it should be {expected_name}"
 
 
-def check_unicode_name(yaml_data: dict):
+def check_unicode_name():
     for k, v in yaml_data.items():
         # Hack to skip characters that are correct but that doesn't show up in 
         # unicodedata.name
@@ -79,19 +78,16 @@ def check_unicode_name(yaml_data: dict):
 
 
 def test_general_yaml_sanity():
-    with open(DEFAULT_DATA_DIR / "named-characters.yml", "r") as yaml_file:
-        yaml_data = yaml.load(yaml_file, Loader=yaml.FullLoader)
-
-        # Check if required attributes are in place
-        check_has_attr(yaml_data, "wl-unicode")
-        check_has_attr(yaml_data, "is-letter-like")
-        check_has_attr(yaml_data, "has-unicode-inverse")
-
-        # Check if attributes that should be invertible are in fact invertible
-        check_attr_is_invertible(yaml_data, "wl-unicode")
-        check_attr_is_invertible(yaml_data, "esc-alias")
-
-        # Check the consistency of the unicode names in the table
-        check_wl_unicode_name(yaml_data)
-        check_unicode_name(yaml_data)
+    # Check if required attributes are in place
+    check_has_attr("wl-unicode")
+    check_has_attr("is-letter-like")
+    check_has_attr("has-unicode-inverse")
+
+    # Check if attributes that should be invertible are in fact invertible
+    check_attr_is_invertible("wl-unicode")
+    check_attr_is_invertible("esc-alias")
+
+    # Check the consistency of the unicode names in the table
+    check_wl_unicode_name()
+    check_unicode_name()
 

test/test_has_unicode_inverse_sanity.py

@@ -1,14 +1,8 @@
 # -*- coding: utf-8 -*-
 
-from mathics_scanner.generate.build_tables import DEFAULT_DATA_DIR
-import yaml
-import json
-
-def check_has_unicode_inverse_sanity(yaml_data: dict, json_data: dict):
-    """
-    Checks if the "has-unicode-inverse" data is self-consistant
-    """
+from util import yaml_data, json_data
 
+def test_has_unicode_inverse_sanity():
     inverses = set()
 
     for k, v in yaml_data.items():
@@ -32,10 +26,3 @@ def check_has_unicode_inverse_sanity(yaml_data: dict, json_data: dict):
             any(v["wl-unicode"] == wl and v.get("unicode-equivalent") == uni and v["has-unicode-inverse"] for v in yaml_data.values())
         ), f"key {uni} is in unicode-to-wl-dict but there is not corresponding entry in the YAML table"
 
-                
-def test_has_unicode_inverse_sanity():
-    with open(DEFAULT_DATA_DIR / "named-characters.yml", "r") as yaml_file, open(DEFAULT_DATA_DIR / "characters.json", "r") as json_file:
-        yaml_data = yaml.load(yaml_file, Loader=yaml.FullLoader)
-        json_data = json.load(json_file)
-        check_has_unicode_inverse_sanity(yaml_data, json_data)
-

test/test_letterlikes_sanity.py

@@ -1,8 +1,6 @@
-from mathics_scanner.generate.build_tables import DEFAULT_DATA_DIR
-import yaml
-import json
+from util import yaml_data, json_data
 
-def check_letterlikes_sanity(yaml_data: dict, json_data: dict):
+def test_letterlikes_sanity():
     letterlikes = json_data["letterlikes"]
 
     yaml_llc = [v["is-letter-like"] for v in yaml_data.values()].count(True)
@@ -12,10 +10,3 @@ def check_letterlikes_sanity(yaml_data: dict, json_data: dict):
         yaml_llc == json_llc
     ), f"the YAML table has {yaml_llc} letter-like characters but the JSON files lists {json_llc} letterlike characters"
 
-
-def test_letterlikes_sanity():
-    with open(DEFAULT_DATA_DIR / "named-characters.yml", "r") as yaml_file, open(DEFAULT_DATA_DIR / "characters.json", "r") as json_file:
-        yaml_data = yaml.load(yaml_file, Loader=yaml.FullLoader)
-        json_data = yaml.load(json_file)
-        check_letterlikes_sanity(yaml_data, json_data)
-

test/test_roundtrip.py

@@ -1,11 +1,9 @@
-from mathics_scanner.generate.build_tables import DEFAULT_DATA_DIR
 from mathics_scanner.characters import replace_wl_with_plain_text as wl_to_unicode
 from mathics_scanner.characters import replace_unicode_with_wl as unicode_to_wl
-import yaml
-import json
+from util import yaml_data, json_data
 
 
-def check_roundtrip(yaml_data: dict, json_data: dict):
+def test_roundtrip():
     wl_to_unicode_dict = json_data["wl-to-unicode-dict"]
     unicode_to_wl_dict = json_data["unicode-to-wl-dict"]
 
@@ -30,9 +28,3 @@ def check_roundtrip(yaml_data: dict, json_data: dict):
                     unicode_to_wl_dict[uni] == wl
                 ), f"key {k} unicode {uni}, {wl_to_unicode[uni]}"
 
-
-def test_roundtrip():
-    with open(DEFAULT_DATA_DIR / "named-characters.yml", "r") as yaml_file, open(DEFAULT_DATA_DIR / "characters.json", "r") as json_file:
-        yaml_data = yaml.load(yaml_file, Loader=yaml.FullLoader)
-        json_data = json.load(json_file)
-        check_roundtrip(yaml_data, json_data)

test/test_wl_to_ascii.py

@@ -1,17 +1,14 @@
 # -*- coding: utf-8 -*-
 
-from mathics_scanner.generate.build_tables import DEFAULT_DATA_DIR
-from mathics_scanner.characters import replace_wl_with_plain_text
-import yaml
-import json
+from util import yaml_data
 
 def wl_to_ascii(wl_input: str) -> str:
     return replace_wl_with_plain_text(wl_input, use_unicode=False)
 
 def is_ascii(s: str) -> bool:
     return all(ord(c) < 127 for c in s)
 
-def check_wl_to_ascii(yaml_data: dict):
+def test_wl_to_ascii():
     for k, v in yaml_data.items():
         wl = v["wl-unicode"]
 
@@ -28,10 +25,3 @@ def check_wl_to_ascii(yaml_data: dict):
                 uni == ascii_c
             ), f"{k}'s unicode equivalent could be used as it's ASCII equivalent but it isn't"
 
-
-
-def test_wl_to_ascii():
-    with open(DEFAULT_DATA_DIR / "named-characters.yml", "r") as yaml_file:
-        yaml_data = yaml.load(yaml_file, Loader=yaml.FullLoader)
-        check_wl_to_ascii(yaml_data)
-

test/util.py

@@ -0,0 +1,9 @@
+from mathics_scanner.generate.build_tables import DEFAULT_DATA_DIR
+import yaml
+import json
+
+with open(DEFAULT_DATA_DIR / "named-characters.yml", "r") as yaml_file:
+    yaml_data = yaml.load(yaml_file, Loader=yaml.FullLoader)
+
+with open(DEFAULT_DATA_DIR / "characters.json", "r") as json_file:
+    json_data = json.load(json_file)