reverse generated-python direction

Make binderspawner_mixin.py canonical, instead of python-in-yaml

easier to edit with editors, linters, etc.

removes use of redundant pyyaml

Author: minrk

binderhub/binderspawner_mixin.py

@@ -2,12 +2,12 @@
 Helpers for creating BinderSpawners
 
 FIXME:
-This file is defined in helm-chart/binderhub/values.yaml and is copied to
-binderhub/binderspawner_mixin.py by setup.py so that it can be used by other JupyterHub
-container spawners.
+This file is defined in binderhub/binderspawner_mixin.py
+and is copied to helm-chart/binderhub/values.yaml
+by ci/check_embedded_chart_code.py
 
-The BinderHub repo is just used as the distribution mechanism for this spawner, BinderHub
-itself doesn't require this code.
+The BinderHub repo is just used as the distribution mechanism for this spawner,
+BinderHub itself doesn't require this code.
 
 Longer term options include:
 - Move BinderSpawnerMixin to a separate Python package and include it in the Z2JH Hub

ci/check_embedded_chart_code.py

@@ -11,7 +11,11 @@
 import difflib
 import os
 import sys
-import yaml
+from ruamel.yaml import YAML
+
+yaml = YAML(typ="rt")
+yaml.preserve_quotes = True
+yaml.indent(mapping=2, sequence=4, offset=2)
 
 parser = argparse.ArgumentParser(description='Check embedded chart code')
 parser.add_argument('--update', action='store_true', help='Update binderhub code from values.yaml')
@@ -21,21 +25,28 @@
 binderspawner_mixin_py = os.path.join(root, 'binderhub', 'binderspawner_mixin.py')
 values_yaml = os.path.join(root, 'helm-chart', 'binderhub', 'values.yaml')
 
-with open(values_yaml) as f:
-    values = yaml.safe_load(f)
-    values_code = values['jupyterhub']['hub']['extraConfig']['0-binderspawnermixin'].splitlines()
+with open(binderspawner_mixin_py, 'r') as f:
+    py_code = f.read()
+
 
 if args.update:
-    with open(binderspawner_mixin_py, 'w') as f:
-        f.write(values['jupyterhub']['hub']['extraConfig']['0-binderspawnermixin'])
+    with open(values_yaml) as f:
+        values = yaml.load(f)
+    values_code = values['jupyterhub']['hub']['extraConfig']['0-binderspawnermixin']
+    if values_code != py_code:
+        print(f"Generating {values_yaml} from {binderspawner_mixin_py}")
+        values['jupyterhub']['hub']['extraConfig']['0-binderspawnermixin'] = py_code
+        with open(values_yaml, "w") as f:
+            yaml.dump(values, f)
 else:
-    with open(binderspawner_mixin_py, 'r') as f:
-        py_code = f.read().splitlines()
+    with open(values_yaml) as f:
+        values = yaml.load(f)
+    values_code = values['jupyterhub']['hub']['extraConfig']['0-binderspawnermixin']
 
-    difflines = list(difflib.context_diff(values_code, py_code))
+    difflines = list(difflib.context_diff(values_code.splitlines(), py_code.splitlines()))
     if difflines:
         print('\n'.join(difflines))
         print('\n')
         print('Values code is not in sync with binderhub/binderspawner_mixin.py')
-        print('Run `python {} --update` to update binderhub/binderspawner_mixin.py from values.yaml'.format(sys.argv[0]))
+        print('Run `python {} --update` to update values.yaml from binderhub/binderspawner_mixin.py'.format(sys.argv[0]))
         sys.exit(1)

dev-requirements.txt

@@ -10,10 +10,6 @@ pycurl
 pytest
 pytest-asyncio
 pytest-cov
-# pyyaml is used by the scripts in tools/ and could absolutely be replaced by
-# using ruamel.yaml. ruamel.yaml is more powerful than pyyaml, and that is
-# relevant when writing YAML files back after having loaded them from disk.
-pyyaml
 jupyter-repo2docker>=2021.08.0
 requests
 ruamel.yaml>=0.15

helm-chart/binderhub/files/binderhub_config.py

@@ -2,7 +2,9 @@
 import os
 from functools import lru_cache
 from urllib.parse import urlparse
-import yaml
+from ruamel.yaml import YAML
+
+yaml = YAML(typ="safe")
 
 
 def _merge_dictionaries(a, b):
@@ -34,7 +36,7 @@ def _load_values():
         if os.path.exists(path):
             print(f"Loading {path}")
             with open(path) as f:
-                values = yaml.safe_load(f)
+                values = yaml.load(f)
             cfg = _merge_dictionaries(cfg, values)
         else:
             print(f"No config at {path}")

helm-chart/binderhub/values.yaml

@@ -68,12 +68,12 @@ jupyterhub:
         Helpers for creating BinderSpawners
 
         FIXME:
-        This file is defined in helm-chart/binderhub/values.yaml and is copied to
-        binderhub/binderspawner_mixin.py by setup.py so that it can be used by other JupyterHub
-        container spawners.
+        This file is defined in binderhub/binderspawner_mixin.py
+        and is copied to helm-chart/binderhub/values.yaml
+        by ci/check_embedded_chart_code.py
 
-        The BinderHub repo is just used as the distribution mechanism for this spawner, BinderHub
-        itself doesn't require this code.
+        The BinderHub repo is just used as the distribution mechanism for this spawner,
+        BinderHub itself doesn't require this code.
 
         Longer term options include:
         - Move BinderSpawnerMixin to a separate Python package and include it in the Z2JH Hub

tools/generate-json-schema.py

@@ -14,7 +14,8 @@
 
 from collections.abc import MutableMapping
 
-import yaml
+from ruamel.yaml import YAML
+yaml = YAML(typ="safe")
 
 here_dir = os.path.abspath(os.path.dirname(__file__))
 schema_yaml = os.path.join(
@@ -50,7 +51,7 @@ def run():
     # Using these sets, we can validate further manually by printing the results
     # of set operations.
     with open(schema_yaml) as f:
-        schema = yaml.safe_load(f)
+        schema = yaml.load(f)
 
     # Drop what isn't relevant for a values.schema.json file packaged with the
     # Helm chart, such as the description keys only relevant for our

tools/validate-against-schema.py

@@ -1,7 +1,9 @@
 #!/usr/bin/env python3
-import jsonschema
 import os
-import yaml
+
+import jsonschema
+from ruamel.yaml import YAML
+yaml = YAML(typ="safe")
 
 here_dir = os.path.abspath(os.path.dirname(__file__))
 schema_yaml = os.path.join(
@@ -18,13 +20,13 @@
 )
 
 with open(schema_yaml) as f:
-    schema = yaml.safe_load(f)
+    schema = yaml.load(f)
 with open(values_yaml) as f:
-    values = yaml.safe_load(f)
+    values = yaml.load(f)
 with open(lint_and_validate_values_yaml) as f:
-    lint_and_validate_values = yaml.safe_load(f)
+    lint_and_validate_values = yaml.load(f)
 with open(binderhub_chart_config_yaml) as f:
-    binderhub_chart_config_yaml = yaml.safe_load(f)
+    binderhub_chart_config_yaml = yaml.load(f)
 
 # Validate values.yaml against schema
 print("Validating values.yaml against schema.yaml...")