feat: implement pin_write/read json type (#168)

* added implementation for json type

* ci: do not test azure on external PRs

* tests: check board pin_write type json

Co-authored-by: Michael Chow <[email protected]>

Author: edavidaja

.github/workflows/ci.yml

@@ -123,7 +123,7 @@ jobs:
       - name: Run tests
         run: |
           # TODO: better way to disable all cloud backend tests?
-          pytest pins -m 'not fs_rsc and not fs_s3 and not fs_gcs and not skip_on_github'
+          pytest pins -m 'not fs_rsc and not fs_s3 and not fs_gcs and not fs_abfs and not skip_on_github'
 
 
   check-cross-compatibility:

pins/drivers.py

@@ -105,6 +105,11 @@ def load_data(
         # TODO: update to handle multiple files
         return [str(Path(fs.open(path_to_file).name).absolute())]
 
+    elif meta.type == "json":
+        import json
+
+        return json.load(fs.open(path_to_file))
+
     raise NotImplementedError(f"No driver for type {meta.type}")
 
 
@@ -151,6 +156,12 @@ def save_data(
         import joblib
 
         joblib.dump(obj, final_name)
+
+    elif type == "json":
+        import json
+
+        json.dump(obj, open(final_name, "w"))
+
     else:
         raise NotImplementedError(f"Cannot save type: {type}")
 

pins/tests/test_boards.py

@@ -128,7 +128,13 @@ def test_board_pin_write_rsc_index_html(board, tmp_dir2, snapshot):
 
 
 @parametrize(
-    "obj, type_", [(df, "csv"), (df, "joblib"), ({"a": 1, "b": [2, 3]}, "joblib")]
+    "obj, type_",
+    [
+        (df, "csv"),
+        (df, "joblib"),
+        ({"a": 1, "b": [2, 3]}, "joblib"),
+        ({"a": 1, "b": [2, 3]}, "json"),
+    ],
 )
 def test_board_pin_write_type(board, obj, type_, request):
     with rm_env(PINS_ENV_INSECURE_READ):

pins/tests/test_drivers.py

@@ -77,6 +77,30 @@ def test_driver_roundtrip(tmp_dir2, type_):
     assert df.equals(obj)
 
 
+@pytest.mark.parametrize(
+    "type_",
+    [
+        "json",
+    ],
+)
+def test_driver_roundtrip_json(tmp_dir2, type_):
+
+    df = {"x": [1, 2, 3]}
+
+    fname = "some_df"
+    full_file = f"{fname}.{type_}"
+
+    p_obj = tmp_dir2 / fname
+    res_fname = save_data(df, p_obj, type_)
+
+    assert Path(res_fname).name == full_file
+
+    meta = MetaRaw(full_file, type_, "my_pin")
+    obj = load_data(meta, fsspec.filesystem("file"), tmp_dir2, allow_pickle_read=True)
+
+    assert df == obj
+
+
 def test_driver_feather_write_error(tmp_dir2):
     import pandas as pd