allow ReferenceFileSystem to hold dicts, which are treated as JSON files (#1562)

Author: bendichter

fsspec/implementations/reference.py

@@ -935,6 +935,10 @@ def _process_references(self, references, template_overrides=None):
 
     def _process_references0(self, references):
         """Make reference dict for Spec Version 0"""
+        references = {
+            key: json.dumps(val) if isinstance(val, dict) else val
+            for key, val in references.items()
+        }
         self.references = references
 
     def _process_references1(self, references, template_overrides=None):
@@ -964,6 +968,8 @@ def _render_jinja(u):
                     else:
                         u = _render_jinja(u)
                 self.references[k] = [u] if len(v) == 1 else [u, v[1], v[2]]
+            elif isinstance(v, dict):
+                self.references[k] = json.dumps(v)
             else:
                 self.references[k] = v
         self.references.update(self._process_gen(references.get("gen", [])))

fsspec/implementations/tests/test_reference.py

@@ -19,6 +19,7 @@ def test_simple(server):  # noqa: F811
         "b": (realfile, 0, 5),
         "c": (realfile, 1, 5),
         "d": b"base64:aGVsbG8=",
+        "e": {"key": "value"},
     }
     h = fsspec.filesystem("http")
     fs = fsspec.filesystem("reference", fo=refs, fs=h)
@@ -27,6 +28,30 @@ def test_simple(server):  # noqa: F811
     assert fs.cat("b") == data[:5]
     assert fs.cat("c") == data[1 : 1 + 5]
     assert fs.cat("d") == b"hello"
+    assert fs.cat("e") == b'{"key": "value"}'
+    with fs.open("d", "rt") as f:
+        assert f.read(2) == "he"
+
+
+def test_simple_ver1(server):  # noqa: F811
+    in_data = {
+        "version": 1,
+        "refs": {
+            "a": b"data",
+            "b": (realfile, 0, 5),
+            "c": (realfile, 1, 5),
+            "d": b"base64:aGVsbG8=",
+            "e": {"key": "value"},
+        },
+    }
+    h = fsspec.filesystem("http")
+    fs = fsspec.filesystem("reference", fo=in_data, fs=h)
+
+    assert fs.cat("a") == b"data"
+    assert fs.cat("b") == data[:5]
+    assert fs.cat("c") == data[1 : 1 + 5]
+    assert fs.cat("d") == b"hello"
+    assert fs.cat("e") == b'{"key": "value"}'
     with fs.open("d", "rt") as f:
         assert f.read(2) == "he"