Fix handling of dict values and templates in ReferenceFileSystem (#1616)

Author: rly

fsspec/implementations/reference.py

@@ -956,6 +956,8 @@ def _render_jinja(u):
                 if v.startswith("base64:"):
                     self.references[k] = base64.b64decode(v[7:])
                 self.references[k] = v
+            elif isinstance(v, dict):
+                self.references[k] = json.dumps(v)
             elif self.templates:
                 u = v[0]
                 if "{{" in u:
@@ -968,8 +970,6 @@ def _render_jinja(u):
                     else:
                         u = _render_jinja(u)
                 self.references[k] = [u] if len(v) == 1 else [u, v[1], v[2]]
-            elif isinstance(v, dict):
-                self.references[k] = json.dumps(v)
             else:
                 self.references[k] = v
         self.references.update(self._process_gen(references.get("gen", [])))

fsspec/implementations/tests/test_reference.py

@@ -229,6 +229,8 @@ def test_defaults(server):  # noqa: F811
 
 def test_spec1_expand():
     pytest.importorskip("jinja2")
+    from fsspec.implementations.reference import json as json_impl
+
     in_data = {
         "version": 1,
         "templates": {"u": "server.domain/path", "f": "{{c}}"},
@@ -252,6 +254,7 @@ def test_spec1_expand():
             "key2": ["http://{{u}}", 10000, 100],
             "key3": ["http://{{f(c='text')}}", 10000, 100],
             "key4": ["http://target_url"],
+            "key5": {"key": "value"},
         },
     }
     fs = fsspec.filesystem(
@@ -263,6 +266,7 @@ def test_spec1_expand():
         "key2": ["http://server.domain/path", 10000, 100],
         "key3": ["http://text", 10000, 100],
         "key4": ["http://target_url"],
+        "key5": json_impl.dumps(in_data["refs"]["key5"]),
         "gen_key0": ["http://server.domain/path_0", 1000, 1000],
         "gen_key1": ["http://server.domain/path_1", 2000, 1000],
         "gen_key2": ["http://server.domain/path_2", 3000, 1000],
@@ -275,13 +279,16 @@ def test_spec1_expand():
 
 def test_spec1_expand_simple():
     pytest.importorskip("jinja2")
+    from fsspec.implementations.reference import json as json_impl
+
     in_data = {
         "version": 1,
         "templates": {"u": "server.domain/path"},
         "refs": {
             "key0": "base64:ZGF0YQ==",
             "key2": ["http://{{u}}", 10000, 100],
             "key4": ["http://target_url"],
+            "key5": {"key": "value"},
         },
     }
     fs = fsspec.filesystem("reference", fo=in_data, target_protocol="http")
@@ -294,6 +301,7 @@ def test_spec1_expand_simple():
     )
     assert fs.references["key2"] == ["http://not.org/p", 10000, 100]
     assert fs.cat("key0") == b"data"
+    assert fs.cat("key5") == json_impl.dumps(in_data["refs"]["key5"]).encode("utf-8")
 
 
 def test_spec1_gen_variants():