Merge pull request #240 from simleo/determine_data_entities

Update data entities reading for 1.2

Author: simleo

requirements.txt

@@ -3,3 +3,4 @@ arcp==0.2.1
 jinja2
 python-dateutil
 click
+packaging

rocrate/model/metadata.py

@@ -25,6 +25,8 @@
 import json
 from pathlib import Path
 
+from packaging.version import Version
+
 from .file import File
 from .dataset import Dataset
 
@@ -45,6 +47,7 @@ def __init__(self, crate, source=None, dest_path=None, properties=None, version=
         if version not in SUPPORTED_VERSIONS:
             raise ValueError(f"version {version!r} not supported")
         self.version = version
+        self.version_obj = Version(version)
         self.profile = f"https://w3id.org/ro/crate/{self.version}"
         if source is None and dest_path is None:
             dest_path = LEGACY_BASENAME if version == "1.0" else BASENAME

rocrate/rocrate.py

@@ -33,6 +33,8 @@
 from pathlib import Path
 from urllib.parse import urljoin
 
+from packaging.version import Version
+
 from .memory_buffer import MemoryBuffer
 from .model import (
     ComputationalWorkflow,
@@ -63,6 +65,15 @@
 from .metadata import read_metadata, find_root_entity_id
 
 
+DATA_ENTITY_TYPES = {"File", "Dataset"}
+
+
+def is_data_entity(entity):
+    if entity["@id"].startswith("#"):
+        return False
+    return DATA_ENTITY_TYPES.intersection(as_list(entity.get("@type", [])))
+
+
 def pick_type(json_entity, type_map, fallback=None):
     try:
         t = json_entity["@type"]
@@ -172,12 +183,14 @@ def __read_data_entities(self, entities, source, gen_preview):
 
     def __add_parts(self, parts, entities, source):
         type_map = OrderedDict((_.__name__, _) for _ in subclasses(FileOrDir))
-        for data_entity_ref in parts:
-            id_ = data_entity_ref['@id']
-            try:
-                entity = entities.pop(id_)
-            except KeyError:
+        for ref in parts:
+            id_ = ref['@id']
+            if id_ not in entities:
                 continue
+            if self.version_obj >= Version("1.2"):
+                if not is_data_entity(entities[id_]):
+                    continue
+            entity = entities.pop(id_)
             assert id_ == entity.pop('@id')
             cls = pick_type(entity, type_map, fallback=DataEntity)
             if cls is DataEntity:
@@ -193,11 +206,13 @@ def __add_parts(self, parts, entities, source):
 
     def __read_contextual_entities(self, entities):
         type_map = {_.__name__: _ for _ in subclasses(ContextEntity)}
-        # types *commonly* used for data entities
-        data_entity_types = {"File", "Dataset"}
         for identifier, entity in entities.items():
-            if data_entity_types.intersection(as_list(entity.get("@type", []))):
-                warnings.warn(f"{entity['@id']} looks like a data entity but it's not listed in the root dataset's hasPart")
+            if is_data_entity(entity):
+                id_ = entity['@id']
+                if self.version_obj >= Version("1.2"):
+                    raise ValueError(f"'{id_}' is a data entity but it's not linked to from the root dataset's hasPart")
+                else:
+                    warnings.warn(f"'{id_}' looks like a data entity but it's not listed in the root dataset's hasPart")
             assert identifier == entity.pop('@id')
             cls = pick_type(entity, type_map, fallback=ContextEntity)
             self.add(cls(self, identifier, entity))
@@ -311,6 +326,10 @@ def mainEntity(self, value):
     def version(self):
         return self.metadata.version
 
+    @property
+    def version_obj(self):
+        return self.metadata.version_obj
+
     @property
     def test_dir(self):
         rval = self.dereference("test")

test/test_metadata.py

@@ -42,7 +42,7 @@ def test_find_root(root, basename):
             "@type": "CreativeWork",
             "about": {"@id": root_id},
             "conformsTo": [
-                {"@id": "https://w3id.org/ro/crate/1.1"},
+                {"@id": "https://w3id.org/ro/crate/1.2"},
                 {"@id": "https://example.org/fancy-ro-crate/1.0"},
             ]
         },
@@ -64,7 +64,7 @@ def test_find_root_bad_entities():
             "@id": "ro-crate-metadata.json",
             "@type": "CreativeWork",
             "about": {"@id": "./"},
-            "conformsTo": {"@id": "https://w3id.org/ro/crate/1.1"},
+            "conformsTo": {"@id": "https://w3id.org/ro/crate/1.2"},
         },
         "./": {
             "@id": "./",
@@ -101,7 +101,7 @@ def test_find_root_multiple_entries():
             "@id": "http://example.org/ro-crate-metadata.json",
             "@type": "CreativeWork",
             "about": {"@id": "http://example.org/"},
-            "conformsTo": {"@id": "https://w3id.org/ro/crate/1.1"},
+            "conformsTo": {"@id": "https://w3id.org/ro/crate/1.2"},
         },
         "http://example.org/": {
             "@id": "http://example.org/",
@@ -115,7 +115,7 @@ def test_find_root_multiple_entries():
             "@id": "http://example.com/ro-crate-metadata.json",
             "@type": "CreativeWork",
             "about": {"@id": "http://example.com/"},
-            "conformsTo": {"@id": "https://w3id.com/ro/crate/1.1"},
+            "conformsTo": {"@id": "https://w3id.com/ro/crate/1.2"},
         },
         "http://example.com/": {
             "@id": "http://example.com/",
@@ -166,7 +166,7 @@ def test_find_root_multiple_types():
             "@id": "ro-crate-metadata.json",
             "@type": "CreativeWork",
             "about": {"@id": "./"},
-            "conformsTo": {"@id": "https://w3id.org/ro/crate/1.1"},
+            "conformsTo": {"@id": "https://w3id.org/ro/crate/1.2"},
         },
         {
             "@id": "./",

test/test_model.py

@@ -315,7 +315,7 @@ def test_self_delete(test_data_dir):
 def test_entity_as_mapping(tmpdir, helpers):
     orcid = "https://orcid.org/0000-0002-1825-0097"
     metadata = {
-        "@context": "https://w3id.org/ro/crate/1.1/context",
+        "@context": "https://w3id.org/ro/crate/1.2/context",
         "@graph": [
             {"@id": "ro-crate-metadata.json",
              "@type": "CreativeWork",
@@ -324,7 +324,7 @@ def test_entity_as_mapping(tmpdir, helpers):
                  "application/json",
                  {"@id": "https://www.json.org"},
              ],
-             "conformsTo": {"@id": "https://w3id.org/ro/crate/1.1"}},
+             "conformsTo": {"@id": "https://w3id.org/ro/crate/1.2"}},
             {"@id": "./",
              "@type": "Dataset",
              "correction": [