Add basic tracking of $import and $include dependencies. (#650)

* Add basic tracking of $import and $include dependencies.

Currently does not round trip them but is suitable for dependency
management.

* Update metaschema.py

* GA: pin tox to 3.x

Co-authored-by: Michael R. Crusoe <[email protected]>

Author: tetron

.github/workflows/ci-tests.yml

@@ -46,7 +46,7 @@ jobs:
       - name: Upgrade setuptools and install tox
         run: |
           pip install -U pip setuptools wheel
-          pip install tox tox-gh-actions
+          pip install 'tox<4' tox-gh-actions
 
       - name: MyPy cache
         if: ${{ matrix.step == 'mypy' }}
@@ -75,7 +75,7 @@ jobs:
 
     env:
       py-semver: "3.11"
-      TOXENV: ${{ format('py310-{0}', matrix.step) }}
+      TOXENV: ${{ format('py311-{0}', matrix.step) }}
 
     steps:
       - uses: actions/checkout@v3
@@ -96,7 +96,7 @@ jobs:
       - name: Upgrade setuptools and install tox
         run: |
           pip install -U pip setuptools wheel
-          pip install tox tox-gh-actions
+          pip install 'tox<4' tox-gh-actions
 
       - if: ${{ matrix.step == 'pydocstyle' && github.event_name == 'pull_request'}}
         name: Create local branch for diff-quality for PRs

README.rst

@@ -107,7 +107,7 @@ using the `CWL v1.2 schema <https://github.com/common-workflow-language/cwl-v1.2
 as an example.
 
 +-------------+---------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-| Language    | Repository                                              | Serializaton Example                                                                                                                                 | Deserialization Example                                                                                                                                                    |
+| Language    | Repository                                              | Serialization Example                                                                                                                                 | Deserialization Example                                                                                                                                                    |
 +=============+=========================================================+======================================================================================================================================================+============================================================================================================================================================================+
 | Python      | https://github.com/common-workflow-language/cwl-utils/  | `create_cwl_from_objects.py <https://github.com/common-workflow-language/cwl-utils/blob/main/create_cwl_from_objects.py>`_                           | `load_document() <https://github.com/common-workflow-language/cwl-utils/blob/main/cwl_utils/parser/__init__.py#L93>`_                                                      |
 +-------------+---------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+

schema_salad/metaschema.py

@@ -58,6 +58,8 @@ class LoadingOptions:
     vocab: Dict[str, str]
     rvocab: Dict[str, str]
     cache: CacheType
+    imports: List[str]
+    includes: List[str]
 
     def __init__(
         self,
@@ -70,6 +72,8 @@ def __init__(
         addl_metadata: Optional[Dict[str, str]] = None,
         baseuri: Optional[str] = None,
         idx: Optional[IdxType] = None,
+        imports: Optional[List[str]] = None,
+        includes: Optional[List[str]] = None,
     ) -> None:
         """Create a LoadingOptions object."""
         self.original_doc = original_doc
@@ -104,6 +108,16 @@ def __init__(
         else:
             self.addl_metadata = copyfrom.addl_metadata if copyfrom is not None else {}
 
+        if imports is not None:
+            self.imports = imports
+        else:
+            self.imports = copyfrom.imports if copyfrom is not None else []
+
+        if includes is not None:
+            self.includes = includes
+        else:
+            self.includes = copyfrom.includes if copyfrom is not None else []
+
         if fetcher is not None:
             self.fetcher = fetcher
         elif copyfrom is not None:
@@ -203,18 +217,22 @@ def load_field(val, fieldtype, baseuri, loadingOptions):
         if "$import" in val:
             if loadingOptions.fileuri is None:
                 raise SchemaSaladException("Cannot load $import without fileuri")
+            url = loadingOptions.fetcher.urljoin(loadingOptions.fileuri, val["$import"])
             result, metadata = _document_load_by_url(
                 fieldtype,
-                loadingOptions.fetcher.urljoin(loadingOptions.fileuri, val["$import"]),
+                url,
                 loadingOptions,
             )
+            loadingOptions.imports.append(url)
             return result
         elif "$include" in val:
             if loadingOptions.fileuri is None:
                 raise SchemaSaladException("Cannot load $import without fileuri")
-            val = loadingOptions.fetcher.fetch_text(
-                loadingOptions.fetcher.urljoin(loadingOptions.fileuri, val["$include"])
+            url = loadingOptions.fetcher.urljoin(
+                loadingOptions.fileuri, val["$include"]
             )
+            val = loadingOptions.fetcher.fetch_text(url)
+            loadingOptions.includes.append(url)
     return fieldtype.load(val, baseuri, loadingOptions)
 
 

schema_salad/python_codegen_support.py

@@ -55,6 +55,8 @@ class LoadingOptions:
     vocab: Dict[str, str]
     rvocab: Dict[str, str]
     cache: CacheType
+    imports: List[str]
+    includes: List[str]
 
     def __init__(
         self,
@@ -67,6 +69,8 @@ def __init__(
         addl_metadata: Optional[Dict[str, str]] = None,
         baseuri: Optional[str] = None,
         idx: Optional[IdxType] = None,
+        imports: Optional[List[str]] = None,
+        includes: Optional[List[str]] = None,
     ) -> None:
         """Create a LoadingOptions object."""
         self.original_doc = original_doc
@@ -101,6 +105,16 @@ def __init__(
         else:
             self.addl_metadata = copyfrom.addl_metadata if copyfrom is not None else {}
 
+        if imports is not None:
+            self.imports = imports
+        else:
+            self.imports = copyfrom.imports if copyfrom is not None else []
+
+        if includes is not None:
+            self.includes = includes
+        else:
+            self.includes = copyfrom.includes if copyfrom is not None else []
+
         if fetcher is not None:
             self.fetcher = fetcher
         elif copyfrom is not None:
@@ -200,18 +214,22 @@ def load_field(val, fieldtype, baseuri, loadingOptions):
         if "$import" in val:
             if loadingOptions.fileuri is None:
                 raise SchemaSaladException("Cannot load $import without fileuri")
+            url = loadingOptions.fetcher.urljoin(loadingOptions.fileuri, val["$import"])
             result, metadata = _document_load_by_url(
                 fieldtype,
-                loadingOptions.fetcher.urljoin(loadingOptions.fileuri, val["$import"]),
+                url,
                 loadingOptions,
             )
+            loadingOptions.imports.append(url)
             return result
         elif "$include" in val:
             if loadingOptions.fileuri is None:
                 raise SchemaSaladException("Cannot load $import without fileuri")
-            val = loadingOptions.fetcher.fetch_text(
-                loadingOptions.fetcher.urljoin(loadingOptions.fileuri, val["$include"])
+            url = loadingOptions.fetcher.urljoin(
+                loadingOptions.fileuri, val["$include"]
             )
+            val = loadingOptions.fetcher.fetch_text(url)
+            loadingOptions.includes.append(url)
     return fieldtype.load(val, baseuri, loadingOptions)