Track 5.0.0 release

CHANGELOG.md

@@ -1,5 +1,27 @@
 # HDMF Changelog
 
+## HDMF 5.0.0 (Upcoming)
+
+### Changed
+- New spec resolution system: Instead of resolving includes during spec loading, resolution now happens after all specs are loaded via `NamespaceCatalog.resolve_all_specs()`. @rly [#1312](https://github.com/hdmf-dev/hdmf/pull/1312)
+  - New methods: `BaseStorageSpec.resolve_inc_spec()` replaces the old `BaseStorageSpec.resolve_spec()` method
+  - Resolution tracking: New properties `BaseStorageSpec.resolved` and `BaseStorageSpec.inc_spec_resolved` track resolution state
+  - Cross-namespace resolution: The system can now resolve specs that include types from different namespaces
+  - `dtype`, `shape`, `dims`, `value`, and `default_value` in `DatasetSpec` and `AttributeSpec` are now inherited and validated from the parent data type spec
+- If `dims` are not provided in a `DatasetSpec` or `AttributeSpec`, but `shape` is provided, `dims` will be set to a list of dummy dimension names, e.g., "dim_0", "dim_1", etc. @rly [#1312](https://github.com/hdmf-dev/hdmf/pull/1312)
+- Deprecated `BaseStorageSpec.add_attribute`, `GroupSpec.add_group`, `GroupSpec.add_dataset`, and `GroupSpec.add_link`. Use `set_attribute`, `set_group`, `set_dataset`, and `set_link` instead. @rly [#1333](https://github.com/hdmf-dev/hdmf/pull/1333)
+- Deprecated unused `BaseStorageSpec.get_data_type_spec` and `BaseStorageSpec.get_namespace_spec`. @rly [#1333](https://github.com/hdmf-dev/hdmf/pull/1333)
+
+### Added
+- Warning when `data_type_def` and `data_type_inc` are the same in a spec. @rly [#1312](https://github.com/hdmf-dev/hdmf/pull/1312)
+- Added abstract methods `HDMFIO.load_namespaces` and `HDMFIO.load_namespaces_io`. @rly [#1299](https://github.com/hdmf-dev/hdmf/pull/1299)
+
+### Removed
+- Removed unused and undocumented `hdmf.monitor` module. @rly [#1327](https://github.com/hdmf-dev/hdmf/pull/1327)
+- Removed deprecated `Data.set_data_io` usage and `HERDManager` methods. @rly [#1328](https://github.com/hdmf-dev/hdmf/pull/1328)
+- Removed deprecated `HDF5IO.copy_file` method. Use the `HDF5IO.export` or the `h5py.File.copy` method instead. @stephprince [#1332](https://github.com/hdmf-dev/hdmf/pull/1332)
+- Removed deprecated `extensions` kwarg for `get_type_map` function. @stephprince [#1332](https://github.com/hdmf-dev/hdmf/pull/1332)
+
 ## HDMF 4.1.1 (Upcoming)
 
 ### Fixed

docs/source/extensions.rst

@@ -207,7 +207,7 @@ The following code demonstrates how to load custom namespaces.
 
 .. code-block:: python
 
-    from hdmf import load_namespaces
+    from hdmf.common import load_namespaces
     namespace_path = 'my_namespace.yaml'
     load_namespaces(namespace_path)
 

src/hdmf/backends/hdf5/h5tools.py

@@ -159,25 +159,34 @@ def __resolve_file_obj(cls, path, file_obj, driver, aws_region=None):
         return file_obj
 
     @classmethod
-    @docval({'name': 'namespace_catalog', 'type': (NamespaceCatalog, TypeMap),
-             'doc': 'the NamespaceCatalog or TypeMap to load namespaces into'},
-            {'name': 'path', 'type': (str, Path), 'doc': 'the path to the HDF5 file', 'default': None},
-            {'name': 'namespaces', 'type': list, 'doc': 'the namespaces to load', 'default': None},
-            {'name': 'file', 'type': File, 'doc': 'a pre-existing h5py.File object', 'default': None},
-            {'name': 'driver', 'type': str, 'doc': 'driver for h5py to use when opening HDF5 file', 'default': None},
-            {'name': 'aws_region', 'type': str, 'doc': 'If driver is ros3, then specify the aws region of the url.',
-             'default': None},
-            returns=("dict mapping the names of the loaded namespaces to a dict mapping included namespace names and "
-                     "the included data types"),
-            rtype=dict)
+    @docval(
+        {
+            'name': 'namespace_catalog',
+            'type': (NamespaceCatalog, TypeMap),
+            'doc': 'the NamespaceCatalog or TypeMap to load namespaces into'
+        },
+        {'name': 'path', 'type': (str, Path), 'doc': 'the path to the HDF5 file', 'default': None},
+        {'name': 'namespaces', 'type': list, 'doc': 'the namespaces to load', 'default': None},
+        {'name': 'file', 'type': File, 'doc': 'a pre-existing h5py.File object', 'default': None},
+        {'name': 'driver', 'type': str, 'doc': 'driver for h5py to use when opening HDF5 file', 'default': None},
+        {
+            'name': 'aws_region',
+            'type': str,
+            'doc': 'If driver is ros3, then specify the aws region of the url.',
+            'default': None
+        },
+        returns=("dict mapping the names of the loaded namespaces to a dict mapping included namespace names and "
+                    "the included data types"),
+        rtype=dict
+    )
     def load_namespaces(cls, **kwargs):
-        """Load cached namespaces from a file.
+        """Load cached namespaces from a file into the provided NamespaceCatalog or TypeMap.
 
         If `file` is not supplied, then an :py:class:`h5py.File` object will be opened for the given `path`, the
         namespaces will be read, and the File object will be closed. If `file` is supplied, then
         the given File object will be read from and not closed.
 
-        :raises ValueError: if both `path` and `file` are supplied but `path` is not the same as the path of `file`.
+        :raises ValueError: if both `path` and `file` are supplied but `path` is not the same as the path of `file`
         """
         namespace_catalog, path, namespaces, file_obj, driver, aws_region = popargs(
             'namespace_catalog', 'path', 'namespaces', 'file', 'driver', 'aws_region', kwargs)
@@ -188,12 +197,25 @@ def load_namespaces(cls, **kwargs):
                 return cls.__load_namespaces(namespace_catalog, namespaces, open_file_obj)
         return cls.__load_namespaces(namespace_catalog, namespaces, open_file_obj)
 
+    @docval(
+        {
+            'name': 'namespace_catalog',
+            'type': (NamespaceCatalog, TypeMap),
+            'doc': 'the NamespaceCatalog or TypeMap to load namespaces into'
+        },
+        {'name': 'namespaces', 'type': list, 'doc': 'the namespaces to load', 'default': None}
+    )
+    def load_namespaces_io(self, **kwargs):
+        """Load cached namespaces from this HDF5IO object into the provided NamespaceCatalog or TypeMap."""
+        namespace_catalog, namespaces = getargs('namespace_catalog', 'namespaces', kwargs)
+        if not self.__file:
+            raise UnsupportedOperation("Cannot load namespaces from closed HDF5 file '%s'" % self.source)
+        return self.__load_namespaces(namespace_catalog, namespaces, self.__file)
+
     @classmethod
     def __load_namespaces(cls, namespace_catalog, namespaces, file_obj):
-        d = {}
-
         if not cls.__check_specloc(file_obj):
-            return d
+            return {}
 
         namespace_versions = cls.__get_namespaces(file_obj)
 
@@ -205,11 +227,9 @@ def __load_namespaces(cls, namespace_catalog, namespaces, file_obj):
         for ns in namespaces:
             latest_version = namespace_versions[ns]
             ns_group = spec_group[ns][latest_version]
-            reader = H5SpecReader(ns_group)
-            readers[ns] = reader
-
-        d.update(namespace_catalog.load_namespaces(cls.__ns_spec_path, reader=readers))
+            readers[ns] = H5SpecReader(ns_group)
 
+        d = namespace_catalog.load_namespaces(cls.__ns_spec_path, reader=readers)
         return d
 
     @classmethod
@@ -274,58 +294,6 @@ def __get_namespaces(cls, file_obj):
 
         return used_version_names
 
-    @classmethod
-    @docval({'name': 'source_filename', 'type': str, 'doc': 'the path to the HDF5 file to copy'},
-            {'name': 'dest_filename', 'type': str, 'doc': 'the name of the destination file'},
-            {'name': 'expand_external', 'type': bool, 'doc': 'expand external links into new objects', 'default': True},
-            {'name': 'expand_refs', 'type': bool, 'doc': 'copy objects which are pointed to by reference',
-             'default': False},
-            {'name': 'expand_soft', 'type': bool, 'doc': 'expand soft links into new objects', 'default': False}
-            )
-    def copy_file(self, **kwargs):
-        """
-        Convenience function to copy an HDF5 file while allowing external links to be resolved.
-
-        .. warning::
-
-            As of HDMF 2.0, this method is no longer supported and may be removed in a future version.
-            Please use the export method or h5py.File.copy method instead.
-
-        .. note::
-
-            The source file will be opened in 'r' mode and the destination file will be opened in 'w' mode
-            using h5py. To avoid possible collisions, care should be taken that, e.g., the source file is
-            not opened already when calling this function.
-
-        """
-
-        warnings.warn("The copy_file class method is no longer supported and may be removed in a future version of "
-                      "HDMF. Please use the export method or h5py.File.copy method instead.",
-                      category=DeprecationWarning,
-                      stacklevel=3)
-
-        source_filename, dest_filename, expand_external, expand_refs, expand_soft = getargs('source_filename',
-                                                                                            'dest_filename',
-                                                                                            'expand_external',
-                                                                                            'expand_refs',
-                                                                                            'expand_soft',
-                                                                                            kwargs)
-        source_file = File(source_filename, 'r')
-        dest_file = File(dest_filename, 'w')
-        for objname in source_file["/"].keys():
-            source_file.copy(source=objname,
-                             dest=dest_file,
-                             name=objname,
-                             expand_external=expand_external,
-                             expand_refs=expand_refs,
-                             expand_soft=expand_soft,
-                             shallow=False,
-                             without_attrs=False,
-                             )
-        for objname in source_file['/'].attrs:
-            dest_file['/'].attrs[objname] = source_file['/'].attrs[objname]
-        source_file.close()
-        dest_file.close()
 
     @docval({'name': 'container', 'type': Container, 'doc': 'the Container object to write'},
             {'name': 'cache_spec', 'type': bool,

src/hdmf/backends/io.py

@@ -1,11 +1,13 @@
 from abc import ABCMeta, abstractmethod
 import os
 from pathlib import Path
+from typing import Union, Optional
 
-from ..build import BuildManager, GroupBuilder
+from ..build import BuildManager, GroupBuilder, TypeMap
 from ..container import Container, HERDManager
 from .errors import UnsupportedOperation
 from ..utils import docval, getargs, popargs, get_basic_array_info, generate_array_html_repr
+from ..spec import  NamespaceCatalog
 from warnings import warn
 
 
@@ -188,6 +190,44 @@ def close(self):
         ''' Close this HDMFIO object to further reading/writing'''
         pass
 
+    @classmethod
+    @abstractmethod
+    def load_namespaces(cls,
+        namespace_catalog: Union[NamespaceCatalog, TypeMap],
+        path: Optional[Union[str, Path]] = None,
+        namespaces: Optional[list[str]] = None,
+        io: Optional['HDMFIO'] = None,
+        **kwargs
+    ) -> dict:
+        """Load the namespaces from the file at the given path into the provided NamespaceCatalog or TypeMap.
+
+        This method should be implemented by subclasses to load the namespaces that are relevant for the backend.
+
+        :param namespace_catalog: The NamespaceCatalog (or TypeMap) to load the namespaces into.
+        :param path: The path to the file from which to load the namespaces.
+        :param namespaces: A list of namespace names to load. If None, all namespaces will be loaded.
+        :param kwargs: Additional keyword arguments that may be needed for the specific backend.
+        :return: A dictionary mapping namespace names to their dependencies.
+        """
+        pass
+
+    @abstractmethod
+    def load_namespaces_io(self,
+        namespace_catalog: Union[NamespaceCatalog, TypeMap],
+        namespaces: Optional[list[str]] = None,
+    ) -> dict:
+        """Load the namespaces from this HDMFIO object into the provided NamespaceCatalog or TypeMap.
+
+        Similar to `load_namespaces`, but uses the already opened HDMFIO object.
+        This method should be implemented by subclasses to load the namespaces that are relevant for the backend.
+
+        :param namespace_catalog: The NamespaceCatalog (or TypeMap) to load the namespaces into.
+        :param namespaces: A list of namespace names to load. If None, all namespaces will be loaded.
+        :return: A dictionary mapping namespace names to their dependencies.
+        """
+        # NOTE: this function is separated from load_namespaces for developer clarity
+        pass
+
     @staticmethod
     def generate_dataset_html(dataset):
         """Generates an html representation for a dataset"""

src/hdmf/common/__init__.py

@@ -168,40 +168,13 @@ def get_class(**kwargs):
     return __TYPE_MAP.get_dt_container_cls(data_type, namespace, post_init_method)
 
 
-@docval({'name': 'extensions', 'type': (str, TypeMap, list),
-         'doc': 'a path to a namespace, a TypeMap, or a list consisting paths to namespaces and TypeMaps',
-         'default': None},
-        returns="the namespaces loaded from the given file", rtype=tuple,
+@docval(returns="the namespaces loaded from the given file", rtype=tuple,
         is_method=False)
-def get_type_map(**kwargs):
+def get_type_map():
     '''
-    Get a BuildManager to use for I/O using the given extensions. If no extensions are provided,
-    return a BuildManager that uses the core namespace
+    Get a BuildManager to use for I/O using the core namespace.
     '''
-    extensions = getargs('extensions', kwargs)
-    type_map = None
-    if extensions is None:
-        type_map = deepcopy(__TYPE_MAP)
-    else:
-        warnings.warn("The 'extensions' argument is deprecated and will be removed in HDMF 5.0", DeprecationWarning)
-        if isinstance(extensions, TypeMap):
-            type_map = extensions
-        else:
-            type_map = deepcopy(__TYPE_MAP)
-        if isinstance(extensions, list):
-            for ext in extensions:
-                if isinstance(ext, str):
-                    type_map.load_namespaces(ext)
-                elif isinstance(ext, TypeMap):
-                    type_map.merge(ext)
-                else:
-                    msg = 'extensions must be a list of paths to namespace specs or a TypeMaps'
-                    raise ValueError(msg)
-        elif isinstance(extensions, str):
-            type_map.load_namespaces(extensions)
-        elif isinstance(extensions, TypeMap):
-            type_map.merge(extensions)
-    return type_map
+    return deepcopy(__TYPE_MAP)
 
 
 @docval(*get_docval(get_type_map),

src/hdmf/container.py

@@ -37,27 +37,6 @@ class HERDManager:
     When this class is used as a mixin for a Container, it enables setting and getting an instance of HERD.
     """
 
-    @docval({'name': 'herd', 'type': 'HERD',
-             'doc': 'The external resources to be used for the container.'},)
-    def link_resources(self, **kwargs):
-        """
-        Method to attach an instance of HERD in order to auto-add terms/references to data.
-        """
-        msg = (
-            "link_resources is deprecated and will be removed in HDMF 5.0. "
-            "Use the external_resources property instead."
-        )
-        warn(msg, DeprecationWarning, stacklevel=2)
-        self.external_resources = kwargs['herd']
-
-    def get_linked_resources(self):
-        msg = (
-            "get_linked_resources is deprecated and will be removed in HDMF 5.0. "
-            "Use the external_resources property instead."
-        )
-        warn(msg, DeprecationWarning, stacklevel=2)
-        return self.external_resources
-
     @property
     def external_resources(self):
         return self._herd if hasattr(self, "_herd") else None
@@ -865,9 +844,9 @@ def set_data_io(
         self,
         dataset_name: str,
         data_io_class: Type[DataIO],
-        data_io_kwargs: dict = None,
+        data_io_kwargs: dict,
         data_chunk_iterator_class: Optional[Type[AbstractDataChunkIterator]] = None,
-        data_chunk_iterator_kwargs: dict = None, **kwargs
+        data_chunk_iterator_kwargs: Optional[dict] = None,
     ):
         """
         Apply DataIO object to a dataset field of the Container.
@@ -884,23 +863,12 @@ def set_data_io(
             Class to use for DataChunkIterator. If None, no DataChunkIterator is used.
         data_chunk_iterator_kwargs: dict
             keyword arguments passed to the constructor of the DataChunkIterator class.
-        **kwargs:
-            DEPRECATED. Use data_io_kwargs instead.
-            kwargs are passed to the constructor of the DataIO class.
 
         Notes
         -----
         If data_chunk_iterator_class is not None, the data is wrapped in the DataChunkIterator before being wrapped in
         the DataIO. This allows for rewriting the backend configuration of hdf5 datasets.
         """
-        if kwargs or (data_io_kwargs is None):
-            warn(
-                "Use of **kwargs in Container.set_data_io() is deprecated. Please pass the DataIO kwargs as a "
-                "dictionary to the `data_io_kwargs` parameter instead.",
-                DeprecationWarning,
-                stacklevel=2
-            )
-            data_io_kwargs = kwargs
         data = self.fields.get(dataset_name)
         data_chunk_iterator_kwargs = data_chunk_iterator_kwargs or dict()
         if data is None: