WIP unfreeze aftermath and update pydantic

Author: FynnBe

bioimageio/spec/_internal/base_nodes.py

@@ -29,13 +29,13 @@
     StringConstraints,
     TypeAdapter,
     ValidationInfo,
-    model_validator,  # type: ignore
+    model_validator,
 )
 from pydantic_core import PydanticUndefined, core_schema
 from typing_extensions import Annotated, LiteralString, Self
 
 from bioimageio.spec._internal.constants import IN_PACKAGE_MESSAGE
-from bioimageio.spec._internal.types import NotEmpty, RdfContent, Version, YamlValue
+from bioimageio.spec._internal.types import RdfContent, Version
 from bioimageio.spec._internal.utils import unindent
 from bioimageio.spec._internal.validation_context import InternalValidationContext, get_internal_validation_context
 from bioimageio.spec.summary import ValidationSummary
@@ -137,9 +137,9 @@ def set_fields_explicitly(cls, data: Union[Any, Dict[Any, Any]]) -> Union[Any, D
 class ResourceDescriptionBase(NodeWithExplicitlySetFields):
     """base class for all resource descriptions"""
 
-    type: str
-    format_version: str
-    _internal_validation_context: InternalValidationContext
+    _internal_validation_context: InternalValidationContext = PrivateAttr(
+        default_factory=get_internal_validation_context
+    )
     _validation_summaries: List[ValidationSummary] = PrivateAttr(default_factory=list)
 
     fields_to_set_explicitly: ClassVar[FrozenSet[LiteralString]] = frozenset({"type", "format_version"})
@@ -174,7 +174,7 @@ def remember_internal_validation_context(self, info: ValidationInfo) -> Self:
     @classmethod
     def __pydantic_init_subclass__(cls, **kwargs: Any):
         super().__pydantic_init_subclass__(**kwargs)
-        if cls.model_fields["format_version"].default is not PydanticUndefined:
+        if "format_version" in cls.model_fields and cls.model_fields["format_version"].default is not PydanticUndefined:
             cls.implemented_format_version = cls.model_fields["format_version"].default
             if "." not in cls.implemented_format_version:
                 cls.implemented_format_version_tuple = (0, 0, 0)
@@ -228,6 +228,8 @@ def model_validate(
 
 
 class StringNode(collections.UserString, ABC):
+    """deprecated! don't use for new spec fields!"""
+
     _pattern: ClassVar[str]
     _node_class: Type[Node]
     _node: Optional[Node] = None
@@ -266,7 +268,7 @@ def __getattr__(self, name: str):
     @classmethod
     def __get_pydantic_core_schema__(cls, source: Type[Any], handler: GetCoreSchemaHandler) -> core_schema.CoreSchema:
         assert issubclass(source, StringNode)
-        return core_schema.general_after_validator_function(
+        return core_schema.with_info_after_validator_function(
             cls._validate,
             core_schema.str_schema(pattern=cls._pattern),
             serialization=core_schema.plain_serializer_function_ser_schema(
@@ -294,5 +296,15 @@ def _serialize(self) -> str:
         return self.data
 
 
-ConfigNode = Dict[NotEmpty[str], YamlValue]
-Kwargs = Dict[NotEmpty[str], YamlValue]
+class KwargsNode(Node):
+    def get(self, item: str, default: Any = None) -> Any:
+        return self[item] if item in self else default
+
+    def __getitem__(self, item: str) -> Any:
+        if item in self.model_fields:
+            return getattr(self, item)
+        else:
+            raise KeyError(item)
+
+    def __contains__(self, item: str) -> int:
+        return item in self.model_fields

bioimageio/spec/_internal/field_warning.py

@@ -5,10 +5,10 @@
 
 import pydantic.functional_validators
 from annotated_types import BaseMetadata, GroupedMetadata
-from pydantic import FieldValidationInfo, TypeAdapter
+from pydantic import TypeAdapter
 from pydantic._internal._decorators import inspect_validator
 from pydantic_core import PydanticCustomError
-from pydantic_core.core_schema import FieldValidatorFunction, NoInfoValidatorFunction
+from pydantic_core.core_schema import NoInfoValidatorFunction, ValidationInfo, WithInfoValidatorFunction
 from typing_extensions import Annotated, LiteralString
 
 from bioimageio.spec._internal.constants import ERROR, WARNING, WARNING_LEVEL_CONTEXT_KEY
@@ -24,7 +24,7 @@
     SLOTS = {"slots": True}
 
 
-ValidatorFunction = Union[NoInfoValidatorFunction, FieldValidatorFunction]
+ValidatorFunction = Union[NoInfoValidatorFunction, WithInfoValidatorFunction]
 
 AnnotationMetaData = Union[BaseMetadata, GroupedMetadata]
 
@@ -46,7 +46,7 @@ def call_validator_func(
     func: "_V2Validator",
     mode: Literal["after", "before", "plain", "wrap"],
     value: Any,
-    info: FieldValidationInfo,
+    info: ValidationInfo,
 ) -> Any:
     info_arg = inspect_validator(func, mode)
     if info_arg:
@@ -83,7 +83,7 @@ def as_warning(
 ) -> ValidatorFunction:
     """turn validation function into a no-op, based on warning level"""
 
-    def wrapper(value: Any, info: FieldValidationInfo) -> Any:
+    def wrapper(value: Any, info: ValidationInfo) -> Any:
         try:
             call_validator_func(func, mode, value, info)
         except (AssertionError, ValueError) as e:

bioimageio/spec/_internal/types/__init__.py

@@ -8,6 +8,7 @@
 from typing_extensions import Annotated
 
 from bioimageio.spec._internal.constants import DOI_REGEX, SI_UNIT_REGEX
+from bioimageio.spec._internal.types._file_source import AbsoluteFilePath as AbsoluteFilePath
 from bioimageio.spec._internal.types._file_source import FileSource as FileSource
 from bioimageio.spec._internal.types._file_source import RelativeFilePath as RelativeFilePath
 from bioimageio.spec._internal.types._generated_spdx_license_type import DeprecatedLicenseId as DeprecatedLicenseId
@@ -21,12 +22,10 @@
     validate_is_not_keyword,
     validate_orcid_id,
 )
-from bioimageio.spec._internal.validation_context import ValidationContext as ValidationContext
 
 T = TypeVar("T")
 S = TypeVar("S", bound=Sequence[Any])
 
-# types to describe RDF as pydantic models
 NotEmpty = Annotated[S, annotated_types.MinLen(1)]
 
 Datetime = Annotated[datetime, BeforeValidator(validate_datetime)]
@@ -68,11 +67,14 @@
 
 # types as loaded from YAML 1.2 (with ruamel.yaml)
 YamlLeafValue = Union[bool, date, datetime, float, int, str, None]
-YamlArray = List["YamlValue"]  # YAML Array is cast to list, but...
-YamlKey = Union[  # ... YAML Arrays are cast to tuples if used as key in mappings
+YamlKey = Union[  # YAML Arrays are cast to tuples if used as key in mappings
     YamlLeafValue, Tuple[YamlLeafValue, ...]  # (nesting is not allowed though)
 ]
-YamlMapping = Dict[YamlKey, "YamlValue"]  # YAML Mappings are cast to dict
+YamlArray = List["YamlValue"]
+YamlMapping = Dict[YamlKey, "YamlValue"]
+# note: for use in pydantic see https://docs.pydantic.dev/latest/concepts/types/#named-recursive-types
+#   and don't open another issue a la https://github.com/pydantic/pydantic/issues/8021
 YamlValue = Union[YamlLeafValue, YamlArray, YamlMapping]
+
 RdfContent = Dict[str, YamlValue]
 Doi = NewType("Doi", Annotated[str, StringConstraints(pattern=DOI_REGEX)])

bioimageio/spec/_internal/types/_file_source.py

@@ -52,7 +52,7 @@ def __repr__(self) -> str:
 
     @classmethod
     def __get_pydantic_core_schema__(cls, _source_type: Any, _handler: GetCoreSchemaHandler) -> core_schema.CoreSchema:
-        return core_schema.general_after_validator_function(
+        return core_schema.with_info_after_validator_function(
             cls._validate,
             core_schema.union_schema(
                 [

bioimageio/spec/_internal/types/field_validation.py

@@ -6,12 +6,13 @@
 from datetime import datetime
 from keyword import iskeyword
 from pathlib import PurePath
-from typing import Any, Hashable, Mapping, Sequence, Type, TypeVar, Union, get_args
+from typing import Any, Hashable, Mapping, Sequence, Tuple, Type, TypeVar, Union, get_args
 
 import annotated_types
 from dateutil.parser import isoparse
 from pydantic import AnyUrl, GetCoreSchemaHandler, functional_validators
 from pydantic_core.core_schema import CoreSchema, no_info_after_validator_function
+from typing_extensions import LiteralString
 
 from bioimageio.spec._internal.constants import SLOTS
 
@@ -41,15 +42,15 @@ def validate(self, value: str) -> str:
 
 @dataclasses.dataclass(frozen=True, **SLOTS)
 class WithSuffix:
-    suffix: Union[str, Sequence[str]]
+    suffix: Union[LiteralString, Tuple[LiteralString, ...]]
     case_sensitive: bool
 
     def __get_pydantic_core_schema__(self, source: Type[Any], handler: GetCoreSchemaHandler) -> CoreSchema:
         if not self.suffix:
             raise ValueError("suffix may not be empty")
 
         schema = handler(source)
-        if schema["type"] != str and source != FileSource and not issubclass(source, (RelativePath, PurePath)):
+        if schema["type"] != str and source != FileSource and not issubclass(source, (AnyUrl, RelativePath, PurePath)):
             raise TypeError("WithSuffix can only be applied to strings, URLs and paths")
 
         return no_info_after_validator_function(
@@ -132,7 +133,7 @@ def is_valid_yaml_value(value: Any) -> bool:
     return any(is_valid(value) for is_valid in (is_valid_yaml_key, is_valid_yaml_mapping, is_valid_yaml_sequence))
 
 
-V_suffix = TypeVar("V_suffix", bound=Union[AnyUrl, PurePath, RelativePath])
+V_suffix = TypeVar("V_suffix", bound=FileSource)
 
 
 def validate_suffix(value: V_suffix, *suffixes: str, case_sensitive: bool) -> V_suffix:

bioimageio/spec/application/v0_2.py

@@ -1,18 +1,24 @@
-from typing import Literal
+from typing import Literal, Optional
 
+from pydantic import Field
 from pydantic import HttpUrl as HttpUrl
+from typing_extensions import Annotated
 
+from bioimageio.spec._internal.types import FileSource as FileSource
 from bioimageio.spec._internal.types import RelativeFilePath as RelativeFilePath
 from bioimageio.spec.generic.v0_2 import Attachments as Attachments
 from bioimageio.spec.generic.v0_2 import Author as Author
 from bioimageio.spec.generic.v0_2 import Badge as Badge
 from bioimageio.spec.generic.v0_2 import CiteEntry as CiteEntry
-from bioimageio.spec.generic.v0_2 import GenericBase
+from bioimageio.spec.generic.v0_2 import GenericBase, WithGenericFormatVersion
 from bioimageio.spec.generic.v0_2 import LinkedResource as LinkedResource
 from bioimageio.spec.generic.v0_2 import Maintainer as Maintainer
 
 
-class Application(GenericBase, title="bioimage.io application specification"):
+class Application(GenericBase, WithGenericFormatVersion, title="bioimage.io application specification"):
     """Bioimage.io description of an application."""
 
     type: Literal["application"] = "application"
+
+    source: Annotated[Optional[FileSource], Field(description="URL or path to the source of the application")] = None
+    """The primary source of the application"""

bioimageio/spec/application/v0_3.py

@@ -1,18 +1,24 @@
-from typing import Literal
+from typing import Literal, Optional
 
+from pydantic import Field
 from pydantic import HttpUrl as HttpUrl
+from typing_extensions import Annotated
 
+from bioimageio.spec._internal.types import FileSource as FileSource
 from bioimageio.spec._internal.types import RelativeFilePath as RelativeFilePath
 from bioimageio.spec.generic.v0_3 import Attachment as Attachment
 from bioimageio.spec.generic.v0_3 import Author as Author
 from bioimageio.spec.generic.v0_3 import Badge as Badge
 from bioimageio.spec.generic.v0_3 import CiteEntry as CiteEntry
-from bioimageio.spec.generic.v0_3 import GenericBase
+from bioimageio.spec.generic.v0_3 import GenericBase, WithGenericFormatVersion
 from bioimageio.spec.generic.v0_3 import LinkedResource as LinkedResource
 from bioimageio.spec.generic.v0_3 import Maintainer as Maintainer
 
 
-class Application(GenericBase, title="bioimage.io application specification"):
+class Application(GenericBase, WithGenericFormatVersion, title="bioimage.io application specification"):
     """Bioimage.io description of an application."""
 
     type: Literal["application"] = "application"
+
+    source: Annotated[Optional[FileSource], Field(description="URL or path to the source of the application")] = None
+    """The primary source of the application"""

bioimageio/spec/collection/v0_2.py

@@ -1,12 +1,12 @@
 import collections.abc
-from typing import Any, ClassVar, Dict, Literal, Optional, Tuple, Union
+from typing import Any, ClassVar, Dict, List, Literal, Optional, Sequence, Union
 
 from pydantic import (
     Field,
     PrivateAttr,
     TypeAdapter,
     field_validator,
-    model_validator,  # type: ignore
+    model_validator,
 )
 from pydantic import (
     HttpUrl as HttpUrl,
@@ -27,7 +27,7 @@
 from bioimageio.spec.generic.v0_2 import Badge as Badge
 from bioimageio.spec.generic.v0_2 import CiteEntry as CiteEntry
 from bioimageio.spec.generic.v0_2 import Generic as Generic
-from bioimageio.spec.generic.v0_2 import GenericBase
+from bioimageio.spec.generic.v0_2 import GenericBase, WithGenericFormatVersion
 from bioimageio.spec.generic.v0_2 import LinkedResource as LinkedResource
 from bioimageio.spec.generic.v0_2 import Maintainer as Maintainer
 from bioimageio.spec.model.v0_4 import Model as Model
@@ -111,24 +111,24 @@ def entry(self) -> EntryNode:
         return self._entry
 
 
-class Collection(GenericBase, extra="allow", title="bioimage.io collection specification"):
+class Collection(GenericBase, WithGenericFormatVersion, extra="allow", title="bioimage.io collection specification"):
     """A bioimage.io collection describes several other bioimage.io resources.
     Note that collections cannot be nested; resources listed under `collection` may not be collections themselves.
     """
 
     type: Literal["collection"] = "collection"
 
-    collection: NotEmpty[Tuple[CollectionEntry, ...]]
+    collection: NotEmpty[List[CollectionEntry]]
     """Collection entries"""
 
     @field_validator("collection")
     @classmethod
-    def check_unique_ids(cls, value: NotEmpty[Tuple[CollectionEntry, ...]]) -> NotEmpty[Tuple[CollectionEntry, ...]]:
+    def check_unique_ids(cls, value: NotEmpty[List[CollectionEntry]]) -> NotEmpty[List[CollectionEntry]]:
         cls.check_unique_ids_impl(value)
         return value
 
     @staticmethod
-    def check_unique_ids_impl(value: NotEmpty[Tuple[CollectionEntryBase, ...]]):
+    def check_unique_ids_impl(value: NotEmpty[Sequence[CollectionEntryBase]]):
         seen: Dict[str, int] = {}
         for i, v in enumerate(value):
             if v.id is None:

bioimageio/spec/collection/v0_3.py

@@ -1,4 +1,4 @@
-from typing import ClassVar, Literal, Tuple, Union
+from typing import ClassVar, List, Literal, Tuple, Union
 
 from pydantic import Field, TypeAdapter, field_validator
 from pydantic import HttpUrl as HttpUrl
@@ -17,7 +17,7 @@
 from bioimageio.spec.generic.v0_3 import Author as Author
 from bioimageio.spec.generic.v0_3 import Badge as Badge
 from bioimageio.spec.generic.v0_3 import CiteEntry as CiteEntry
-from bioimageio.spec.generic.v0_3 import Generic, GenericBase
+from bioimageio.spec.generic.v0_3 import Generic, GenericBase, WithGenericFormatVersion
 from bioimageio.spec.generic.v0_3 import LinkedResource as LinkedResource
 from bioimageio.spec.generic.v0_3 import Maintainer as Maintainer
 from bioimageio.spec.model.v0_4 import Model as Model04
@@ -55,7 +55,7 @@ def entry(self) -> EntryNode:
         return self._entry
 
 
-class Collection(GenericBase, extra="allow", title="bioimage.io collection specification"):
+class Collection(GenericBase, WithGenericFormatVersion, extra="allow", title="bioimage.io collection specification"):
     """A bioimage.io collection resource description file (collection RDF) describes a collection of bioimage.io
     resources.
     The resources listed in a collection RDF have types other than 'collection'; collections cannot be nested.
@@ -70,12 +70,12 @@ def _update_context(cls, context: InternalValidationContext, data: RdfContent) -
         assert "collection_base_content" not in context or context["collection_base_content"] == collection_base_content
         context["collection_base_content"] = collection_base_content
 
-    collection: NotEmpty[Tuple[CollectionEntry, ...]]
+    collection: NotEmpty[List[CollectionEntry]]
     """Collection entries"""
 
     @field_validator("collection")
     @classmethod
-    def check_unique_ids(cls, value: NotEmpty[Tuple[CollectionEntry, ...]]) -> NotEmpty[Tuple[CollectionEntry, ...]]:
+    def check_unique_ids(cls, value: NotEmpty[List[CollectionEntry]]) -> NotEmpty[List[CollectionEntry]]:
         v0_2.Collection.check_unique_ids_impl(value)
         return value
 

bioimageio/spec/dataset/v0_2.py

@@ -1,24 +1,31 @@
-from typing import Literal
+from typing import Literal, Optional
 
-from bioimageio.spec._internal.types import DatasetId
+from pydantic import Field
+from typing_extensions import Annotated
+
+from bioimageio.spec._internal.base_nodes import Node
+from bioimageio.spec._internal.types import DatasetId as DatasetId
+from bioimageio.spec._internal.types import FileSource as FileSource
 from bioimageio.spec.generic.v0_2 import Attachments as Attachments
 from bioimageio.spec.generic.v0_2 import Author as Author
 from bioimageio.spec.generic.v0_2 import Badge as Badge
 from bioimageio.spec.generic.v0_2 import CiteEntry as CiteEntry
-from bioimageio.spec.generic.v0_2 import GenericBase
-from bioimageio.spec.generic.v0_2 import LinkedResource as LinkedResource
+from bioimageio.spec.generic.v0_2 import GenericBase, WithGenericFormatVersion
 from bioimageio.spec.generic.v0_2 import Maintainer as Maintainer
 
 
-class Dataset(GenericBase, title="bioimage.io dataset specification"):
+class Dataset(GenericBase, WithGenericFormatVersion, title="bioimage.io dataset specification"):
     """A bioimage.io dataset resource description file (dataset RDF) describes a dataset relevant to bioimage
     processing.
     """
 
     type: Literal["dataset"] = "dataset"
 
+    source: Annotated[Optional[FileSource], Field(description="URL or path to the source of the dataset.")] = None
+    """The primary source of the dataset"""
+
 
-class LinkedDataset(LinkedResource):
+class LinkedDataset(Node):
     """Reference to a bioimage.io dataset."""
 
     id: DatasetId