Merge pull request #1732 from dandi/rf-schema-version

Allow for client to provide records with newer COMPATIBLE or older (upgradable) version of schema to the server

Author: yarikoptic

dandi/dandiapi.py

@@ -18,6 +18,8 @@
 
 import click
 from dandischema import models
+import dandischema.consts
+from packaging.version import Version as PackagingVersion
 from pydantic import BaseModel, Field, PrivateAttr
 import requests
 import tenacity
@@ -646,12 +648,21 @@ def create_dandiset(
 
     def check_schema_version(self, schema_version: str | None = None) -> None:
         """
-        Confirms that the server is using the same version of the DANDI schema
-        as the client.  If it is not, a `SchemaVersionError` is raised.
+        Confirms that the given schema version at the client is "compatible" with the server.
 
-        :param schema_version: the schema version to confirm that the server
-            uses; if not set, the schema version for the installed
-            ``dandischema`` library is used
+        Compatibility here means that the server's schema version can be either
+
+        - lower than client has, but within the same MAJOR.MINOR component of the version
+          number for 0.x series, and same MAJOR version for/after 1.x series;
+        - the same;
+        - higher than the client has, but only if the client's schema version is listed
+          among the server's `allowed_schema_versions` (as returned by the `/info` API endpoint),
+          or if not there -- `dandischema.consts.ALLOWED_INPUT_SCHEMAS` is consulted.
+
+        If neither of above, a `SchemaVersionError` is raised.
+
+        :param schema_version: the schema version to be confirmed for compatibility with the server;
+           if not set, the schema version for the installed ``dandischema`` library is used.
         """
         if schema_version is None:
             schema_version = models.get_schema_version()
@@ -662,11 +673,48 @@ def check_schema_version(self, schema_version: str | None = None) -> None:
                 "Server did not provide schema_version in /info/;"
                 f" returned {server_info!r}"
             )
-        if server_schema_version != schema_version:
+        server_ver, our_ver = PackagingVersion(server_schema_version), PackagingVersion(
+            schema_version
+        )
+        if server_ver > our_ver:
+            # TODO: potentially adjust here if name would be different: see
+            # https://github.com/dandi/dandi-archive/issues/2624
+            allowed_schema_versions = server_info.get(
+                "allowed_schema_versions", dandischema.consts.ALLOWED_INPUT_SCHEMAS
+            )
+            if schema_version not in allowed_schema_versions:
+                raise SchemaVersionError(
+                    f"Server uses schema version {server_schema_version};"
+                    f" client only supports prior {schema_version} and it"
+                    f" is not among any of the allowed upgradable schema versions"
+                    f" ({', '.join(allowed_schema_versions)}) .  You may need to"
+                    " upgrade dandi and/or dandischema."
+                )
+
+            # TODO: check current server behavior which is likely to just not care!
+            # So that is where server might need to provide support for upgrades upon
+            # providing metadata.
+        elif (
+            server_ver.major == 0 and server_ver.release[:2] != our_ver.release[:2]
+        ) or (
+            server_ver.major != our_ver.major
+        ):  # MAJOR, MINOR within 0.x.y and MAJOR within 1.x.y
             raise SchemaVersionError(
-                f"Server requires schema version {server_schema_version};"
-                f" client only supports {schema_version}.  You may need to"
-                " upgrade dandi and/or dandischema."
+                f"Server uses older incompatible schema version {server_schema_version};"
+                f" client supports {schema_version}."
+            )
+        elif server_ver < our_ver:
+            # Compatible older server version -- all good, but inform the user
+            # TODO: potentially downgrade the record to match the schema,
+            #       see https://github.com/dandi/dandi-schema/issues/343
+            lgr.warning(
+                "Server uses schema version %s older than client's %s (dandischema library %s). "
+                "Server might fail to validate such assets and you might not be able to "
+                "publish this dandiset until server is upgraded. "
+                "Alternatively, you may downgrade dandischema and reupload.",
+                server_ver,
+                our_ver,
+                dandischema.__version__,
             )
 
     def get_asset(self, asset_id: str) -> BaseRemoteAsset:

dandi/tests/test_dandiapi.py

@@ -272,35 +272,62 @@ def test_remote_asset_json_dict(text_dandiset: SampleDandiset) -> None:
     }
 
 
+@pytest.mark.parametrize(
+    "server_schema_version,local_schema_version,should_raise,expected_message_start",
+    [
+        # Current/identity matching -- always ok
+        (get_schema_version(), None, False, None),
+        (get_schema_version(), get_schema_version(), False, None),
+        ("0.6.7", "0.6.7", False, None),
+        # Less - is not good since we  might be missing fields and it is easy
+        # for client to upgrade.
+        (
+            "4.5.6",
+            "4.5.5",
+            True,
+            "Server uses schema version 4.5.6; client only supports prior 4.5.5 "
+            "and it is not among any of the allowed upgradable schema versions",
+        ),
+        (
+            "0.6.7",
+            "0.3.0",
+            True,
+            "Server uses schema version 0.6.7; client only supports prior 0.3.0",
+        ),
+        (
+            "0.6.7",
+            "0.6.6",  # can be upgraded and thus uploaded!
+            False,
+            None,
+        ),
+        # Now - incompatible, for 0.x -- rely on MAJOR.MINOR
+        (
+            "0.6.7",
+            "0.7.0",
+            True,
+            "Server uses older incompatible schema version 0.6.7; client supports 0.7.0",
+        ),
+        # After 1.x -- rely on MAJOR.
+        ("1.0.0", "1.2.3", False, None),
+        ("1.6.7", "1.7.0", False, None),
+        ("1.6.7", "1.8.3", False, None),
+        (
+            "1.6.7",
+            "2.7.0",
+            True,
+            "Server uses older incompatible schema version 1.6.7; client supports 2.7.0",
+        ),
+    ],
+)
 @responses.activate
-def test_check_schema_version_matches_default() -> None:
-    server_info = {
-        "schema_version": get_schema_version(),
-        "version": "0.0.0",
-        "services": {
-            "api": {"url": "https://test.nil/api"},
-        },
-        "cli-minimal-version": "0.0.0",
-        "cli-bad-versions": [],
-    }
-    responses.add(
-        responses.GET,
-        "https://test.nil/server-info",
-        json=server_info,
-    )
-    responses.add(
-        responses.GET,
-        "https://test.nil/api/info/",
-        json=server_info,
-    )
-    client = DandiAPIClient("https://test.nil/api")
-    client.check_schema_version()
-
-
-@responses.activate
-def test_check_schema_version_mismatch() -> None:
+def test_check_schema_version(
+    server_schema_version: str,
+    local_schema_version: str | None,
+    should_raise: bool,
+    expected_message_start: str | None,
+) -> None:
     server_info = {
-        "schema_version": "4.5.6",
+        "schema_version": server_schema_version,
         "version": "0.0.0",
         "services": {
             "api": {"url": "https://test.nil/api"},
@@ -319,13 +346,13 @@ def test_check_schema_version_mismatch() -> None:
         json=server_info,
     )
     client = DandiAPIClient("https://test.nil/api")
-    with pytest.raises(SchemaVersionError) as excinfo:
-        client.check_schema_version("1.2.3")
-    assert (
-        str(excinfo.value)
-        == "Server requires schema version 4.5.6; client only supports 1.2.3.  "
-        "You may need to upgrade dandi and/or dandischema."
-    )
+    if should_raise:
+        with pytest.raises(SchemaVersionError) as excinfo:
+            client.check_schema_version(local_schema_version)
+        if expected_message_start:
+            assert str(excinfo.value).startswith(expected_message_start)
+    else:
+        client.check_schema_version(local_schema_version)
 
 
 def test_get_dandisets(text_dandiset: SampleDandiset) -> None: