Add vendorization support (#2584)

* feat: have the `/info/` endpoint provide instance config info

* test: update `test_rest_info()`

To include the output of instance config
value

* tmp: temporarily set the dandischema dependency to the devendorize branch

* test: provide env var to set schema config instance name

* feat: make dandischema instance config available through Django settings

* feat: provide instance name through dandischema instance config

instead of a hardcoded value

* feat: provide instance name through dandischema instance config

instead of a hardcoded value

* feat: replace `"DANDI` with instance name in `DandisetList` vue component

* ci: add DJANGO_DANDI_INSTANCE_NAME env var to frontend CI

* feat: replace hardcoded RRID with one set in schema instance config

* ci: add instance identifier to backend CI env

* feat: replace the use of hardcoded license

* feat: use dandiarchive's version in producing `PublishActivity` metadata

Per @yarikoptic this is an improvement over
the hardcoded '0.1.0'

* feat: vendorize software name in `PublishActivity` metadata

* test: update tests for the `PublishActivity` metadata changes

* style: replace use of `dandiapi.__version__` with `importlib.metadata.version`

Co-authored-by: Mike VanDenburgh <37340715+mvandenburgh@users.noreply.github.com>

* rf: remove outdated patching of JSON schema

For discussion about this removal,
see #2584 (comment) for details

* style: replace use of `dandiapi.__version__` with `importlib.metadata.version` in tests

* rf: import `dandischema.conf.get_instance_config` as `get_schema_instance_config`

Importing the function under this name
emphasizes the returned config pertains
to dandischema and its usage

* feat: call avoid using `dandiapi/settings/base.py` to store schema instance config

`dandiapi/settings/base.py` is reserved for used for
configuring settings, possibly for the Django app alone.
See #2584 (comment) for rationale for the change.

* revert: revert naming of API service in generated asset metadata

* Always use first sorted license value in create_dev_dandiset

* Don't rename `get_instance_config` to `get_schema_instance_config`

* Don't use global variables for instance config

* Reuse identical schema object for tests

* Require DJANGO_DANDI_INSTANCE_IDENTIFIER env var

* Use dummy instance config values for local dev and CI

* Revert change of version value in PublishableMetadataMixin

* Update tests regarding Pydantic and None values

This more directly tests the issue we care about. As it was written, the
test could still pass unintentionally, if the default values were the
same as the values we had set them to.

* Require DJANGO_DANDI_DOI_API_PREFIX env var

* Use schema vendorization for DOIs

* Use schema vendorization for id/identifier

* Revert unnecessary change to `$route.query`

* Use function to supply default license value

* Fix linting errors

* Fix malformed license field in create_dev_dandiset

* Move `get_default_license` to conftest.py and rename

* Set dandischema dependency to 0.12.0

---------

Co-authored-by: Mike VanDenburgh <37340715+mvandenburgh@users.noreply.github.com>
Co-authored-by: Jacob Nesbitt <jjnesbitt2@gmail.com>

Author: 3 people

.github/workflows/backend-ci.yml

@@ -53,6 +53,9 @@ jobs:
       DJANGO_DANDI_WEB_APP_URL: http://localhost:8085
       DJANGO_DANDI_API_URL: http://localhost:8000
       DJANGO_DANDI_JUPYTERHUB_URL: https://hub.dandiarchive.org/
+      DJANGO_DANDI_INSTANCE_NAME: DEV-DANDI
+      DJANGO_DANDI_INSTANCE_IDENTIFIER: "RRID:ABC_123456"
+      DJANGO_DANDI_DOI_API_PREFIX: "10.80507"
     steps:
       - uses: actions/checkout@v5
         with:

.github/workflows/frontend-ci.yml

@@ -78,6 +78,9 @@ jobs:
       DJANGO_DANDI_WEB_APP_URL: http://localhost:8085
       DJANGO_DANDI_API_URL: http://localhost:8000
       DJANGO_DANDI_JUPYTERHUB_URL: https://hub.dandiarchive.org/
+      DJANGO_DANDI_INSTANCE_NAME: DEV-DANDI
+      DJANGO_DANDI_INSTANCE_IDENTIFIER: "RRID:ABC_123456"
+      DJANGO_DANDI_DOI_API_PREFIX: "10.80507"
 
       # Web client env vars
       VITE_APP_DANDI_API_ROOT: http://localhost:8000/api/

dandiapi/api/doi.py

@@ -3,6 +3,7 @@
 import logging
 from typing import TYPE_CHECKING
 
+from dandischema.conf import get_instance_config
 from django.conf import settings
 import requests
 
@@ -29,10 +30,11 @@ def _generate_doi_data(version: Version):
 
     publish = settings.DANDI_DOI_PUBLISH
     # Use the DANDI test datacite instance as a placeholder if PREFIX isn't set
-    prefix = settings.DANDI_DOI_API_PREFIX or '10.80507'
+    prefix = settings.DANDI_DOI_API_PREFIX
+    instance_name: str = get_instance_config().instance_name
     dandiset_id = version.dandiset.identifier
     version_id = version.version
-    doi = f'{prefix}/dandi.{dandiset_id}/{version_id}'
+    doi = f'{prefix}/{instance_name.lower()}.{dandiset_id}/{version_id}'
     metadata = version.metadata
     metadata['doi'] = doi
     return (doi, to_datacite(metadata, publish=publish))

dandiapi/api/management/commands/create_dev_dandiset.py

@@ -2,6 +2,7 @@
 
 from uuid import uuid4
 
+from dandischema.conf import get_instance_config
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.core.files.uploadedfile import SimpleUploadedFile
@@ -32,7 +33,7 @@ def create_dev_dandiset(*, name: str, email: str, num_extra_owners: int):
 
     version_metadata = {
         'description': 'An informative description',
-        'license': ['spdx:CC0-1.0'],
+        'license': [sorted(x.value for x in get_instance_config().licenses)[0]],
     }
     dandiset, draft_version = create_open_dandiset(
         user=owner, version_name=name, version_metadata=version_metadata

dandiapi/api/models/metadata.py

@@ -3,13 +3,17 @@
 from typing import TYPE_CHECKING
 from uuid import uuid4
 
+from dandischema.conf import get_instance_config
+
 if TYPE_CHECKING:
     import datetime
 
 
 class PublishableMetadataMixin:
     @classmethod
     def published_by(cls, now: datetime.datetime):
+        schema_config = get_instance_config()
+
         return {
             'id': uuid4().urn,
             'name': 'DANDI publish',
@@ -20,8 +24,8 @@ def published_by(cls, now: datetime.datetime):
             'wasAssociatedWith': [
                 {
                     'id': uuid4().urn,
-                    'identifier': 'RRID:SCR_017571',
-                    'name': 'DANDI API',
+                    'identifier': schema_config.instance_identifier,
+                    'name': f'{schema_config.instance_name} API',
                     # TODO: version the API
                     'version': '0.1.0',
                     'schemaKey': 'Software',

dandiapi/api/models/version.py

@@ -4,6 +4,7 @@
 import logging
 from typing import TypedDict
 
+from dandischema.conf import get_instance_config
 from dandischema.models import AccessType
 from django.conf import settings
 from django.contrib.postgres.indexes import HashIndex
@@ -229,6 +230,7 @@ def _populate_access_metadata(self):
     def _populate_metadata(self):
         from dandiapi.api.manifests import manifest_location
 
+        schema_config = get_instance_config()
         metadata = {
             **self.metadata,
             '@context': (
@@ -237,9 +239,9 @@ def _populate_metadata(self):
             ),
             'manifestLocation': manifest_location(self),
             'name': self.name,
-            'identifier': f'DANDI:{self.dandiset.identifier}',
+            'identifier': (f'{schema_config.instance_name}:{self.dandiset.identifier}'),
             'version': self.version,
-            'id': f'DANDI:{self.dandiset.identifier}/{self.version}',
+            'id': (f'{schema_config.instance_name}:{self.dandiset.identifier}/{self.version}'),
             'repository': settings.DANDI_WEB_APP_URL,
             'url': (
                 f'{settings.DANDI_WEB_APP_URL}/dandiset/{self.dandiset.identifier}/{self.version}'

dandiapi/api/services/metadata/__init__.py

@@ -3,6 +3,7 @@
 from typing import TYPE_CHECKING
 
 from celery.utils.log import get_task_logger
+from dandischema.conf import get_instance_config
 import dandischema.exceptions
 from dandischema.metadata import aggregate_assets_summary, validate
 from django.conf import settings
@@ -108,6 +109,8 @@ def version_aggregate_assets_summary(version: Version) -> None:
 
 def validate_version_metadata(*, version: Version) -> None:
     def _build_validatable_version_metadata(version: Version) -> dict:
+        schema_config = get_instance_config()
+
         # since Version.Status.VALID is a proxy for a version being publishable, we need to
         # validate against the PublishedDandiset schema even though we lack several things
         # at validation time: id, url, doi, and assetsSummary. this tricks the validator into
@@ -116,13 +119,16 @@ def _build_validatable_version_metadata(version: Version) -> dict:
         metadata_for_validation = publishable_version.metadata
 
         metadata_for_validation['id'] = (
-            f'DANDI:{publishable_version.dandiset.identifier}/{publishable_version.version}'
+            f'{schema_config.instance_name}:'
+            f'{publishable_version.dandiset.identifier}/{publishable_version.version}'
         )
         metadata_for_validation['url'] = (
             f'{settings.DANDI_WEB_APP_URL}/dandiset/'
             f'{publishable_version.dandiset.identifier}/{publishable_version.version}'
         )
-        metadata_for_validation['doi'] = '10.80507/dandi.123456/0.123456.1234'
+        metadata_for_validation['doi'] = (
+            f'{schema_config.doi_prefix}/{schema_config.instance_name.lower()}.123456/0.123456.1234'
+        )
         metadata_for_validation['assetsSummary'] = {
             'schemaKey': 'AssetsSummary',
             'numberOfBytes': 1

dandiapi/api/services/publish/__init__.py

@@ -4,6 +4,7 @@
 import datetime
 from typing import TYPE_CHECKING
 
+from dandischema.conf import get_instance_config
 from dandischema.metadata import aggregate_assets_summary, validate
 from django.contrib.auth.models import User
 from django.db import transaction
@@ -182,7 +183,10 @@ def _publish_dandiset(dandiset_id: int, user_id: int) -> None:
         old_version.save()
 
         # Inject a dummy DOI so the metadata is valid
-        new_version.metadata['doi'] = '10.80507/dandi.123456/0.123456.1234'
+        schema_config = get_instance_config()
+        new_version.metadata['doi'] = (
+            f'{schema_config.doi_prefix}/{schema_config.instance_name.lower()}.123456/0.123456.1234'
+        )
 
         validate(new_version.metadata, schema_key='PublishedDandiset', json_validation=True)
 

dandiapi/api/tests/factories.py

@@ -4,6 +4,7 @@
 import hashlib
 
 from allauth.socialaccount.models import SocialAccount
+from dandischema.conf import get_instance_config
 from dandischema.models import AccessType
 from django.conf import settings
 from django.contrib.auth.models import User
@@ -113,6 +114,8 @@ class Meta:
 
     @factory.lazy_attribute
     def metadata(self) -> dict:
+        from dandiapi.conftest import get_first_allowed_license
+
         metadata = {
             **faker.Faker().pydict(value_types=['str', 'float', 'int']),
             'schemaVersion': settings.DANDI_SCHEMA_VERSION,
@@ -134,7 +137,7 @@ def metadata(self) -> dict:
                     'schemaKey': 'Person',
                 }
             ],
-            'license': ['spdx:CC0-1.0'],
+            'license': [get_first_allowed_license()],
         }
         # Remove faked data that might conflict with the schema types
         for key in ['about']:
@@ -162,7 +165,9 @@ class DraftVersionFactory(BaseVersionFactory):
 
 class PublishedVersionFactory(BaseVersionFactory):
     doi = factory.LazyAttribute(
-        lambda self: f'10.80507/dandi.{self.dandiset.identifier}/{self.version}'
+        lambda self: f'{get_instance_config().doi_prefix}/'
+        f'{get_instance_config().instance_name}.'
+        f'{self.dandiset.identifier}/{self.version}'
     )
     status = Version.Status.PUBLISHED
 

dandiapi/api/tests/fuzzy.py

@@ -2,6 +2,8 @@
 
 import re
 
+from dandischema.conf import get_instance_config
+
 
 class Re:
     def __init__(self, pattern):
@@ -23,12 +25,22 @@ def __hash__(self):
         return hash(self.pattern)
 
 
+schema_config = get_instance_config()
+
 TIMESTAMP_RE = Re(r'\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}\.\d{6}Z')
 UTC_ISO_TIMESTAMP_RE = Re(r'\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}\.\d{6}\+[0-9]{2}:[0-9]{2}')
 DATE_RE = Re(r'\d{4}-\d{2}-\d{2}')
 DANDISET_ID_RE = Re(r'\d{6}')
-DANDISET_SCHEMA_ID_RE = Re(r'DANDI:\d{6}')
+DANDISET_SCHEMA_ID_RE = Re(rf'{schema_config.instance_name}:\d{{6}}')
 VERSION_ID_RE = Re(r'0\.\d{6}\.\d{4}')
 HTTP_URL_RE = Re(r'http[s]?\://[^/]+(/[^/]+)*[/]?(&.+)?')
 UUID_RE = Re(r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}')
 URN_RE = Re(r'urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}')
+
+DEFAULT_WAS_ASSOCIATED_WITH = {
+    'id': URN_RE,
+    'identifier': schema_config.instance_identifier,
+    'name': f'{schema_config.instance_name} API',
+    'version': '0.1.0',
+    'schemaKey': 'Software',
+}