Skip to content

Commit 9f25a33

Browse files
NEStockNEStock
authored
Merge pull request #154 from aplbrain/apl-setup
Update prod (Dec. 22 Week sync)
2 parents fb04eb2 + f9a51dd commit 9f25a33

38 files changed

+504
-442
lines changed

‎.github/workflows/backend-ci.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,9 @@ jobs:
5353
DJANGO_DANDI_WEB_APP_URL: http://localhost:8085
5454
DJANGO_DANDI_API_URL: http://localhost:8000
5555
DJANGO_DANDI_JUPYTERHUB_URL: https://hub.dandiarchive.org/
56+
DJANGO_DANDI_INSTANCE_NAME: DEV-EMBER-DANDI
57+
DJANGO_DANDI_INSTANCE_IDENTIFIER: "RRID:SCR_026700"
58+
DJANGO_DANDI_DOI_API_PREFIX: "10.82754"
5659
steps:
5760
- uses: actions/checkout@v5
5861
with:

‎.github/workflows/cli-integration.yml‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,15 +68,19 @@ jobs:
6868
if: matrix.dandi-version == 'release'
6969
run: >
7070
uvx --with dandi[test]
71-
pytest --pyargs -v --dandi-api dandi
71+
pytest --pyargs -v --dandi-api --deselect=tests/test_keyring.py::test_dandi_authenticate_no_env_var dandi
72+
# TODO: Revert back to
73+
# pytest --pyargs -v --dandi-api dandi
7274
env:
7375
DANDI_TESTS_PERSIST_DOCKER_COMPOSE: "1"
7476

7577
- name: Run dandi-api tests in dandi-cli
7678
if: matrix.dandi-version == 'prod'
7779
run: >
7880
uvx --with "dandi[test] @ git+https://github.com/dandi/dandi-cli"
79-
pytest --pyargs -v --dandi-api dandi
81+
pytest --pyargs -v --dandi-api --deselect=tests/test_keyring.py::test_dandi_authenticate_no_env_var dandi
82+
# TODO: Revert back to
83+
# pytest --pyargs -v --dandi-api dandi
8084
env:
8185
DANDI_TESTS_PERSIST_DOCKER_COMPOSE: "1"
8286

‎.github/workflows/frontend-ci.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,9 @@ jobs:
7878
DJANGO_DANDI_WEB_APP_URL: http://localhost:8085
7979
DJANGO_DANDI_API_URL: http://localhost:8000
8080
DJANGO_DANDI_JUPYTERHUB_URL: https://hub.dandiarchive.org/
81+
DJANGO_DANDI_INSTANCE_NAME: DEV-EMBER-DANDI
82+
DJANGO_DANDI_INSTANCE_IDENTIFIER: "RRID:SCR_026700"
83+
DJANGO_DANDI_DOI_API_PREFIX: "10.82754"
8184

8285
# Web client env vars
8386
VITE_APP_DANDI_API_ROOT: http://localhost:8000/api/

‎CHANGELOG.md‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,39 @@
1+
# v0.20.0 (Wed Dec 17 2025)
2+
3+
### Release Notes
4+
5+
#### Add vendorization support ([#2584](https://github.com/dandi/dandi-archive/pull/2584))
6+
7+
This PR makes the instance config as defined in `dandischema.conf`, per https://github.com/dandi/dandi-schema/pull/294, available through the `api/info/` endpoint and replaces the hardcoded `DAND:`, the hardcoded RRID, and the supported licenses with the corresponding attributes in schema instance config defined in `dandischema.conf`.
8+
9+
---
10+
11+
#### 🚀 Enhancement
12+
13+
- Add vendorization support [#2584](https://github.com/dandi/dandi-archive/pull/2584) ([@candleindark](https://github.com/candleindark) [@jjnesbitt](https://github.com/jjnesbitt))
14+
15+
#### 🐛 Bug Fix
16+
17+
- Empty commit for release [#2680](https://github.com/dandi/dandi-archive/pull/2680) ([@jjnesbitt](https://github.com/jjnesbitt))
18+
- Update management command for schema migration [#2650](https://github.com/dandi/dandi-archive/pull/2650) ([@jjnesbitt](https://github.com/jjnesbitt))
19+
20+
#### 🏠 Internal
21+
22+
- Remove `DANDI_SCHEMA_VERSION` as a setting [#2644](https://github.com/dandi/dandi-archive/pull/2644) ([@jjnesbitt](https://github.com/jjnesbitt))
23+
- Auto approve users from McGill University [#2657](https://github.com/dandi/dandi-archive/pull/2657) ([@kabilar](https://github.com/kabilar))
24+
- Don't use django setting to store allowed schema versions [#2643](https://github.com/dandi/dandi-archive/pull/2643) ([@jjnesbitt](https://github.com/jjnesbitt))
25+
- Add create_test_user management command [#2642](https://github.com/dandi/dandi-archive/pull/2642) ([@jjnesbitt](https://github.com/jjnesbitt))
26+
- Add docker volume for pre-commit cache [#2640](https://github.com/dandi/dandi-archive/pull/2640) ([@mvandenburgh](https://github.com/mvandenburgh))
27+
28+
#### Authors: 4
29+
30+
- Isaac To ([@candleindark](https://github.com/candleindark))
31+
- Jacob Nesbitt ([@jjnesbitt](https://github.com/jjnesbitt))
32+
- Kabilar Gunalan ([@kabilar](https://github.com/kabilar))
33+
- Mike VanDenburgh ([@mvandenburgh](https://github.com/mvandenburgh))
34+
35+
---
36+
137
# v0.19.0 (Thu Nov 13 2025)
238

339
### Release Notes

‎dandiapi/api/doi.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import logging
44
from typing import TYPE_CHECKING
55

6+
from dandischema.conf import get_instance_config
67
from django.conf import settings
78
import requests
89

@@ -29,10 +30,11 @@ def _generate_doi_data(version: Version):
2930

3031
publish = settings.DANDI_DOI_PUBLISH
3132
# Use the DANDI test datacite instance as a placeholder if PREFIX isn't set
32-
prefix = settings.DANDI_DOI_API_PREFIX or '10.80507'
33+
prefix = settings.DANDI_DOI_API_PREFIX
34+
instance_name: str = get_instance_config().instance_name
3335
dandiset_id = version.dandiset.identifier
3436
version_id = version.version
35-
doi = f'{prefix}/dandi.{dandiset_id}/{version_id}'
37+
doi = f'{prefix}/{instance_name.lower()}.{dandiset_id}/{version_id}'
3638
metadata = version.metadata
3739
metadata['doi'] = doi
3840
return (doi, to_datacite(metadata, publish=publish))

‎dandiapi/api/management/commands/create_dev_dandiset.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22

33
from uuid import uuid4
44

5-
from django.conf import settings
5+
from dandischema.conf import get_instance_config
6+
from dandischema.consts import DANDI_SCHEMA_VERSION
67
from django.contrib.auth.models import User
78
from django.core.files.uploadedfile import SimpleUploadedFile
89
import djclick as click
@@ -32,7 +33,7 @@ def create_dev_dandiset(*, name: str, email: str, num_extra_owners: int):
3233

3334
version_metadata = {
3435
'description': 'An informative description',
35-
'license': ['spdx:CC0-1.0'],
36+
'license': [sorted(x.value for x in get_instance_config().licenses)[0]],
3637
}
3738
dandiset, draft_version = create_open_dandiset(
3839
user=owner, version_name=name, version_metadata=version_metadata
@@ -68,7 +69,7 @@ def create_dev_dandiset(*, name: str, email: str, num_extra_owners: int):
6869
calculate_sha256(blob_id=asset_blob.blob_id)
6970
asset_blob.refresh_from_db()
7071
asset_metadata = {
71-
'schemaVersion': settings.DANDI_SCHEMA_VERSION,
72+
'schemaVersion': DANDI_SCHEMA_VERSION,
7273
'encodingFormat': 'text/plain',
7374
'schemaKey': 'Asset',
7475
'path': 'foo/bar.txt',

‎dandiapi/api/management/commands/create_test_user.py‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
from __future__ import annotations
2+
3+
from allauth.socialaccount.models import SocialAccount
4+
from django.contrib.auth.models import User
5+
from django.db import transaction
6+
import djclick as click
7+
import faker
8+
9+
from dandiapi.api.models.user import UserMetadata
10+
11+
12+
@click.command()
13+
@click.option(
14+
'--auto-approve',
15+
is_flag=True,
16+
help='Auto approve this user, skipping the questionnaire.',
17+
)
18+
@click.option(
19+
'--password',
20+
default='password',
21+
show_default=True,
22+
help='The password for this user.',
23+
)
24+
def create_test_user(*, auto_approve: bool, password: str):
25+
fake = faker.Faker()
26+
27+
with transaction.atomic():
28+
email = fake.email()
29+
user = User.objects.create(
30+
first_name=fake.first_name(),
31+
last_name=fake.last_name(),
32+
username=email,
33+
email=email,
34+
)
35+
36+
user.set_password(password)
37+
user.save()
38+
39+
UserMetadata.objects.create(
40+
user=user,
41+
status=UserMetadata.Status.APPROVED if auto_approve else UserMetadata.Status.INCOMPLETE,
42+
)
43+
44+
uid = fake.random_number(digits=8, fix_len=True)
45+
SocialAccount.objects.create(
46+
user=user,
47+
provider='github',
48+
uid=uid,
49+
extra_data={
50+
'id': uid,
51+
'name': user.get_full_name(),
52+
'email': user.email,
53+
'login': fake.user_name(),
54+
},
55+
)
56+
57+
click.echo(
58+
click.style(
59+
f'Created user "{user.email}" with password "{password}"',
60+
fg='green',
61+
bold=True,
62+
)
63+
)

‎dandiapi/api/management/commands/migrate_version_metadata.py‎

Lines changed: 79 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,90 @@
11
from __future__ import annotations
22

3-
from dandischema import migrate
3+
import logging
4+
5+
from dandischema.consts import DANDI_SCHEMA_VERSION
6+
from dandischema.metadata import migrate
7+
from django.db import transaction
48
import djclick as click
59

610
from dandiapi.api.models import Version
11+
from dandiapi.api.services import audit
12+
from dandiapi.api.services.metadata import validate_version_metadata
13+
14+
logger = logging.getLogger(__name__)
715

816

917
@click.command()
10-
@click.argument('to_version')
11-
def migrate_version_metadata(*, to_version: str):
12-
click.echo(f'Migrating all version metadata to version {to_version}')
13-
for version in Version.objects.filter(version='draft'):
14-
click.echo(f'Migrating {version.dandiset.identifier}/{version.version}')
15-
16-
metadata = version.metadata
17-
18-
try:
19-
metanew = migrate(metadata, to_version=to_version, skip_validation=True)
20-
except Exception as e: # noqa: BLE001
21-
click.echo(f'Failed to migrate {version.dandiset.identifier}/{version.version}')
22-
click.echo(e)
23-
continue
24-
25-
if version.metadata != metanew:
18+
@click.argument(
19+
'dandisets',
20+
type=click.INT,
21+
nargs=-1,
22+
)
23+
@click.option(
24+
'-a',
25+
'--all',
26+
'include_all',
27+
is_flag=True,
28+
help='Run on all dandisets.',
29+
)
30+
def migrate_version_metadata(dandisets: tuple[int, ...], *, include_all: bool):
31+
if bool(dandisets) == include_all:
32+
raise click.ClickException("Must specify exactly one of 'dandisets' or --all")
33+
34+
versions = Version.objects.filter(version='draft')
35+
if dandisets:
36+
versions = versions.filter(dandiset_id__in=dandisets)
37+
38+
logger.info(
39+
'Migrating %s dandiset draft versions to schema version %s',
40+
versions.count(),
41+
DANDI_SCHEMA_VERSION,
42+
)
43+
44+
migrated_count = 0
45+
failed_count = 0
46+
unchanged_count = 0
47+
for version in versions.iterator():
48+
logger.info('-----------------------------------------')
49+
logger.info('Migrating %s', version)
50+
51+
with transaction.atomic():
52+
locked_version = Version.objects.select_for_update().get(id=version.id)
53+
54+
try:
55+
metanew = migrate(locked_version.metadata, skip_validation=True)
56+
except Exception as e:
57+
logger.exception('Failed to migrate %s', version, exc_info=e)
58+
failed_count += 1
59+
continue
60+
61+
if locked_version.metadata == metanew:
62+
logger.info('No change in metadata for %s. Skipping save...', version)
63+
unchanged_count += 1
64+
continue
65+
2666
version.metadata = metanew
2767
version.status = Version.Status.PENDING
2868
version.save()
69+
70+
audit.update_metadata(
71+
dandiset=locked_version.dandiset,
72+
metadata=locked_version.metadata,
73+
user=None,
74+
admin=True,
75+
description=f'Update schema version to {DANDI_SCHEMA_VERSION}',
76+
)
77+
78+
migrated_count += 1
79+
logger.info('Metadata migrated for version %s', version)
80+
81+
# Validate outside of transaction, since this function uses `select_for_update` itself
82+
validate_version_metadata(version=version)
83+
84+
logger.info(
85+
'%d migrated, %d failed, %d left unchanged, out of %d total selected versions',
86+
migrated_count,
87+
failed_count,
88+
unchanged_count,
89+
versions.count(),
90+
)

‎dandiapi/api/models/metadata.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,17 @@
33
from typing import TYPE_CHECKING
44
from uuid import uuid4
55

6+
from dandischema.conf import get_instance_config
7+
68
if TYPE_CHECKING:
79
import datetime
810

911

1012
class PublishableMetadataMixin:
1113
@classmethod
1214
def published_by(cls, now: datetime.datetime):
15+
schema_config = get_instance_config()
16+
1317
return {
1418
'id': uuid4().urn,
1519
'name': 'DANDI publish',
@@ -20,8 +24,8 @@ def published_by(cls, now: datetime.datetime):
2024
'wasAssociatedWith': [
2125
{
2226
'id': uuid4().urn,
23-
'identifier': 'RRID:SCR_017571',
24-
'name': 'DANDI API',
27+
'identifier': schema_config.instance_identifier,
28+
'name': f'{schema_config.instance_name} API',
2529
# TODO: version the API
2630
'version': '0.1.0',
2731
'schemaKey': 'Software',

‎dandiapi/api/models/version.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import logging
55
from typing import TypedDict
66

7+
from dandischema.conf import get_instance_config
78
from dandischema.models import AccessType
89
from django.conf import settings
910
from django.contrib.postgres.indexes import HashIndex
@@ -229,6 +230,7 @@ def _populate_access_metadata(self):
229230
def _populate_metadata(self):
230231
from dandiapi.api.manifests import manifest_location
231232

233+
schema_config = get_instance_config()
232234
metadata = {
233235
**self.metadata,
234236
'@context': (
@@ -237,9 +239,9 @@ def _populate_metadata(self):
237239
),
238240
'manifestLocation': manifest_location(self),
239241
'name': self.name,
240-
'identifier': f'DANDI:{self.dandiset.identifier}',
242+
'identifier': (f'{schema_config.instance_name}:{self.dandiset.identifier}'),
241243
'version': self.version,
242-
'id': f'DANDI:{self.dandiset.identifier}/{self.version}',
244+
'id': (f'{schema_config.instance_name}:{self.dandiset.identifier}/{self.version}'),
243245
'repository': settings.DANDI_WEB_APP_URL,
244246
'url': (
245247
f'{settings.DANDI_WEB_APP_URL}/dandiset/{self.dandiset.identifier}/{self.version}'

0 commit comments

Comments
 (0)