From cd976b512e158045c1a24dd7efd38c8868fb1cec Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 17 Sep 2025 16:09:17 -0500 Subject: [PATCH 01/16] add array of keys to associations.coordsystem Co-authored-by: Ross Blair --- src/schema/associations.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/schema/associations.ts b/src/schema/associations.ts index 6849f6d8..677fe180 100644 --- a/src/schema/associations.ts +++ b/src/schema/associations.ts @@ -3,6 +3,7 @@ import type { Schema as MetaSchema } from '@bids/schema/metaschema' import type { BIDSFile } from '../types/filetree.ts' import type { BIDSContext } from './context.ts' +import { loadJSON } from '../files/json.ts' import { loadTSV } from '../files/tsv.ts' import { parseBvalBvec } from '../files/dwi.ts' import { walkBack } from '../files/inheritance.ts' @@ -85,6 +86,16 @@ const associationLookup = { sampling_frequency: columns.get('sampling_frequency'), } }, + coordsystem: async (file: BIDSFile, options: { maxRows: number }): Promise => { + const keys = Object.keys(await loadJSON(file, options.maxRows) + .catch((e) => { + return new Map() + })) + return { + path: file.path, + keys: keys, + } + }, } export async function buildAssociations( From 5c32a786d8992b713b6b87b8f787cb555c45cc94 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 17 Sep 2025 17:10:42 -0500 Subject: [PATCH 02/16] Apply suggestions from code review Co-authored-by: Chris Markiewicz --- src/schema/associations.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/schema/associations.ts b/src/schema/associations.ts index 677fe180..efe7178a 100644 --- a/src/schema/associations.ts +++ b/src/schema/associations.ts @@ -86,10 +86,10 @@ const associationLookup = { sampling_frequency: columns.get('sampling_frequency'), } }, - coordsystem: async (file: BIDSFile, options: { maxRows: number }): Promise => { + coordsystem: async (file: BIDSFile, options: { maxRows: number }): Promise<{path: string, keys: string[]}> => { const keys = Object.keys(await loadJSON(file, options.maxRows) .catch((e) => { - return new Map() + return [] })) return { path: file.path, From cd30d924a4d75310d4b9729b83441649a8f41e8e Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 17 Sep 2025 20:36:11 -0500 Subject: [PATCH 03/16] Update src/schema/associations.ts Co-authored-by: Chris Markiewicz --- src/schema/associations.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/schema/associations.ts b/src/schema/associations.ts index efe7178a..bb87deff 100644 --- a/src/schema/associations.ts +++ b/src/schema/associations.ts @@ -87,10 +87,7 @@ const associationLookup = { } }, coordsystem: async (file: BIDSFile, options: { maxRows: number }): Promise<{path: string, keys: string[]}> => { - const keys = Object.keys(await loadJSON(file, options.maxRows) - .catch((e) => { - return [] - })) + const keys = Object.keys(await loadJSON(file).catch((e) => {})) return { path: file.path, keys: keys, From 0c520204ed7484a729240c8306eb0eaa407c27da Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 17 Sep 2025 21:39:22 -0400 Subject: [PATCH 04/16] Update src/schema/associations.ts --- src/schema/associations.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/schema/associations.ts b/src/schema/associations.ts index bb87deff..2b9a1f8d 100644 --- a/src/schema/associations.ts +++ b/src/schema/associations.ts @@ -87,7 +87,7 @@ const associationLookup = { } }, coordsystem: async (file: BIDSFile, options: { maxRows: number }): Promise<{path: string, keys: string[]}> => { - const keys = Object.keys(await loadJSON(file).catch((e) => {})) + const keys = Object.keys(await loadJSON(file).catch((e) => {return {}})) return { path: file.path, keys: keys, From 68c5f65f19290ad30c1614d6d854e2d183f663cc Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 19 Sep 2025 10:29:42 -0400 Subject: [PATCH 05/16] feat: Enable multi-file associations, load coordsystems --- src/schema/associations.ts | 71 ++++++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 26 deletions(-) diff --git a/src/schema/associations.ts b/src/schema/associations.ts index 2b9a1f8d..3312b7f9 100644 --- a/src/schema/associations.ts +++ b/src/schema/associations.ts @@ -9,9 +9,13 @@ import { parseBvalBvec } from '../files/dwi.ts' import { walkBack } from '../files/inheritance.ts' import { evalCheck } from './applyRules.ts' import { expressionFunctions } from './expressionLanguage.ts' +import { readEntities } from './entities.ts' import { readText } from '../files/access.ts' +type LoadFunction = (file: BIDSFile, options: any) => Promise +type MultiLoadFunction = (files: BIDSFile[], options: any) => Promise + function defaultAssociation(file: BIDSFile, _options: any): Promise<{ path: string }> { return Promise.resolve({ path: file.path }) } @@ -24,7 +28,7 @@ function defaultAssociation(file: BIDSFile, _options: any): Promise<{ path: stri * * Many associations only consist of a path; this object is for more complex associations. */ -const associationLookup = { +const associationLookup: Record = { events: async (file: BIDSFile, options: { maxRows: number }): Promise => { const columns = await loadTSV(file, options.maxRows) .catch((e) => { @@ -86,11 +90,22 @@ const associationLookup = { sampling_frequency: columns.get('sampling_frequency'), } }, - coordsystem: async (file: BIDSFile, options: { maxRows: number }): Promise<{path: string, keys: string[]}> => { - const keys = Object.keys(await loadJSON(file).catch((e) => {return {}})) +} +const multiAssociationLookup: Record = { + coordsystems: async ( + files: BIDSFile[], + options: any, + ): Promise<{ paths: string[]; spaces: string[]; parents: string[] }> => { + const jsons = await Promise.allSettled( + files.map((f) => loadJSON(f).catch(() => ({} as Record))), + ) + const parents = jsons.map((j) => + j.status === 'fulfilled' ? j.value?.ParentCoordinateSystem : undefined + ).filter((p) => p) as string[] return { - path: file.path, - keys: keys, + paths: files.map((f) => f.path), + spaces: files.map((f) => readEntities(f.name).entities?.space), + ParentCoordinateSystems: parents, } }, } @@ -131,33 +146,37 @@ export async function buildAssociations( rule.target.suffix, rule.target?.entities ?? [], ).next().value - if (Array.isArray(file)) { - file = file[0] - } - } catch (error) { - if ( - error && typeof error === 'object' && 'code' in error && - error.code === 'MULTIPLE_INHERITABLE_FILES' - ) { - // @ts-expect-error + } catch (error: any) { + if (error?.code === 'MULTIPLE_INHERITABLE_FILES') { context.dataset.issues.add(error) - break + continue } else { throw error } } - if (file) { - // @ts-expect-error - const load = associationLookup[key] ?? defaultAssociation - // @ts-expect-error - associations[key] = await load(file, { maxRows: context.dataset.options?.maxRows }).catch( - (error: any) => { - if (error.code) { - context.dataset.issues.add({ ...error, location: file.path }) - } - }, - ) + if (file && !(Array.isArray(file) && file.length === 0)) { + const options = { maxRows: context.dataset.options?.maxRows } + if (key in multiAssociationLookup) { + const load = multiAssociationLookup[key] + if (!Array.isArray(file)) { + file = [file] + } + associations[key as keyof Associations] = await load(file, options) + } else { + const load = associationLookup[key] ?? defaultAssociation + if (Array.isArray(file)) { + file = file[0] + } + const location = file.path + associations[key as keyof Associations] = await load(file, { maxRows: context.dataset.options?.maxRows }).catch( + (error: any) => { + if (error.code) { + context.dataset.issues.add({ ...error, location }) + } + }, + ) + } } } return Promise.resolve(associations) From e6abc849fe91482868581574f23ce83a36ba84b8 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 19 Sep 2025 13:21:32 -0400 Subject: [PATCH 06/16] fix: Exact match check must include target entities --- src/files/inheritance.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/files/inheritance.ts b/src/files/inheritance.ts index 5cf4f656..ca7c8632 100644 --- a/src/files/inheritance.ts +++ b/src/files/inheritance.ts @@ -50,7 +50,7 @@ export function* walkBack( if (candidates.length > 1) { const exactMatch = candidates.find((file) => { const { entities } = readEntities(file.name) - return Object.keys(sourceParts.entities).every((entity) => + return [...Object.keys(sourceParts.entities), ...targetEntities].every((entity) => entities[entity] === sourceParts.entities[entity] ) }) From 60b5168f0cc5de3fc2d71678b3200ec9255e034c Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 19 Sep 2025 13:22:45 -0400 Subject: [PATCH 07/16] hack: Load datatype before associations --- src/schema/context.ts | 2 ++ src/validators/filenameIdentify.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/schema/context.ts b/src/schema/context.ts index 92f9a51c..adca1ea0 100644 --- a/src/schema/context.ts +++ b/src/schema/context.ts @@ -25,6 +25,7 @@ import { loadHeader } from '../files/nifti.ts' import { buildAssociations } from './associations.ts' import type { ValidatorOptions } from '../setup/options.ts' import { logger } from '../utils/logger.ts' +import { datatypeFromDirectory } from '../validators/filenameIdentify.ts' export class BIDSContextDataset implements Dataset { #dataset_description: Record = {} @@ -162,6 +163,7 @@ export class BIDSContext implements Context { this.columns = new ColumnsMap() as Record this.json = {} this.associations = {} as Associations + datatypeFromDirectory(this.dataset.schema, this) } get schema(): Schema { diff --git a/src/validators/filenameIdentify.ts b/src/validators/filenameIdentify.ts index 25df198e..10b5a240 100644 --- a/src/validators/filenameIdentify.ts +++ b/src/validators/filenameIdentify.ts @@ -20,7 +20,7 @@ import type { CheckFunction } from '../types/check.ts' import { lookupEntityLiteral } from './filenameValidate.ts' const CHECKS: CheckFunction[] = [ - datatypeFromDirectory, + // datatypeFromDirectory, findRuleMatches, hasMatch, cleanContext, From 5757fc030056ce938ee360158c9f1cb268a15ece Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 19 Sep 2025 14:08:56 -0400 Subject: [PATCH 08/16] fix: Type checking errors --- src/files/inheritance.ts | 2 +- src/schema/associations.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/files/inheritance.ts b/src/files/inheritance.ts index ca7c8632..5b082343 100644 --- a/src/files/inheritance.ts +++ b/src/files/inheritance.ts @@ -50,7 +50,7 @@ export function* walkBack( if (candidates.length > 1) { const exactMatch = candidates.find((file) => { const { entities } = readEntities(file.name) - return [...Object.keys(sourceParts.entities), ...targetEntities].every((entity) => + return [...Object.keys(sourceParts.entities), ...(targetEntities ?? [])].every((entity) => entities[entity] === sourceParts.entities[entity] ) }) diff --git a/src/schema/associations.ts b/src/schema/associations.ts index 3312b7f9..60214367 100644 --- a/src/schema/associations.ts +++ b/src/schema/associations.ts @@ -95,7 +95,7 @@ const multiAssociationLookup: Record = { coordsystems: async ( files: BIDSFile[], options: any, - ): Promise<{ paths: string[]; spaces: string[]; parents: string[] }> => { + ): Promise<{ paths: string[]; spaces: string[]; ParentCoordinateSystems: string[] }> => { const jsons = await Promise.allSettled( files.map((f) => loadJSON(f).catch(() => ({} as Record))), ) From e6de7389e890ea4c65178052515a3c272932df59 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 19 Sep 2025 14:09:17 -0400 Subject: [PATCH 09/16] fix: Fallout from moving datatype parsing --- src/schema/context.ts | 4 +++- src/validators/validateFiles.test.ts | 20 +++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/schema/context.ts b/src/schema/context.ts index adca1ea0..c6164f39 100644 --- a/src/schema/context.ts +++ b/src/schema/context.ts @@ -163,7 +163,9 @@ export class BIDSContext implements Context { this.columns = new ColumnsMap() as Record this.json = {} this.associations = {} as Associations - datatypeFromDirectory(this.dataset.schema, this) + if (this.dataset.schema.objects) { // schema may be empty for some tests + datatypeFromDirectory(this.dataset.schema, this) + } } get schema(): Schema { diff --git a/src/validators/validateFiles.test.ts b/src/validators/validateFiles.test.ts index da28a5be..f1094d18 100644 --- a/src/validators/validateFiles.test.ts +++ b/src/validators/validateFiles.test.ts @@ -1,18 +1,19 @@ import { assert, assertEquals } from '@std/assert' import { filenameIdentify } from './filenameIdentify.ts' import { filenameValidate } from './filenameValidate.ts' -import { BIDSContext } from '../schema/context.ts' +import { BIDSContext, BIDSContextDataset } from '../schema/context.ts' import { loadSchema } from '../setup/loadSchema.ts' import type { GenericSchema, Schema } from '../types/schema.ts' import type { DatasetIssues } from '../issues/datasetIssues.ts' import { pathToFile } from '../files/filetree.ts' -const schema = await loadSchema() as unknown as GenericSchema +const schema = await loadSchema() -function validatePath(path: string): DatasetIssues { - const context = new BIDSContext(pathToFile(path)) - filenameIdentify(schema, context) - filenameValidate(schema, context) +async function validatePath(path: string): Promise { + const dataset = new BIDSContextDataset({ schema }) + const context = new BIDSContext(pathToFile(path), dataset) + await filenameIdentify(schema, context) + await filenameValidate(schema as unknown as GenericSchema, context) return context.dataset.issues } @@ -54,7 +55,7 @@ Deno.test('test valid paths', async (t) => { ] for (const filename of validFiles) { await t.step(filename, async () => { - const issues = validatePath(filename) + const issues = await validatePath(filename) assertEquals( issues.get({ location: filename }).length, 0, @@ -111,9 +112,10 @@ Deno.test('test invalid paths', async (t) => { ] for (const filename of invalidFiles) { await t.step(filename, async () => { - const context = new BIDSContext(pathToFile(filename)) + const dataset = new BIDSContextDataset({ schema }) + const context = new BIDSContext(pathToFile(filename), dataset) await filenameIdentify(schema, context) - await filenameValidate(schema, context) + await filenameValidate(schema as unknown as GenericSchema, context) assert( context.dataset.issues.get({ location: context.file.path, From 79248e84b54d2a604dbfd8d50263492875b41a0c Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Fri, 19 Sep 2025 17:30:43 -0500 Subject: [PATCH 10/16] changelog --- ...9_172910_dan_add_keys_to_assoc_coordsys.md | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 changelog.d/20250919_172910_dan_add_keys_to_assoc_coordsys.md diff --git a/changelog.d/20250919_172910_dan_add_keys_to_assoc_coordsys.md b/changelog.d/20250919_172910_dan_add_keys_to_assoc_coordsys.md new file mode 100644 index 00000000..271e3ade --- /dev/null +++ b/changelog.d/20250919_172910_dan_add_keys_to_assoc_coordsys.md @@ -0,0 +1,48 @@ + + + +### Added + +- New `associations.coordsystems` to collate coordsystem files. + + + + + + + From 50337ca056721c0a52773799a7734059632aec3d Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Thu, 25 Sep 2025 15:35:49 -0400 Subject: [PATCH 11/16] Update changelog.d/20250919_172910_dan_add_keys_to_assoc_coordsys.md --- changelog.d/20250919_172910_dan_add_keys_to_assoc_coordsys.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/changelog.d/20250919_172910_dan_add_keys_to_assoc_coordsys.md b/changelog.d/20250919_172910_dan_add_keys_to_assoc_coordsys.md index 271e3ade..3c9307fa 100644 --- a/changelog.d/20250919_172910_dan_add_keys_to_assoc_coordsys.md +++ b/changelog.d/20250919_172910_dan_add_keys_to_assoc_coordsys.md @@ -8,7 +8,8 @@ For top level release notes, leave all the headers commented out. ### Added -- New `associations.coordsystems` to collate coordsystem files. +- Implement `associations.coordsystems` to collate multiple `coordsystem.json` files, + as required by BEP 042 (EMG).