feat: Validate directories, do not recurse into schema-declared opaque dirs

src/schema/context.ts

@@ -25,6 +25,7 @@ import { buildAssociations } from './associations.ts'
 import type { ValidatorOptions } from '../setup/options.ts'
 import { logger } from '../utils/logger.ts'
 
+
 export class BIDSContextDataset implements Dataset {
   #dataset_description: Record<string, unknown> = {}
   tree: FileTree
@@ -38,6 +39,7 @@ export class BIDSContextDataset implements Dataset {
   options?: ValidatorOptions
   schema: Schema
   pseudofileExtensions: Set<string>
+  opaqueDirectories: Set<string>
 
   // Opaque object for HED validator
   hedSchemas: object | undefined | null = undefined
@@ -63,13 +65,21 @@ export class BIDSContextDataset implements Dataset {
           ?.filter((ext) => ext.endsWith('/'))
         : [],
     )
+    this.opaqueDirectories = new Set<string>(
+      args.schema
+        ? Object.values(this.schema.rules.directories.raw)
+          ?.filter((rule) => rule?.opaque && 'name' in rule)
+          ?.map((dir) => `/${dir.name}`)
+        : [],
+    )
     // @ts-ignore
     this.subjects = args.subjects || null
   }
 
   get dataset_description(): Record<string, unknown> {
     return this.#dataset_description
   }
+
   set dataset_description(value: Record<string, unknown>) {
     this.#dataset_description = value
     if (!this.dataset_description.DatasetType) {
@@ -87,6 +97,10 @@ export class BIDSContextDataset implements Dataset {
       this.pseudofileExtensions.has(`${extension}/`)
     )
   }
+  isOpaqueDirectory(file: FileTree): boolean {
+    return this.opaqueDirectories.has(file.path)
+  }
+
 }
 
 class BIDSContextDatasetSubjects implements Subjects {
@@ -123,6 +137,7 @@ export class BIDSContext implements Context {
   nifti_header?: NiftiHeader
   ome?: Ome
   tiff?: Tiff
+  directory?: boolean
 
   file: BIDSFile
   filenameRules: string[]

src/schema/walk.test.ts

@@ -3,10 +3,13 @@ import { BIDSContext, BIDSContextDataset } from './context.ts'
 import { walkFileTree } from './walk.ts'
 import type { DatasetIssues } from '../issues/datasetIssues.ts'
 import { simpleDataset, simpleDatasetFileCount } from '../tests/simple-dataset.ts'
+import { pathsToTree } from '../files/filetree.ts'
+import { loadSchema } from '../setup/loadSchema.ts'
 
 Deno.test('file tree walking', async (t) => {
+  const schema = await loadSchema()
   await t.step('visits each file and creates a BIDSContext', async () => {
-    const dsContext = new BIDSContextDataset({ tree: simpleDataset })
+    const dsContext = new BIDSContextDataset({ tree: simpleDataset, schema: schema })
     for await (const context of walkFileTree(dsContext)) {
       assert(
         context instanceof BIDSContext,
@@ -15,19 +18,41 @@ Deno.test('file tree walking', async (t) => {
     }
   })
   await t.step('visits every file expected', async () => {
-    const dsContext = new BIDSContextDataset({ tree: simpleDataset })
+    const dsContext = new BIDSContextDataset({ tree: simpleDataset, schema: schema })
     let accumulator = 0
     for await (const context of walkFileTree(dsContext)) {
       assert(
         context instanceof BIDSContext,
         'walk file tree did not return a BIDSContext',
       )
-      accumulator = accumulator + 1
+      if (!context.directory) {
+        accumulator = accumulator + 1
+      }
     }
     assertEquals(
       accumulator,
       simpleDatasetFileCount,
       'visited file count does not match expected value',
     )
   })
+  await t.step('produces context for opaque directory', async () => {
+    simpleDataset.directories.push(pathsToTree(['/code/code.sh']).directories[0])
+    const dsContext = new BIDSContextDataset({ tree: simpleDataset, schema: schema })
+    let accumulator = 0
+    for await (const context of walkFileTree(dsContext)) {
+      assert(
+        context instanceof BIDSContext,
+        'walk file tree did not return a BIDSContext',
+      )
+      if (!context.directory || context.file.name === "code/") {
+        accumulator = accumulator + 1
+      }
+    }
+    assertEquals(
+      accumulator,
+      simpleDatasetFileCount + 1,
+      'visited file count does not match expected value',
+    )
+  })
+
 })

src/schema/walk.ts

@@ -18,11 +18,11 @@ const nullFile = {
   readBytes: async (size: number, offset?: number) => new Uint8Array(),
 }
 
-function pseudoFile(dir: FileTree): BIDSFile {
+function pseudoFile(dir: FileTree, opaque: boolean): BIDSFile {
   return {
     name: `${dir.name}/`,
     path: `${dir.path}/`,
-    size: [...quickWalk(dir)].reduce((acc, file) => acc + file.size, 0),
+    size: opaque ? [...quickWalk(dir)].reduce((acc, file) => acc + file.size, 0) : 0,
     ignored: dir.ignored,
     parent: dir.parent as FileTree,
     viewed: dir.viewed,
@@ -39,9 +39,12 @@ async function* _walkFileTree(
     yield new BIDSContext(file, dsContext)
   }
   for (const dir of fileTree.directories) {
-    if (dsContext.isPseudoFile(dir)) {
-      yield new BIDSContext(pseudoFile(dir), dsContext)
-    } else {
+    const pseudo = dsContext.isPseudoFile(dir)
+    const opaque = pseudo || dsContext.isOpaqueDirectory(dir)
+    const context = new BIDSContext(pseudoFile(dir, opaque), dsContext)
+    context.directory = !pseudo
+    yield context
+    if (!opaque) {
       yield* _walkFileTree(dir, dsContext)
     }
   }

src/summary/summary.ts

@@ -108,6 +108,10 @@ export class Summary {
       return
     }
 
+    if (context.directory === true && context.size === 0) {
+      return
+    }
+
     this.totalFiles++
     this.size += await context.file.size
 

src/types/schema.ts

@@ -30,6 +30,16 @@ export interface SchemaRules {
   entities: string[]
   files: SchemaFiles
   modalities: Record<string, unknown>
+  directories: Record<string, Record<string, DirectoryRule>>
+}
+
+export interface DirectoryRule {
+  name: string
+  entity: string
+  level: string
+  value: string
+  subdirs: string[]
+  opaque: boolean
 }
 
 export interface SchemaFiles {

src/validators/filenameIdentify.test.ts

@@ -1,7 +1,7 @@
 import { assertEquals } from '@std/assert'
 import { SEPARATOR_PATTERN } from '@std/path'
 import { BIDSContext } from '../schema/context.ts'
-import { _findRuleMatches, datatypeFromDirectory, hasMatch } from './filenameIdentify.ts'
+import { findDirRuleMatches, _findRuleMatches, datatypeFromDirectory, hasMatch } from './filenameIdentify.ts'
 import { BIDSFileDeno } from '../files/deno.ts'
 import { FileIgnoreRules } from '../files/ignore.ts'
 import { loadSchema } from '../setup/loadSchema.ts'
@@ -95,3 +95,34 @@ Deno.test('test hasMatch', async (t) => {
     assertEquals(context.filenameRules.length, 2)
   })
 })
+
+Deno.test('test directoryIdentify', async (t) => {
+  await t.step('Test entity based rule', async () => {
+    const fileName = '/sub-01/'
+    const file = new BIDSFileDeno(PATH, fileName, ignore)
+    const context = new BIDSContext(file)
+    context.directory = true
+    await findDirRuleMatches(schema, context)
+    assertEquals(context.filenameRules.length, 1)
+    assertEquals(context.filenameRules[0], "rules.directories.raw.subject")
+  })
+  await t.step('Test name based rule', async () => {
+    const fileName = '/derivatives/'
+    const file = new BIDSFileDeno(PATH, fileName, ignore)
+    const context = new BIDSContext(file)
+    context.directory = true
+    await findDirRuleMatches(schema, context)
+    assertEquals(context.filenameRules.length, 1)
+    assertEquals(context.filenameRules[0], "rules.directories.raw.derivatives")
+  })
+  await t.step('Test value based rule', async () => {
+    const fileName = '/func/'
+    const file = new BIDSFileDeno(`${PATH}/sub-01/ses-01`, fileName, ignore)
+    const context = new BIDSContext(file)
+    context.directory = true
+    await findDirRuleMatches(schema, context)
+    assertEquals(context.filenameRules.length, 1)
+    assertEquals(context.filenameRules[0], "rules.directories.raw.datatype")
+  })
+})
+

src/validators/filenameIdentify.ts

@@ -26,12 +26,56 @@ const CHECKS: CheckFunction[] = [
   cleanContext,
 ]
 
+const DIR_CHECKS: CheckFunction[] = [
+  findDirRuleMatches,
+  hasMatch,
+  cleanContext,
+]
+
 export async function filenameIdentify(schema, context) {
-  for (const check of CHECKS) {
+  const checks = context?.directory ? DIR_CHECKS : CHECKS
+  for (const check of checks) {
     await check(schema as unknown as GenericSchema, context)
   }
 }
 
+export async function findDirRuleMatches(schema, context) {
+  const datasetType = context.dataset.dataset_description?.DatasetType || 'raw'
+  const schemaPath = `rules.directories.${datasetType}`
+  const directoryRule = schema[schemaPath]
+  const schemaObjects = schema['objects']
+  const schemaEntities = schema['objects.entities']
+  for (const key of Object.keys(directoryRule)) {
+    const path = `${schemaPath}.${key}`
+    const node = directoryRule[key]
+    if ('name' in node) {
+      if (node.name === context.file.name.replaceAll('/', "")) {
+        context.filenameRules.push(path)
+        break
+      }
+    }
+    if ('entity' in node) {
+      let entityDef = schemaEntities[node.entity]
+      if (
+        entityDef && 'name' in entityDef && context.file.name.startsWith(`${entityDef['name']}-`)
+      ) {
+        context.filenameRules.push(path)
+        break
+      }
+    }
+    if ('value' in node) {
+      // kludge, entries in schema.objects are plural, value specified as singular
+      // will fail for modalities
+      for (const valueObj of Object.keys(schemaObjects[`${node.value}s`])) {
+        if (valueObj === context.file.name.replaceAll('/', "")) {
+          context.filenameRules.push(path)
+          break
+        }
+      }
+    }
+  }
+  return Promise.resolve()
+}
 function findRuleMatches(schema, context) {
   const schemaPath = 'rules.files'
   Object.keys(schema[schemaPath]).map((key) => {

src/validators/internal/emptyFile.ts

@@ -2,7 +2,7 @@ import type { ContextCheckFunction } from '../../types/check.ts'
 
 // Non-schema EMPTY_FILE implementation
 export const emptyFile: ContextCheckFunction = (schema, context) => {
-  if (context.file.size === 0) {
+  if (context.file.size === 0 && !context.directory) {
     context.dataset.issues.add({
       code: 'EMPTY_FILE',
       location: context.path,