Merge pull request #1781 from Remi-Gau/enh/1748

[ENH] Add `matches` and `source_entity` to glossary and file templates

Author: effigies

src/metaschema.json

@@ -243,6 +243,16 @@
           },
           "additionalProperties": false
         },
+        "metaentities": {
+          "type": "object",
+          "patternProperties": {
+            "^[a-zA-Z0-9_]+$": {
+              "$ref": "#/definitions/termTypes/general",
+              "unevaluatedProperties": false
+            },
+            "additionalProperties": false
+          }
+        },
         "modalities": {
           "type": "object",
           "patternProperties": {
@@ -288,6 +298,7 @@
         "files",
         "formats",
         "metadata",
+        "metaentities",
         "modalities",
         "suffixes"
       ],
@@ -430,6 +441,10 @@
           },
           "additionalProperties": false
         },
+        "metaentities": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
         "common_principles": {
           "type": "array",
           "items": { "type": "string" }
@@ -486,6 +501,7 @@
 	"json",
         "sidecars",
         "tabular_data",
+        "metaentities",
         "common_principles",
         "dataset_metadata",
         "directories",

src/modality-specific-files/physiological-recordings.md

@@ -14,23 +14,13 @@ JSON file for storing metadata fields (see below).
       -   [`7t_trt`](https://github.com/bids-standard/bids-examples/tree/master/7t_trt)
       -   [`ds210`](https://github.com/bids-standard/bids-examples/tree/master/ds210)
 
-Template:
-
-```Text
-sub-<label>/[ses-<label>/]
-    <datatype>/
-        <matches>[_recording-<label>]_physio.tsv.gz
-        <matches>[_recording-<label>]_physio.json
-```
-
-For the template directory name, `<datatype>` can correspond to any data
-recording modality, for example `func`, `anat`, `dwi`, `meg`, `eeg`, `ieeg`,
-or `beh`.
-
-In the template filenames, the `<matches>` part corresponds to task filename
-before the suffix.
-For example for the file `sub-control01_task-nback_run-1_bold.nii.gz`,
-`<matches>` would correspond to `sub-control01_task-nback_run-1`.
+{{ MACROS___make_filename_template(
+       "raw",
+       placeholders=True,
+       show_entities=["recording"],
+       suffixes=["physio"]
+   )
+}}
 
 !!! warning "Caution"
 

src/modality-specific-files/task-events.md

@@ -11,17 +11,12 @@ are supported (in contrast to "block" designs) - each "block of events" can be
 represented by an individual row in the `events.tsv` file (with a long
 duration).
 
-Template:
-
-```Text
-sub-<label>/[ses-<label>]
-    <data_type>/
-        <matches>_events.tsv
-        <matches>_events.json
-```
-
-Where `<matches>` corresponds to task filename. For example:
-`sub-control01_task-nback`.
+{{ MACROS___make_filename_template(
+       "raw",
+       placeholders=True,
+       suffixes=["events"]
+   )
+}}
 
 Each task events file REQUIRES a corresponding task data file.
 It is also possible to have a single `events.tsv` file describing events

src/schema/README.md

@@ -369,6 +369,7 @@ The namespaces are:
 | --------------------------- | ----------------------------------------------------------------------------------- | ---------------- |
 | `objects.common_principles` | Terms that are used throughout BIDS                                                 | General terms    |
 | `objects.modalities`        | Broad categories of data represented in BIDS, roughly matching recording instrument | General terms    |
+| `objects.metaentities`      | Placeholders and wildcards to reduce verbosity of some templates in BIDS            | General terms    |
 | `objects.entities`          | Name-value pairs appearing in filenames                                             | Name/value terms |
 | `objects.metadata`          | Name-value pairs appearing in JSON files                                            | Name/value terms |
 | `objects.columns`           | Column headings and values appearing in TSV files                                   | Name/value terms |
@@ -505,6 +506,12 @@ The convention can be summed up in the following rules:
     | `display_name` | Human-friendly name |
     | `description`  | Term definition     |
 
+-   `objects.metaentities`
+    | Field          | Description         |
+    | -------------- | ------------------- |
+    | `display_name` | Human-friendly name |
+    | `description`  | Term definition     |
+
 -   `objects.entities`
 
     | Field          | Description                                             |
@@ -1004,6 +1011,9 @@ EventsMissing:
 -   `rules.common_principles` - This file contains a list of terms that appear in `objects.common_principles`
     that determines the order they appear in the specification
 
+-   `rules.metaentities` - This file contains a list of terms that appear in `objects.metaentities`
+    that determines the order they appear in the specification
+
 ### One-off rules
 
 -   `rules.modalities` - The keys in this file are the modalities, the values objects with the following field:

src/schema/objects/metaentities.yaml

@@ -0,0 +1,53 @@
+---
+# This file describes metaentities (generic placeholder designations for one or more entities)
+# present in BIDS filenames.
+# WARNING: The metaentities are presented here in alphabetical order!
+# The appropriate order of entities in filenames is defined in `rules/entities.yaml`,
+# rather than this file (`objects/entities.yaml`).
+#
+# Example of wildcard "any" that could be used to denote all available data types:
+#
+# any:
+#   name: any
+#   display_name: any
+#   description: |
+#     `any` is used as a wildcard in BIDS filenames templates to denote
+#     that any acceptable value for the corresponding entity match the
+#     template pattern.
+
+#         For example, if employed in replacement of the `data_type` entity:
+
+#         ```Text
+#         sub-<label>/
+#             [ses-<label>/]
+#                 <any>/
+#                     sub-<label>_[ses-<label>]_events.<extension>
+#         ```
+
+#         this indicates that the filename pattern `sub-<label>_[ses-<label>]_events.<extension>`
+#         can be placed under any of the valid `data_type` directories.
+
+matches:
+  display_name: matches
+  description: |
+    `matches` is used as a placeholder in BIDS filenames templates to denote that several files
+    share exactly the same sequence of entities and labels in their basename.
+
+    For example, in the following filename template:
+
+    ```Text
+    <matches>_bold.nii.gz
+    <matches>_events.tsv
+    <matches>_events.json
+    ```
+
+    `<matches>` could correspond to `sub-control01_task-nback_run-1`.
+
+source_entities:
+  display_name: source entities
+  description: |
+    `source_entities` is used as a placeholder in BIDS derivatives filenames templates.
+
+    `source_entities` MUST be the entire source filename, with the omission of
+    the source suffix and extension.
+    One exception to this rule is filename entities that are no longer relevant.

src/schema/rules/metaentities.yaml

@@ -0,0 +1,5 @@
+---
+# This file simply defines the order in which the meta-entities and placeholders are presented in the specification.
+# The actual term definitions appear in `objects/metaentities.yaml`.
+- matches
+- source_entities

tools/schemacode/src/bidsschematools/data/tests/test_rules.py

@@ -91,6 +91,7 @@ def test_rule_objects(schema_obj):
                     "files",
                     "formats",
                     "metadata",
+                    "metaentities",
                     "modalities",
                 ]:
                     # But other object types are referenced via their keys

tools/schemacode/src/bidsschematools/render/text.py

@@ -21,6 +21,7 @@
     "files": "files and directories",
     "formats": "format",
     "metadata": "metadata",
+    "metaentities": "meta-entity",
     "top_level_files": "top level file",
     "suffixes": "suffix",
 }
@@ -150,12 +151,6 @@ def make_glossary(schema, src_path=None):
         obj_desc = obj_def.get("description", None)
         if obj_desc is None:
             raise ValueError(f"{obj_marker} has no description.")
-        # A backslash before a newline means continue a string
-        obj_desc = obj_desc.replace("\\\n", "")
-        # Two newlines should be respected
-        obj_desc = obj_desc.replace("\n\n", "<br>")
-        # Otherwise a newline corresponds to a space
-        obj_desc = obj_desc.replace("\n", " ")
 
         text += f'\n<a name="{obj_marker}"></a>'
         text += f"\n## {obj_key}\n\n"
@@ -183,7 +178,9 @@ def make_glossary(schema, src_path=None):
             levels = [level["name"] if isinstance(level, dict) else level for level in levels]
             text += f"**Allowed values**: `{'`, `'.join(levels)}`\n\n"
 
-        text += f"**Description**:\n{obj_desc}\n\n"
+        # Convert description into markdown and append to text
+        obj_desc = MarkdownIt().render(f"**Description**:\n{obj_desc}")
+        text += f"{obj_desc}\n\n"
 
         reduced_obj_def = {k: v for k, v in obj_def.items() if k not in keys_to_drop}
 
@@ -235,6 +232,9 @@ def make_filename_template(
     src_path=None,
     n_dupes_to_combine=6,
     pdf_format=False,
+    placeholders=False,
+    empty_dirs=None,
+    show_entities=tuple(),
     **kwargs,
 ):
     """Create codeblocks containing example filename patterns for a given datatype.
@@ -262,6 +262,20 @@ def make_filename_template(
         If False, the filename template will use HTML and include hyperlinks.
         This works on the website.
         Default is False.
+    placeholders : bool, optional
+        If True, placeholder meta-entities will replace keyword-value entities in the
+        filename.
+        If ``dstype`` is ``"raw"``, the placeholder meta-entity is ``<matches>``.
+        If ``dstype`` is ``"derivatives"``, the placeholder meta-entity is ``<source_entities>``.
+        Default is False.
+    empty_dirs: bool, optional
+        If False, empty datatype directories are not included. If ``placeholders`` is True,
+        this option is set False.
+        Default is True.
+    show_entities: tuple, optional
+        If ``placeholders`` is ``False`` this argument is ignored.
+        When using placeholders, this argument can be set to a list or tuple of entity
+        names that will be "extracted" out of the placeholder.
 
     Other Parameters
     ----------------
@@ -313,19 +327,39 @@ def make_filename_template(
         for datatype in rule.datatypes:
             file_groups.setdefault(datatype, []).append(rule)
 
+    if empty_dirs is None:
+        empty_dirs = not placeholders
+
+    entity_list = schema.rules.entities
+    start_string = ""
+    if placeholders:
+        metaentity_name = "matches" if dstype == "raw" else "source_entities"
+        start_string = (
+            lt
+            + utils._link_with_html(
+                metaentity_name,
+                html_path=GLOSSARY_PATH + ".html",
+                heading=f"{metaentity_name}-metaentities",
+                pdf_format=pdf_format,
+            )
+            + gt
+        )
+        entity_list = show_entities
+
     for datatype in sorted(file_groups):
+        group_lines = []
         datatype_string = utils._link_with_html(
             datatype,
             html_path=GLOSSARY_PATH + ".html",
             heading=f"{datatype.lower()}-datatypes",
             pdf_format=pdf_format,
         )
-        lines.append(f"\t\t{datatype_string}/")
+        group_lines.append(f"\t\t{datatype_string}/")
 
         # Unique filename patterns
         for group in file_groups[datatype]:
-            ent_string = ""
-            for ent in schema.rules.entities:
+            ent_string = start_string
+            for ent in entity_list:
                 if ent not in group.entities:
                     continue
 
@@ -413,12 +447,18 @@ def make_filename_template(
                 pdf_format=pdf_format,
             )
 
-            lines.extend(
+            group_lines.extend(
                 f"\t\t\t{ent_string}_{suffix}{extension}"
                 for suffix in sorted(suffixes)
                 for extension in sorted(extensions)
             )
 
+        # If the datatype does not have any files, skip
+        if not empty_dirs and len(group_lines) == 1:
+            continue
+
+        lines.extend(group_lines)
+
     paragraph = "\n".join(lines)
     if pdf_format:
         codeblock = f"Template:\n```Text\n{paragraph}\n```"
@@ -459,6 +499,11 @@ def append_filename_template_legend(text, pdf_format=False):
   """
 
     legend = f"""{info_str}
+- `<matches>` is a placeholder to denote an arbitrary (and valid) sequence of entities
+  and labels at the beginning of the filename (only BIDS "raw").
+- `<source_entities>` is a placeholder to denote an arbitrary sequence of entities and labels
+  at the beginning of the filename matching a source file from which the file derives
+  (only BIDS-Derivatives).
 - Filename entities or directories between square brackets
   (for example, `[_ses-<label>]`) are OPTIONAL.
 - Some entities may only allow specific values,