Merge pull request #804 from ypriverol/feat/yaml-driven-template-docs

Feat/yaml driven template docs

Author: ypriverol

.github/workflows/build-docs.yml

@@ -15,6 +15,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - name: Setup Ruby
         uses: ruby/setup-ruby@v1
@@ -26,6 +28,9 @@ jobs:
           gem install asciidoctor
           gem install asciidoctor-diagram
 
+      - name: Install Python dependencies
+        run: pip install pyyaml jinja2
+
       - name: Set output directory based on branch
         id: set-output-dir
         run: |
@@ -37,45 +42,14 @@ jobs:
             echo "base_url=/dev" >> $GITHUB_OUTPUT
           fi
 
-      - name: Create output directory
-        run: mkdir -p ${{ steps.set-output-dir.outputs.output_dir }}
-
-      - name: Build AsciiDoc documentation
-        run: |
-          chmod +x scripts/build-asciidoc.sh
-          ./scripts/build-asciidoc.sh ${{ steps.set-output-dir.outputs.output_dir }}
-
-      - name: Copy static assets
-        run: |
-          chmod +x scripts/copy-assets.sh
-          ./scripts/copy-assets.sh ${{ steps.set-output-dir.outputs.output_dir }}
-
-      - name: Build SDRF Explorer index
-        run: |
-          python3 site/build-sdrf-index.py
-          cp site/sdrf-data.json ${{ steps.set-output-dir.outputs.output_dir }}/
-
-      - name: Inject navigation headers
-        run: |
-          python3 scripts/inject-headers.py ${{ steps.set-output-dir.outputs.output_dir }}
-
-      - name: Transform SDRF links to viewer
-        run: |
-          python3 scripts/transform-links.py ${{ steps.set-output-dir.outputs.output_dir }}
-
-      - name: Transform SDRF example tables
-        run: |
-          python3 scripts/transform-sdrf-tables.py ${{ steps.set-output-dir.outputs.output_dir }}
-
-      - name: Build search index
+      - name: Build documentation site
         run: |
-          python3 site/build-search-index.py . ${{ steps.set-output-dir.outputs.output_dir }}/search-index.json
-
-      - name: Add version banner for dev branch
-        if: github.ref_name == 'dev'
-        run: |
-          chmod +x scripts/add-dev-banner.sh
-          ./scripts/add-dev-banner.sh ${{ steps.set-output-dir.outputs.output_dir }}
+          chmod +x scripts/build-docs.sh
+          if [ "${{ github.ref_name }}" == "dev" ]; then
+            ./scripts/build-docs.sh ${{ steps.set-output-dir.outputs.output_dir }} --dev
+          else
+            ./scripts/build-docs.sh ${{ steps.set-output-dir.outputs.output_dir }}
+          fi
 
       - name: Checkout gh-pages branch for merging
         if: github.ref_name == 'dev'

DEVELOPMENT.md

@@ -8,34 +8,30 @@ This guide explains how to build and maintain the SDRF-Proteomics documentation
 proteomics-metadata-standard/
 ├── scripts/                   # Build and utility scripts
 │   ├── build-docs.sh          # Main build script (local & CI/CD)
-│   ├── build-asciidoc.sh      # AsciiDoc build helper
-│   ├── copy-assets.sh         # Asset copying helper
+│   ├── build_template_pages.py # Generate per-template HTML from YAML
+│   ├── build_index_templates.py # Generate index.html template section
+│   ├── resolve_templates.py   # Template inheritance resolver
 │   ├── inject-headers.py      # Navigation header injection
-│   ├── transform-links.py     # SDRF link transformation
+│   ├── transform-links.py     # .adoc→.html link transformation
 │   ├── transform-sdrf-tables.py # SDRF example table styling
+│   ├── build_sdrf_builder_data.py # SDRF builder JSON data compiler
 │   └── add-dev-banner.sh      # Dev version banner
 ├── sdrf-proteomics/           # Main specification source
 │   ├── README.adoc            # Main specification (AsciiDoc)
-│   ├── tool-support.adoc      # Tool support page
-│   ├── metadata-guidelines/   # General metadata guidelines
-│   │   ├── sample-metadata.adoc
+│   ├── TOOLS.adoc             # Tool support page
+│   ├── TEMPLATES.adoc         # Templates guide
+│   ├── SAMPLE-GUIDELINES.adoc # Sample metadata guidelines
+│   ├── metadata-guidelines/   # Detailed metadata guidelines
+│   │   ├── data-analysis-metadata.adoc
 │   │   └── sdrf-terms.tsv
-│   ├── templates/             # All templates (core + specialized)
-│   │   ├── human/             # Human template (includes clinical metadata)
-│   │   ├── vertebrates/       # Non-human vertebrates
-│   │   ├── invertebrates/     # Invertebrates (Drosophila, C. elegans)
-│   │   ├── plants/            # Plant organisms
-│   │   ├── ms-proteomics/     # MS proteomics template (base for MS experiments)
-│   │   ├── cell-lines/        # Cell line experiments
-│   │   ├── crosslinking/      # XL-MS experiments
-│   │   ├── immunopeptidomics/ # Immunopeptidomics
-│   │   └── ...
+│   ├── sdrf-templates/        # YAML template definitions (submodule)
 │   └── images/                # Specification images
 ├── site/                      # Website assets and build scripts
 │   ├── css/style.css          # Main stylesheet
 │   ├── js/search.js           # Search functionality
+│   ├── js/sdrf-builder.js     # Interactive SDRF builder logic
 │   ├── index.html             # Homepage
-│   ├── quickstart.html        # Quick start guide
+│   ├── quickstart.html        # Quick start guide + interactive builder
 │   ├── sdrf-explorer.html     # SDRF dataset explorer
 │   ├── sdrf-terms.html        # Terms reference page
 │   ├── build-sdrf-index.py    # Builds dataset index
@@ -88,7 +84,7 @@ The easiest way to build the documentation is using the provided build script. T
 
 1. **Converts AsciiDoc to HTML** using Asciidoctor with proper styling options
 2. **Copies static assets** (CSS, JavaScript, images)
-3. **Copies static HTML pages** (index.html, quickstart.html, sdrf-explorer.html, sdrf-terms.html)
+3. **Copies static HTML pages** (index.html, quickstart.html, sdrf-explorer.html, sdrf-editor.html, sdrf-terms.html)
 4. **Builds SDRF Explorer index** from annotated-projects
 5. **Injects navigation headers** into all generated HTML pages
 6. **Transforms SDRF links** to use the SDRF Explorer viewer
@@ -345,7 +341,6 @@ To force an immediate update, the specification website needs to be rebuilt:
 | File | Purpose |
 |------|---------|
 | `site/sdrf-editor.html` | Editor page (loads Angular app from jsDelivr) |
-| `scripts/copy-assets.sh` | Copies editor page during build |
 | `scripts/inject-headers.py` | Adds Editor link to navigation |
 
 ### Testing Editor Changes Locally
@@ -361,6 +356,32 @@ npm run build -- --configuration=production
 # (temporarily change jsDelivr URLs to local paths for testing)
 ```
 
+## Interactive SDRF Builder
+
+The quickstart page (`/quickstart.html`) includes an interactive SDRF template builder that helps users create custom SDRF templates by answering questions.
+
+### How It Works
+
+1. **Build time:** `scripts/build_sdrf_builder_data.py` resolves all YAML templates and sdrf-terms.tsv into `sdrf-builder-data.json`
+2. **Runtime:** `site/js/sdrf-builder.js` loads the JSON and renders a branching questionnaire
+3. Users select technology, organism, experiment type, review columns, and download a TSV
+
+### Builder Files
+
+| File | Purpose |
+|------|---------|
+| `scripts/build_sdrf_builder_data.py` | Compiles YAML templates + terms into JSON |
+| `site/js/sdrf-builder.js` | Builder UI logic (vanilla JS) |
+| `site/quickstart.html` | Page with simplified intro + builder scaffold |
+| `site/css/style.css` | Builder-specific styles (`.builder-*`, `.option-*`) |
+
+### Updating the Builder
+
+When templates change (new columns, new templates):
+1. The build script automatically recompiles `sdrf-builder-data.json`
+2. No JS changes needed unless new template layers or combination rules are added
+3. New templates appear automatically in the appropriate step
+
 ## Contributing
 
 1. Make changes to the appropriate AsciiDoc or HTML files

scripts/build-asciidoc.sh

@@ -109,53 +109,24 @@ echo "Building tool support page..."
 asciidoctor \
     -D "$OUTPUT_DIR" \
     $ASCIIDOCTOR_OPTS \
-    -o tool-support.html \
-    sdrf-proteomics/tool-support.adoc
+    -o tools.html \
+    sdrf-proteomics/TOOLS.adoc
 
-# Build metadata guidelines
-echo "Building metadata guidelines..."
+# Build sample guidelines page
+echo "Building sample guidelines page..."
 asciidoctor \
-    -D "$OUTPUT_DIR/metadata-guidelines" \
-    -a stylesheet=../css/style.css \
-    -a linkcss \
-    -a toc=left \
-    -a toclevels=3 \
-    -a sectanchors \
-    -a sectlinks \
-    --backend=html5 \
-    -o sample-metadata.html \
-    sdrf-proteomics/metadata-guidelines/sample-metadata.adoc
-
-# Check if human-sample-metadata.adoc exists and build it
-if [ -f "sdrf-proteomics/metadata-guidelines/human-sample-metadata.adoc" ]; then
-    echo "Building human sample metadata guidelines..."
-    asciidoctor \
-        -D "$OUTPUT_DIR/metadata-guidelines" \
-        -a stylesheet=../css/style.css \
-        -a linkcss \
-        -a toc=left \
-        -a toclevels=3 \
-        -a sectanchors \
-        -a sectlinks \
-        --backend=html5 \
-        -o human-sample-metadata.html \
-        sdrf-proteomics/metadata-guidelines/human-sample-metadata.adoc
-fi
+    -D "$OUTPUT_DIR" \
+    $ASCIIDOCTOR_OPTS \
+    -o sample-guidelines.html \
+    sdrf-proteomics/SAMPLE-GUIDELINES.adoc
 
 # Build templates guide
 if [ -f "sdrf-proteomics/TEMPLATES.adoc" ]; then
     echo "Building templates guide..."
     asciidoctor \
         -D "$OUTPUT_DIR" \
-        -a stylesheet=css/style.css \
-        -a linkcss \
-        -a toc=left \
-        -a toclevels=3 \
-        -a sectanchors \
-        -a sectlinks \
-        -a source-highlighter=highlight.js \
-        --backend=html5 \
-        -o TEMPLATES.html \
+        $ASCIIDOCTOR_OPTS \
+        -o templates.html \
         sdrf-proteomics/TEMPLATES.adoc
 fi
 
@@ -176,25 +147,10 @@ if [ -f "sdrf-proteomics/metadata-guidelines/data-analysis-metadata.adoc" ]; the
         sdrf-proteomics/metadata-guidelines/data-analysis-metadata.adoc
 fi
 
-# Build template documentation
-echo "Building templates..."
-for dir in sdrf-proteomics/templates/*/; do
-    if [ -f "${dir}README.adoc" ]; then
-        template_name=$(basename "$dir")
-        echo "  Building template: $template_name"
-        asciidoctor \
-            -D "$OUTPUT_DIR/templates" \
-            -a stylesheet=../css/style.css \
-            -a linkcss \
-            -a toc=left \
-            -a toclevels=3 \
-            -a sectanchors \
-            -a sectlinks \
-            --backend=html5 \
-            -o "${template_name}.html" \
-            "${dir}README.adoc"
-    fi
-done
+# Build template pages from YAML definitions
+echo "Building template pages from YAML..."
+python3 scripts/build_template_pages.py \
+    sdrf-proteomics/sdrf-templates "$OUTPUT_DIR/templates"
 
 # Copy assets
 echo "Copying assets..."
@@ -204,6 +160,7 @@ cp site/css/style.css "$OUTPUT_DIR/css/"
 
 # Copy JavaScript
 cp site/js/search.js "$OUTPUT_DIR/js/"
+cp site/js/sdrf-builder.js "$OUTPUT_DIR/js/"
 
 # Copy images
 cp -r sdrf-proteomics/images/* "$OUTPUT_DIR/images/" 2>/dev/null || true
@@ -215,9 +172,16 @@ cp site/index.html "$OUTPUT_DIR/"
 cp site/sdrf-terms.html "$OUTPUT_DIR/"
 cp site/quickstart.html "$OUTPUT_DIR/"
 cp site/sdrf-explorer.html "$OUTPUT_DIR/"
+cp site/sdrf-editor.html "$OUTPUT_DIR/"
 
-# Copy SDRF terms TSV
-cp sdrf-proteomics/metadata-guidelines/sdrf-terms.tsv "$OUTPUT_DIR/"
+# Copy SDRF terms TSV (if present) — also create TERMS.tsv alias for AsciiDoc links
+cp sdrf-proteomics/metadata-guidelines/sdrf-terms.tsv "$OUTPUT_DIR/" 2>/dev/null || true
+cp sdrf-proteomics/TERMS.tsv "$OUTPUT_DIR/TERMS.tsv" 2>/dev/null || true
+
+# Auto-generate index.html template section from YAML
+echo "Updating index template section..."
+python3 scripts/build_index_templates.py \
+    sdrf-proteomics/sdrf-templates "$OUTPUT_DIR/index.html"
 
 # Build SDRF Explorer index
 echo "Building SDRF Explorer index..."
@@ -236,6 +200,11 @@ python3 scripts/transform-links.py "$OUTPUT_DIR"
 echo "Transforming SDRF example tables..."
 python3 scripts/transform-sdrf-tables.py "$OUTPUT_DIR"
 
+# Build SDRF builder data
+echo "Building SDRF builder data..."
+python3 scripts/build_sdrf_builder_data.py \
+    sdrf-proteomics/sdrf-templates "$OUTPUT_DIR/sdrf-builder-data.json"
+
 # Build search index
 echo "Building search index..."
 python3 site/build-search-index.py . "$OUTPUT_DIR/search-index.json"