engine: Add support for dynamic dependencies pt2

Because documentation generation follows the same code path as
build-time when it comes to instantiating the layer manager, if a layer
refers to an env var when specifying dependencies, unless that var is
set when the documentation is generated, the layer will be skipped
because it can't be found. This is the desired behaviour for build-time
but not for docgen-time.

To combat this, pass a flag down from the top level so we can be more
lenient when generating documentation for layers with dynamic
dependencies.

Also, update the layer jinja template to strip whitespace. Next time we
autogenerate docs there will be removal of blank lines in some cases,
but that's better anyway.

Author: learmj

docs/generate.py

@@ -88,7 +88,7 @@ def main():
         str(script_dir.parent / 'image'),
         str(script_dir.parent / 'layer')
     ]
-    manager = LayerManager(layer_paths)
+    manager = LayerManager(layer_paths, doc_mode=True)
 
     # All layer names
     layer_names = sorted(manager.layers.keys())

site/env_types.py

@@ -313,7 +313,7 @@ def __init__(self, name: str, description: str = "", version: str = "1.0.0",
 
     @classmethod
     def from_metadata_fields(cls, metadata_dict: Dict[str, str],
-                           filepath: str = "") -> Optional['EnvLayer']:
+                           filepath: str = "", doc_mode: bool = False) -> Optional['EnvLayer']:
         """Create an EnvLayer from metadata fields."""
         # Check if this has layer information
         layer_name = metadata_dict.get(XEnv.layer_name(), "")
@@ -329,16 +329,16 @@ def from_metadata_fields(cls, metadata_dict: Dict[str, str],
 
         # Parse dependency lists
         requires_str = metadata_dict.get(XEnv.layer_requires(), "")
-        requires = cls._parse_dependency_list(requires_str)
+        requires = cls._parse_dependency_list(requires_str, doc_mode)
 
         provides_str = metadata_dict.get(XEnv.layer_provides(), "")
-        provides = cls._parse_dependency_list(provides_str)
+        provides = cls._parse_dependency_list(provides_str, doc_mode)
 
         requires_provider_str = metadata_dict.get(XEnv.layer_requires_provider(), "")
-        requires_provider = cls._parse_dependency_list(requires_provider_str)
+        requires_provider = cls._parse_dependency_list(requires_provider_str, doc_mode)
 
         conflicts_str = metadata_dict.get(XEnv.layer_conflicts(), "")
-        conflicts = cls._parse_dependency_list(conflicts_str)
+        conflicts = cls._parse_dependency_list(conflicts_str, doc_mode)
 
         # Infer config file from filepath if not provided
         import os
@@ -357,7 +357,7 @@ def from_metadata_fields(cls, metadata_dict: Dict[str, str],
         )
 
     @staticmethod
-    def _parse_dependency_list(depends_str: str) -> List[str]:
+    def _parse_dependency_list(depends_str: str, doc_mode: bool = False) -> List[str]:
         """Parse dependency string into list of layer names/IDs with environment variable evaluation."""
         if not depends_str.strip():
             return []
@@ -369,19 +369,22 @@ def _parse_dependency_list(depends_str: str) -> List[str]:
             if dep_name:
                 # Find and evaluate environment variables in dependency names
                 if '${' in dep_name:
-                    dep_name = EnvLayer._evaluate_env_variables(dep_name)
+                    dep_name = EnvLayer._evaluate_env_variables(dep_name, doc_mode)
 
                 # Validate dependency name format
                 if re.search(r"\s", dep_name):
                     raise ValueError(
                         f"Invalid dependency token '{dep_name}' - dependencies must be comma-separated without spaces/newlines inside a token")
-                if not re.match(r'^[A-Za-z0-9_-]+$', dep_name):
+                # In doc_mode, allow environment variable placeholders like ${VAR}-suffix
+                if doc_mode and not re.match(r'^[A-Za-z0-9_${}-]+$', dep_name):
+                    raise ValueError(f"Invalid dependency name '{dep_name}' - only alphanum, dash, underscore, and environment variable placeholders allowed")
+                elif not doc_mode and not re.match(r'^[A-Za-z0-9_-]+$', dep_name):
                     raise ValueError(f"Invalid dependency name '{dep_name}' - only alphanum, dash, underscore allowed")
                 deps.append(dep_name)
         return deps
 
     @staticmethod
-    def _evaluate_env_variables(text: str) -> str:
+    def _evaluate_env_variables(text: str, doc_mode: bool = False) -> str:
         """Evaluate ${VAR} environment variable substitutions in text."""
         import re
         import os
@@ -390,7 +393,11 @@ def replacer(match):
             var_name = match.group(1)
             env_value = os.environ.get(var_name)
             if env_value is None:
-                raise ValueError(f"Environment variable '{var_name}' not found for dependency evaluation")
+                if doc_mode:
+                    # In documentation mode, return the original placeholder
+                    return match.group(0)
+                else:
+                    raise ValueError(f"Environment variable '{var_name}' not found for dependency evaluation")
             return env_value
 
         return re.sub(r'\$\{([A-Za-z_][A-Za-z0-9_]*)\}', replacer, text)
@@ -460,7 +467,7 @@ def __init__(self, filepath: str = ""):
 
     @classmethod
     def from_metadata_dict(cls, metadata_dict: Dict[str, str],
-                          filepath: str = "") -> 'MetadataContainer':
+                          filepath: str = "", doc_mode: bool = False) -> 'MetadataContainer':
         """Create a MetadataContainer from a metadata dictionary."""
         container = cls(filepath)
         container.raw_metadata = metadata_dict.copy()
@@ -472,7 +479,7 @@ def from_metadata_dict(cls, metadata_dict: Dict[str, str],
         container.var_prefix = container.raw_metadata.get(XEnv.var_prefix(), "").lower()
 
         # Extract layer information
-        container.layer = EnvLayer.from_metadata_fields(container.raw_metadata, filepath)
+        container.layer = EnvLayer.from_metadata_fields(container.raw_metadata, filepath, doc_mode)
 
         # Extract variables
         for key in container.raw_metadata.keys():

site/layer_manager.py

@@ -17,7 +17,7 @@
 
 # Handles discovery, dependency resolution, and orchestration
 class LayerManager:
-    def __init__(self, search_paths: Optional[List[str]] = None, file_patterns: Optional[List[str]] = None, *, show_loaded: bool = False):
+    def __init__(self, search_paths: Optional[List[str]] = None, file_patterns: Optional[List[str]] = None, *, show_loaded: bool = False, doc_mode: bool = False):
         if search_paths is None:
             search_paths = ['./layer']
         if file_patterns is None:
@@ -28,6 +28,7 @@ def __init__(self, search_paths: Optional[List[str]] = None, file_patterns: Opti
         self.layers: Dict[str, Metadata] = {}  # layer_name -> Metadata object
         self.layer_files: Dict[str, str] = {}  # layer_name -> file_path
         self.show_loaded = show_loaded
+        self.doc_mode = doc_mode  # When True, load all layers regardless of environment variables
         # provider index will be built after layers are loaded
         self.provider_index: Dict[str, str] = {}
         self.provider_conflicts: Dict[str, Set[str]] = {}
@@ -74,7 +75,7 @@ def load_layers(self):
 
             for metadata_file in all_files:
                 try:
-                    meta = Metadata(metadata_file)
+                    meta = Metadata(metadata_file, doc_mode=self.doc_mode)
                 except Exception:
                     # Malformed YAML or metadata – skip
                     continue
@@ -715,6 +716,13 @@ def get_layer_documentation_data(self, layer_name: str):
         # Check for companion doc
         companion_doc = self._get_companion_doc(layer_name, format='asciidoc')
 
+        # Process dependencies to categorise them as static or dynamic
+        dependencies = self._categorise_dependencies(layer_name)
+
+        # Reverse dependencies don't need categorisation - we can't determine them
+        # if they use env vars, so we only report static rdeps.
+        reverse_dependencies = self.get_reverse_dependencies(layer_name)
+
         return {
             'layer_info': layer.get_layer_info(),
             'variables': variables,
@@ -723,8 +731,8 @@ def get_layer_documentation_data(self, layer_name: str):
             'mmdebstrap': mmdebstrap_config,
             'file_path': relative_path,
             'companion_doc': companion_doc,
-            'dependencies': self.get_dependencies(layer_name),
-            'reverse_dependencies': self.get_reverse_dependencies(layer_name)
+            'dependencies': dependencies,
+            'reverse_dependencies': reverse_dependencies
         }
 
     def _get_companion_doc(self, layer_name: str, format: str = 'markdown') -> str:
@@ -793,6 +801,29 @@ def _get_raw_metadata_fields(self, layer_name: str) -> dict:
         except Exception:
             return {}
 
+    def _categorise_dependencies(self, layer_name: str) -> dict:
+        """Categorise dependencies as static or dynamic based on environment variable usage."""
+        layer_info = self.get_layer_info(layer_name)
+        if not layer_info:
+            return {'static_dep': [], 'dyn_dep': []}
+
+        static_deps = []
+        dyn_deps = []
+
+        for dep in layer_info.get('depends', []):
+            if '${' in dep and '}' in dep:
+                # Contains env variable substitution (dynamic)
+                dyn_deps.append(dep)
+            else:
+                # static
+                static_deps.append(dep)
+
+        return {
+            'static_dep': static_deps,
+            'dyn_dep': dyn_deps
+        }
+
+
 
 def _generate_layer_boilerplate():
     """Generate boilerplate example layer with metadata"""

site/metadata_parser.py

@@ -152,12 +152,12 @@ def get_supported_fields_list() -> list:
 class Metadata:
     """Metadata parser with modular classes."""
 
-    def __init__(self, filepath):
+    def __init__(self, filepath, doc_mode: bool = False):
         self.filepath = filepath
         raw_metadata = self._load_metadata(filepath)
 
         # Create the container (applies placeholder substitutions internally)
-        self._container = MetadataContainer.from_metadata_dict(raw_metadata, filepath)
+        self._container = MetadataContainer.from_metadata_dict(raw_metadata, filepath, doc_mode)
 
         # Create validation result builder
         self._result_builder = ValidationResultBuilder(filepath)

templates/docs/html/layer.html

@@ -136,15 +136,18 @@ <h2>Additional Documentation</h2>
     </div>
     {% endif %}
 
-    {% if layer.dependencies or layer.reverse_dependencies or layer.layer_info.provides or layer.layer_info.provider_requires %}
+    {% if layer.dependencies.static_dep or layer.dependencies.dyn_dep or layer.reverse_dependencies or layer.layer_info.provides or layer.layer_info.provider_requires %}
     <div class="section">
         <h2>Relationships</h2>
-        {% if layer.dependencies %}
+        {% if layer.dependencies.static_dep or layer.dependencies.dyn_dep %}
         <p><strong>Depends on:</strong></p>
         <div class="deps">
-            {% for dep in layer.dependencies %}
+            {%- for dep in layer.dependencies.static_dep %}
             <a href="{{ dep }}.html" class="dep-badge">{{ dep }}</a>
-            {% endfor %}
+            {%- endfor %}
+            {%- for dep in layer.dependencies.dyn_dep %}
+            <span class="dep-badge" style="background: #6c757d; cursor: default;" title="Dynamic dependency (contains environment variables)">{{ dep }}</span>
+            {%- endfor %}
         </div>
         {% endif %}
         {% if layer.reverse_dependencies %}