add custom section mappings and template support

Author: Jiloc

contrib/tools/config-docs-generator/Dockerfile

@@ -16,12 +16,14 @@ WORKDIR /project_root
 
 # Set environment variables for generate-config-docs.sh
 ENV PROJECT_ROOT=/project_root
-ENV BUILD_ROOT=/build
 ENV CARGO_HOME=/project_root/.cargo
+ENV CARGO_TARGET_DIR=/tmp/stacks-config-docs/target
+ENV TEMP_DIR=/tmp/stacks-config-docs/doc-generation
 ENV EXTRACT_DOCS_BIN=/build/target/release/extract-docs
 ENV GENERATE_MARKDOWN_BIN=/build/target/release/generate-markdown
 ENV SKIP_BUILD=true
 
-# Set the entrypoint to run the config docs generation script
-# The script ends up at /build/generate-config-docs.sh due to the copy operation
+# Create the Docker-specific temp directory
+RUN mkdir -p /tmp/stacks-config-docs
+
 ENTRYPOINT ["/build/generate-config-docs.sh"]

contrib/tools/config-docs-generator/README.md

@@ -9,6 +9,9 @@ This tool automatically generates markdown documentation from Rust configuration
 The easiest way to generate configuration documentation:
 
 ```bash
+# Navigate to the config-docs-generator directory
+cd contrib/tools/config-docs-generator
+
 # Build the Docker image (one-time setup)
 docker build -t config-docs-generator .
 
@@ -28,8 +31,11 @@ If you prefer to run without Docker:
 # Install nightly toolchain if needed
 rustup toolchain install nightly
 
+# Navigate to the config-docs-generator directory
+cd contrib/tools/config-docs-generator
+
 # Generate documentation
-./contrib/tools/config-docs-generator/generate-config-docs.sh
+./generate-config-docs.sh
 ```
 
 ## What It Does
@@ -40,8 +46,8 @@ The tool processes these configuration structs from the Stacks codebase:
 - `MinerConfig` → `[miner]` section
 - `ConnectionOptionsFile` → `[connection_options]` section
 - `FeeEstimationConfigFile` → `[fee_estimation]` section
-- `EventObserverConfigFile` → `[event_observer]` section
-- `InitialBalanceFile` → `[initial_balances]` section
+- `EventObserverConfigFile` → `[[events_observer]]` section
+- `InitialBalanceFile` → `[[ustx_balance]]` section
 
 For each configuration field, it extracts:
 - Field documentation from `///` comments
@@ -335,31 +341,20 @@ pub struct YourNewConfig {
 - **@deprecated**: Deprecation message
 - **@toml_example**: Example TOML configuration
 
-### 3. Add Section Mapping (Optional)
+### 3. Generate
 
-If you want a custom TOML section name, edit `src/generate_markdown.rs`:
-
-```rust
-fn struct_to_section_name(struct_name: &str) -> String {
-    match struct_name {
-        "YourNewConfig" => "[your_custom_section]".to_string(),
-        // ... existing mappings
-        _ => format!("[{}]", struct_name.to_lowercase()),
-    }
-}
-```
-
-### 4. Generate and Verify
+Override TOML section names using JSON configuration:
 
 ```bash
-# Using Docker (recommended)
-docker run --rm -v "$(pwd)/../../../:/project_root" config-docs-generator
+# Using Docker with custom mappings and template
+cd contrib/tools/config-docs-generator
+docker run --rm -v "$(pwd)/../../../:/project_root" \
+  -e SECTION_MAPPINGS_PATH="/build/contrib/tools/config-docs-generator/custom_mappings.json" \
+  -e TEMPLATE_PATH="/build/contrib/tools/config-docs-generator/templates/custom_template.md" \
+  config-docs-generator
 
 # OR using local setup
-./contrib/tools/config-docs-generator/generate-config-docs.sh
-
-# Check that your struct appears
-grep -A 5 "your_custom_section" docs/generated/configuration-reference.md
+./generate-config-docs.sh --section-name-mappings custom_mappings.json --template custom_template.md
 ```
 
 ## How It Works

contrib/tools/config-docs-generator/generate-config-docs.sh

@@ -11,20 +11,22 @@ NC='\033[0m' # No Color
 # Configuration - Allow environment variable overrides
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 PROJECT_ROOT="${PROJECT_ROOT:-$(cd "$SCRIPT_DIR/../../../" && pwd)}"
-BUILD_ROOT="${BUILD_ROOT:-$PROJECT_ROOT}"
 OUTPUT_DIR="$PROJECT_ROOT/docs/generated"
-TEMP_DIR="$PROJECT_ROOT/target/doc-generation"
-CONFIG_SOURCE_FILE="$PROJECT_ROOT/stackslib/src/config/mod.rs"
+CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-$PROJECT_ROOT/target}"
+TEMP_DIR="${TEMP_DIR:-$CARGO_TARGET_DIR/doc-generation}"
 
-# Paths to binaries - allow override via environment
-EXTRACT_DOCS_BIN="${EXTRACT_DOCS_BIN:-$BUILD_ROOT/target/release/extract-docs}"
-GENERATE_MARKDOWN_BIN="${GENERATE_MARKDOWN_BIN:-$BUILD_ROOT/target/release/generate-markdown}"
+# Binary paths - allow override via environment
+EXTRACT_DOCS_BIN="${EXTRACT_DOCS_BIN:-$CARGO_TARGET_DIR/release/extract-docs}"
+GENERATE_MARKDOWN_BIN="${GENERATE_MARKDOWN_BIN:-$CARGO_TARGET_DIR/release/generate-markdown}"
+
+# Template and mappings paths - allow override via environment
+TEMPLATE_PATH="${TEMPLATE_PATH:-$SCRIPT_DIR/templates/reference_template.md}"
+SECTION_MAPPINGS_PATH="${SECTION_MAPPINGS_PATH:-$SCRIPT_DIR/section_name_mappings.json}"
 
 # Check if binaries are pre-built (skip build step)
 SKIP_BUILD="${SKIP_BUILD:-false}"
-if [[ -f "$EXTRACT_DOCS_BIN" && -f "$GENERATE_MARKDOWN_BIN" ]]; then
-    SKIP_BUILD=true
-fi
+
+export CARGO_TARGET_DIR
 
 log_info() {
     echo -e "${GREEN}[INFO]${NC} $1"
@@ -55,16 +57,8 @@ main() {
 
     cd "$PROJECT_ROOT"
 
-    # Verify source file exists
-    if [[ ! -f "$CONFIG_SOURCE_FILE" ]]; then
-        log_error "Config source file not found: $CONFIG_SOURCE_FILE"
-        exit 1
-    fi
-
-    # Step 1: Build the documentation generation tools (skip if pre-built)
-    if [[ "$SKIP_BUILD" == "true" ]]; then
-        log_info "Using pre-built documentation generation tools..."
-    else
+    # Step 1: Build the documentation generation tools
+    if [[ "$SKIP_BUILD" != "true" ]]; then
         log_info "Building documentation generation tools..."
         cargo build --package config-docs-generator --release
     fi
@@ -84,9 +78,9 @@ main() {
     # Step 3: Generate Markdown
     log_info "Generating Markdown documentation..."
     MARKDOWN_OUTPUT="$OUTPUT_DIR/configuration-reference.md"
-    "$GENERATE_MARKDOWN_BIN" \
-        --input "$EXTRACTED_JSON" \
-        --output "$MARKDOWN_OUTPUT"
+
+    # Call the command
+    "$GENERATE_MARKDOWN_BIN" --input "$EXTRACTED_JSON" --output "$MARKDOWN_OUTPUT" --template "$TEMPLATE_PATH" --section-name-mappings "$SECTION_MAPPINGS_PATH"
 
     log_info "Documentation generation complete!"
     log_info "Generated files:"
@@ -145,6 +139,7 @@ while [[ $# -gt 0 ]]; do
             exit 1
             ;;
     esac
+    shift
 done
 
 main "$@"

contrib/tools/config-docs-generator/section_name_mappings.json

@@ -0,0 +1,9 @@
+{
+    "BurnchainConfig": "[burnchain]",
+    "NodeConfig": "[node]",
+    "MinerConfig": "[miner]",
+    "ConnectionOptionsFile": "[connection_options]",
+    "FeeEstimationConfigFile": "[fee_estimation]",
+    "EventObserverConfigFile": "[[events_observer]]",
+    "InitialBalanceFile": "[[ustx_balance]]"
+}

contrib/tools/config-docs-generator/src/extract_docs.rs

@@ -75,7 +75,7 @@ fn main() -> Result<()> {
                 .long("structs")
                 .value_name("NAMES")
                 .help("Comma-separated list of struct names to extract")
-                .required(false),
+                .required(true),
         )
         .get_matches();
 
@@ -109,6 +109,11 @@ fn generate_rustdoc_json(package: &str) -> Result<serde_json::Value> {
     // constants referenced in doc comments are added to the project
     let additional_crates = ["stacks-common"];
 
+    // Respect CARGO_TARGET_DIR environment variable for rustdoc output
+    let rustdoc_target_dir = std::env::var("CARGO_TARGET_DIR")
+        .unwrap_or_else(|_| "target".to_string())
+        + "/rustdoc-json";
+
     // WARNING: This tool relies on nightly rustdoc JSON output (-Z unstable-options --output-format json)
     // The JSON format is subject to change with new Rust nightly versions and could break this tool.
     // Use cargo rustdoc with nightly to generate JSON for the main package
@@ -120,7 +125,7 @@ fn generate_rustdoc_json(package: &str) -> Result<serde_json::Value> {
             "-p",
             package,
             "--target-dir",
-            "target/rustdoc-json",
+            &rustdoc_target_dir,
             "--",
             "-Z",
             "unstable-options",
@@ -150,7 +155,7 @@ fn generate_rustdoc_json(package: &str) -> Result<serde_json::Value> {
                 "-p",
                 additional_crate,
                 "--target-dir",
-                "target/rustdoc-json",
+                &rustdoc_target_dir,
                 "--",
                 "-Z",
                 "unstable-options",
@@ -180,7 +185,7 @@ fn generate_rustdoc_json(package: &str) -> Result<serde_json::Value> {
     };
 
     // Read the generated JSON file - rustdoc generates it based on library name
-    let json_file_path = format!("target/rustdoc-json/doc/{}.json", lib_name);
+    let json_file_path = format!("{}/doc/{}.json", rustdoc_target_dir, lib_name);
     let json_content = std::fs::read_to_string(json_file_path)
         .context("Failed to read generated rustdoc JSON file")?;
 
@@ -443,7 +448,10 @@ fn parse_field_documentation(
                 "false" | "no" | "0" => false,
                 _ => {
                     // Default to false for invalid values, but could log a warning in the future
-                    eprintln!("Warning: Invalid @required value '{}' for field '{}', defaulting to false", required_text, field_name);
+                    eprintln!(
+                        "Warning: Invalid @required value '{}' for field '{}', defaulting to false",
+                        required_text, field_name
+                    );
                     false
                 }
             };
@@ -480,7 +488,8 @@ fn parse_literal_block_scalar(lines: &[&str], _base_indent: usize) -> String {
     }
 
     // Find the first non-empty content line to determine block indentation
-    let content_lines: Vec<&str> = lines.iter()
+    let content_lines: Vec<&str> = lines
+        .iter()
         .skip_while(|line| line.trim().is_empty())
         .copied()
         .collect();
@@ -531,7 +540,8 @@ fn parse_folded_block_scalar(lines: &[&str], _base_indent: usize) -> String {
     }
 
     // Find the first non-empty content line to determine block indentation
-    let content_lines: Vec<&str> = lines.iter()
+    let content_lines: Vec<&str> = lines
+        .iter()
         .skip_while(|line| line.trim().is_empty())
         .copied()
         .collect();
@@ -642,7 +652,11 @@ fn extract_annotation(metadata_section: &str, annotation_name: &str) -> Option<S
         if trimmed_after_colon.starts_with('|') {
             // Literal block scalar mode (|)
             // Content starts from the next line, ignoring any text after | on the same line
-            let block_lines = collect_annotation_block_lines(&all_lines, annotation_line_idx + 1, annotation_line);
+            let block_lines = collect_annotation_block_lines(
+                &all_lines,
+                annotation_line_idx + 1,
+                annotation_line,
+            );
 
             // Convert to owned strings for the parser
             let owned_lines: Vec<String> = block_lines.iter().map(|s| s.to_string()).collect();
@@ -659,7 +673,11 @@ fn extract_annotation(metadata_section: &str, annotation_name: &str) -> Option<S
         } else if trimmed_after_colon.starts_with('>') {
             // Folded block scalar mode (>)
             // Content starts from the next line, ignoring any text after > on the same line
-            let block_lines = collect_annotation_block_lines(&all_lines, annotation_line_idx + 1, annotation_line);
+            let block_lines = collect_annotation_block_lines(
+                &all_lines,
+                annotation_line_idx + 1,
+                annotation_line,
+            );
 
             // Convert to owned strings for the parser
             let owned_lines: Vec<String> = block_lines.iter().map(|s| s.to_string()).collect();
@@ -684,7 +702,11 @@ fn extract_annotation(metadata_section: &str, annotation_name: &str) -> Option<S
             }
 
             // Collect subsequent lines that belong to this annotation
-            let block_lines = collect_annotation_block_lines(&all_lines, annotation_line_idx + 1, annotation_line);
+            let block_lines = collect_annotation_block_lines(
+                &all_lines,
+                annotation_line_idx + 1,
+                annotation_line,
+            );
 
             // For default mode, preserve relative indentation within the block
             if !block_lines.is_empty() {
@@ -741,7 +763,7 @@ fn extract_annotation(metadata_section: &str, annotation_name: &str) -> Option<S
 fn collect_annotation_block_lines<'a>(
     all_lines: &[&'a str],
     start_idx: usize,
-    annotation_line: &str
+    annotation_line: &str,
 ) -> Vec<&'a str> {
     let mut block_lines = Vec::new();
     let annotation_indent = annotation_line.len() - annotation_line.trim_start().len();
@@ -2108,7 +2130,10 @@ and includes various formatting.
         let result = parse_field_documentation(doc_text, "test_field").unwrap();
 
         assert_eq!(result.0.name, "test_field");
-        assert_eq!(result.0.description, "Field with required and units annotations.");
+        assert_eq!(
+            result.0.description,
+            "Field with required and units annotations."
+        );
         assert_eq!(result.0.default_value, Some("`5000`".to_string()));
         assert_eq!(result.0.required, Some(true));
         assert_eq!(result.0.units, Some("milliseconds".to_string()));
@@ -2232,7 +2257,10 @@ and includes various formatting.
         let result = parse_field_documentation(doc_text, "test_field").unwrap();
         let (field_doc, referenced_constants) = result;
 
-        assert_eq!(field_doc.units, Some("[`DEFAULT_TIMEOUT_MS`] milliseconds".to_string()));
+        assert_eq!(
+            field_doc.units,
+            Some("[`DEFAULT_TIMEOUT_MS`] milliseconds".to_string())
+        );
         // Check that constants were collected from units
         assert!(referenced_constants.contains("DEFAULT_TIMEOUT_MS"));
     }
@@ -2402,7 +2430,10 @@ and includes various formatting.
         // Test empty @required annotation (should return None, not Some(false))
         let doc_text_empty = "Test field.\n---\n@required:";
         let result_empty = parse_field_documentation(doc_text_empty, "test_field").unwrap();
-        assert_eq!(result_empty.0.required, None, "Empty @required should not be parsed");
+        assert_eq!(
+            result_empty.0.required, None,
+            "Empty @required should not be parsed"
+        );
     }
 
     #[test]