Add config-docs-generator tool for automated documentation generation

Author: Jiloc

.gitignore

@@ -61,6 +61,7 @@ testnet/index.html
 /testnet/helium/target/
 /contrib/tools/puppet-chain/target/
 /contrib/core-contract-tests/.cache/
+/contrib/tools/config-docs-generator/target/
 
 # These are backup files generated by rustfmt
 **/*.rs.bk

Cargo.lock

@@ -10,7 +10,8 @@ members = [
     "contrib/tools/relay-server",
     "libsigner",
     "stacks-signer",
-    "testnet/stacks-node"]
+    "testnet/stacks-node",
+    "contrib/tools/config-docs-generator"]
 
 # Dependencies we want to keep the same between workspace members
 [workspace.dependencies]

Cargo.toml

@@ -1,16 +1,25 @@
-FROM rust:bookworm AS build
+# Use a specific nightly toolchain for reproducible builds
+FROM rustlang/rust@sha256:04690ffa09cddd358b349272173155319f384e57816614eea0840ec7f9422862
 
-ARG STACKS_NODE_VERSION="No Version Info"
-ARG GIT_BRANCH='No Branch Info'
-ARG GIT_COMMIT='No Commit Info'
+# Set the working directory for building
+WORKDIR /build
 
-WORKDIR /src
+# Copy the entire project to build the binaries
 COPY . .
-RUN mkdir /out
-RUN rustup toolchain install stable
-RUN cargo build --features monitoring_prom,slog_json --release
-RUN cp -R target/release/. /out
 
-FROM debian:bookworm-slim
-COPY --from=build /out/stacks-node /out/stacks-signer /out/stacks-inspect /bin/
-CMD ["stacks-node", "mainnet"]
+# Pre-build the config-docs-generator binaries during image build
+RUN cargo build --package config-docs-generator --release
+
+# Set the working directory where the project will be mounted at runtime
+WORKDIR /project_root
+
+# Set environment variables for generate-config-docs.sh
+ENV PROJECT_ROOT=/project_root
+ENV BUILD_ROOT=/build
+ENV CARGO_HOME=/project_root/.cargo
+ENV EXTRACT_DOCS_BIN=/build/target/release/extract-docs
+ENV GENERATE_MARKDOWN_BIN=/build/target/release/generate-markdown
+ENV SKIP_BUILD=true
+
+# Set the entrypoint to run the config docs generation script
+ENTRYPOINT ["/build/contrib/tools/config-docs-generator/generate-config-docs.sh"]

Dockerfile

@@ -0,0 +1,20 @@
+[package]
+name = "config-docs-generator"
+version = "0.1.0"
+edition = "2024"
+
+[[bin]]
+name = "extract-docs"
+path = "src/extract_docs.rs"
+
+[[bin]]
+name = "generate-markdown"
+path = "src/generate_markdown.rs"
+
+[dependencies]
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+clap = { version = "4.0", features = ["derive"] }
+regex = "1.0"
+anyhow = "1.0"
+once_cell = "1.18"

contrib/tools/config-docs-generator/Cargo.toml

@@ -0,0 +1,137 @@
+# Configuration Documentation Generator
+
+A tool that automatically generates comprehensive Markdown documentation for Stacks node TOML configuration options. The documentation is extracted directly from Rust source code comments and generates a complete configuration reference.
+
+## Quick Start
+
+### Using Docker (Recommended)
+
+The easiest way to generate configuration documentation:
+
+```bash
+# Build the Docker image (one-time setup)
+docker build -t config-docs-generator .
+
+# Generate documentation
+docker run --rm -v "$(pwd):/project_root" --user "$(id -u):$(id -g)" config-docs-generator
+```
+
+This approach:
+- Uses a consistent nightly Rust environment
+- Generates `docs/generated/configuration-reference.md`
+
+### Using Local Setup (Alternative)
+
+If you prefer to run without Docker:
+
+```bash
+# Install nightly toolchain if needed
+rustup toolchain install nightly
+
+# Generate documentation
+./contrib/tools/config-docs-generator/generate-config-docs.sh
+```
+
+## What It Does
+
+The tool processes these configuration structs from the Stacks codebase:
+- `BurnchainConfig` → `[burnchain]` section
+- `NodeConfig` → `[node]` section
+- `MinerConfig` → `[miner]` section
+- `ConnectionOptionsFile` → `[connection_options]` section
+- `FeeEstimationConfigFile` → `[fee_estimation]` section
+- `EventObserverConfigFile` → `[event_observer]` section
+- `InitialBalanceFile` → `[initial_balances]` section
+
+For each configuration field, it extracts:
+- Field documentation from `///` comments
+- Default values (including constant references)
+- Usage notes and examples
+- Deprecation warnings
+
+## Output Files
+
+- **Primary**: `docs/generated/configuration-reference.md` - Complete configuration reference
+- **Intermediate**: `target/doc-generation/extracted-config-docs.json` - Raw extracted data
+
+## Adding New Configuration Structs
+
+### 1. Update the Target List
+
+Edit `contrib/tools/config-docs-generator/generate-config-docs.sh`:
+
+```bash
+TARGET_STRUCTS="BurnchainConfig,NodeConfig,MinerConfig,YourNewConfig"
+```
+
+### 2. Document Your Struct
+
+Add proper documentation to your Rust configuration struct:
+
+```rust
+/// Configuration for your new feature.
+///
+/// This controls how the feature operates and integrates
+/// with the existing node functionality.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct YourNewConfig {
+    /// Enable or disable the new feature.
+    /// ---
+    /// @default: `false`
+    /// @notes:
+    ///   - Requires restart to take effect
+    ///   - May impact performance when enabled
+    /// @toml_example: |
+    ///   enabled = true
+    pub enabled: bool,
+
+    /// Timeout for feature operations in milliseconds.
+    /// ---
+    /// @default: [`DEFAULT_TIMEOUT`]
+    pub timeout: u64,
+}
+```
+
+### Supported Annotations
+
+- **@default**: Default value (supports constant references like `[`CONSTANT_NAME`]`)
+- **@notes**: Bullet-pointed usage notes
+- **@deprecated**: Deprecation message
+- **@toml_example**: Example TOML configuration
+
+### 3. Add Section Mapping (Optional)
+
+If you want a custom TOML section name, edit `src/generate_markdown.rs`:
+
+```rust
+fn struct_to_section_name(struct_name: &str) -> String {
+    match struct_name {
+        "YourNewConfig" => "[your_custom_section]".to_string(),
+        // ... existing mappings
+        _ => format!("[{}]", struct_name.to_lowercase()),
+    }
+}
+```
+
+### 4. Generate and Verify
+
+```bash
+# Using Docker (recommended)
+docker run --rm -v "$(pwd):/project_root" --user "$(id -u):$(id -g)" config-docs-generator
+
+# OR using local setup
+./contrib/tools/config-docs-generator/generate-config-docs.sh
+
+# Check that your struct appears
+grep -A 5 "your_custom_section" docs/generated/configuration-reference.md
+```
+
+## How It Works
+
+The tool uses a three-step process:
+
+1. **Extract**: Uses `cargo +nightly rustdoc --output-format json` to generate documentation JSON
+2. **Parse**: Extracts field information, resolves constant references across crates
+3. **Generate**: Converts to Markdown with proper cross-references and formatting
+
+The process is automated by the shell script which coordinates building the tools and running the extraction/generation pipeline.

contrib/tools/config-docs-generator/README.md

@@ -0,0 +1,150 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+# Configuration - Allow environment variable overrides
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="${PROJECT_ROOT:-$(cd "$SCRIPT_DIR/../../../" && pwd)}"
+BUILD_ROOT="${BUILD_ROOT:-$PROJECT_ROOT}"
+OUTPUT_DIR="$PROJECT_ROOT/docs/generated"
+TEMP_DIR="$PROJECT_ROOT/target/doc-generation"
+CONFIG_SOURCE_FILE="$PROJECT_ROOT/stackslib/src/config/mod.rs"
+
+# Paths to binaries - allow override via environment
+EXTRACT_DOCS_BIN="${EXTRACT_DOCS_BIN:-$BUILD_ROOT/target/release/extract-docs}"
+GENERATE_MARKDOWN_BIN="${GENERATE_MARKDOWN_BIN:-$BUILD_ROOT/target/release/generate-markdown}"
+
+# Check if binaries are pre-built (skip build step)
+SKIP_BUILD="${SKIP_BUILD:-false}"
+if [[ -f "$EXTRACT_DOCS_BIN" && -f "$GENERATE_MARKDOWN_BIN" ]]; then
+    SKIP_BUILD=true
+fi
+
+log_info() {
+    echo -e "${GREEN}[INFO]${NC} $1"
+}
+
+log_warn() {
+    echo -e "${YELLOW}[WARN]${NC} $1"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR]${NC} $1"
+}
+
+cleanup() {
+    if [[ -d "$TEMP_DIR" ]]; then
+        rm -rf "$TEMP_DIR"
+    fi
+}
+
+trap cleanup EXIT
+
+main() {
+    log_info "Starting config documentation generation..."
+
+    # Create necessary directories
+    mkdir -p "$OUTPUT_DIR"
+    mkdir -p "$TEMP_DIR"
+
+    cd "$PROJECT_ROOT"
+
+    # Verify source file exists
+    if [[ ! -f "$CONFIG_SOURCE_FILE" ]]; then
+        log_error "Config source file not found: $CONFIG_SOURCE_FILE"
+        exit 1
+    fi
+
+    # Step 1: Build the documentation generation tools (skip if pre-built)
+    if [[ "$SKIP_BUILD" == "true" ]]; then
+        log_info "Using pre-built documentation generation tools..."
+    else
+        log_info "Building documentation generation tools..."
+        cargo build --package config-docs-generator --release
+    fi
+
+    # Step 2: Extract documentation from source code using rustdoc
+    log_info "Extracting configuration documentation using rustdoc..."
+    EXTRACTED_JSON="$TEMP_DIR/extracted-config-docs.json"
+    # List of specific Rust struct names to be documented
+    # NOTE: This variable must be manually updated if this list changes
+    # (e.g., new config structs are added or removed from the project)
+    TARGET_STRUCTS="BurnchainConfig,NodeConfig,MinerConfig,ConnectionOptionsFile,FeeEstimationConfigFile,EventObserverConfigFile,InitialBalanceFile"
+    "$EXTRACT_DOCS_BIN" \
+        --package stackslib \
+        --structs "$TARGET_STRUCTS" \
+        --output "$EXTRACTED_JSON"
+
+    # Step 3: Generate Markdown
+    log_info "Generating Markdown documentation..."
+    MARKDOWN_OUTPUT="$OUTPUT_DIR/configuration-reference.md"
+    "$GENERATE_MARKDOWN_BIN" \
+        --input "$EXTRACTED_JSON" \
+        --output "$MARKDOWN_OUTPUT"
+
+    log_info "Documentation generation complete!"
+    log_info "Generated files:"
+    log_info "  - Configuration reference: $MARKDOWN_OUTPUT"
+    log_info "  - Intermediate JSON: $EXTRACTED_JSON"
+
+    # Verify output
+    if [[ -f "$MARKDOWN_OUTPUT" ]]; then
+        WORD_COUNT=$(wc -w < "$MARKDOWN_OUTPUT")
+        log_info "Generated Markdown contains $WORD_COUNT words"
+    else
+        log_error "Expected output file not found: $MARKDOWN_OUTPUT"
+        exit 1
+    fi
+}
+
+# Help function
+show_help() {
+    cat << EOF
+generate-config-docs.sh - Generate configuration documentation for Stacks node
+
+USAGE:
+    $0 [OPTIONS]
+
+OPTIONS:
+    -h, --help      Show this help message
+
+DESCRIPTION:
+    This script generates comprehensive Markdown documentation for all TOML
+    configuration options available in the Stacks node. The documentation is
+    automatically extracted from Rust source code comments.
+
+    The process involves:
+    1. Building the documentation generation tools
+    2. Extracting configuration struct documentation from source code
+    3. Converting to Markdown format
+
+    Source file: stackslib/src/config/mod.rs
+
+OUTPUT:
+    docs/generated/configuration-reference.md
+
+EOF
+}
+
+# Parse command line arguments
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -h|--help)
+            show_help
+            exit 0
+            ;;
+        *)
+            log_error "Unknown option: $1"
+            show_help
+            exit 1
+            ;;
+    esac
+done
+
+main "$@"