fix(external docs): render all top level configuration fields

lib/vector-core/src/config/global_options.rs

@@ -58,7 +58,7 @@ pub enum WildcardMatching {
 //
 // If this is modified, make sure those changes are reflected in the `ConfigBuilder::append`
 // function!
-#[configurable_component(global_option("global_option"))]
+#[configurable_component]
 #[derive(Clone, Debug, Default, PartialEq)]
 pub struct GlobalOptions {
     /// The directory used for persisting Vector state data.

scripts/generate-component-docs.rb

@@ -1722,28 +1722,134 @@ def render_and_import_component_schema(root_schema, schema_name, component_type,
   )
 end
 
-def render_and_import_generated_global_option_schema(root_schema, global_options)
-  global_option_schema = {}
+def render_and_import_generated_top_level_config_schema(root_schema)
+  top_level_config_schema = {}
+
+  # Define logical groupings for top-level configuration fields
+  # These groups will be used to organize separate documentation pages
+  field_groups = {
+    # Pipeline component containers
+    'sources' => 'pipeline_components',
+    'transforms' => 'pipeline_components',
+    'sinks' => 'pipeline_components',
+    'enrichment_tables' => 'pipeline_components',
+
+    # Individual feature pages
+    'api' => 'api',
+    'schema' => 'schema',
+    'log_schema' => 'schema',
+    'secret' => 'secrets',
+
+    # Global options (everything else defaults to this)
+  }
+
+  group_metadata = {
+    'global_options' => {
+      'title' => 'Global Options',
+      'description' => 'Global configuration options that apply to Vector as a whole.',
+      'order' => 1
+    },
+    'pipeline_components' => {
+      'title' => 'Pipeline Components',
+      'description' => 'Configure sources, transforms, sinks, and enrichment tables for your observability pipeline.',
+      'order' => 2
+    },
+    'api' => {
+      'title' => 'API',
+      'description' => 'Configure Vector\'s observability API.',
+      'order' => 3
+    },
+    'schema' => {
+      'title' => 'Schema',
+      'description' => 'Configure Vector\'s internal schema system for type tracking and validation.',
+      'order' => 4
+    },
+    'secrets' => {
+      'title' => 'Secrets',
+      'description' => 'Configure secrets management for secure configuration.',
+      'order' => 5
+    }
+  }
+
+  # Usage of #[serde(flatten)] creates multiple schemas in the `allOf` array:
+  # - One or more schemas contain ConfigBuilder's direct fields
+  # - One or more schemas contain flattened GlobalOptions fields
+  all_of_schemas = root_schema['allOf'] || []
+
+  if all_of_schemas.empty?
+    @logger.error "Could not find ConfigBuilder allOf schemas in root schema"
+    return
+  end
+
+  # Collect all properties from all allOf schemas into a single hash.
+  # Since ConfigBuilder uses #[serde(flatten)], field names are unique across all schemas.
+  all_properties = all_of_schemas.reduce({}) do |acc, schema|
+    acc.merge(schema['properties'] || {})
+  end
 
-  global_options.each do |component_name, schema_name|
-    friendly_name = "'#{component_name}' #{schema_name} configuration"
+  @logger.info "[*] Found #{all_properties.keys.length} total properties across #{all_of_schemas.length} allOf schemas"
 
-    if component_name == "global_option"
-      # Flattening global options
-      unwrap_resolved_schema(root_schema, schema_name, friendly_name)
-        .each { |name, schema| global_option_schema[name] = schema }
+  # Process each property once
+  all_properties.each do |field_name, field_schema|
+    # Skip fields marked with docs::hidden
+    metadata = field_schema['_metadata'] || {}
+    if metadata['docs::hidden']
+      @logger.info "[*] Skipping '#{field_name}' (marked as docs::hidden)"
+      next
+    end
+
+    # Extract and resolve the field
+    @logger.info "[*] Extracting '#{field_name}' field from ConfigBuilder..."
+    resolved_field = resolve_schema(root_schema, field_schema)
+
+    # Assign group metadata to organize the documentation
+    if field_groups.key?(field_name)
+      group_name = field_groups[field_name]
+      resolved_field['group'] = group_name
+      @logger.debug "Assigned '#{field_name}' to group '#{group_name}'"
     else
-      # Resolving and assigning other global options
-      global_option_schema[component_name] = resolve_schema_by_name(root_schema, schema_name)
+      # Default to global_options for any fields not explicitly grouped
+      resolved_field['group'] = 'global_options'
+      @logger.debug "Assigned '#{field_name}' to default group 'global_options'"
     end
+
+    top_level_config_schema[field_name] = resolved_field
+    @logger.info "[✓] Resolved '#{field_name}'"
   end
 
-  render_and_import_schema(
-    global_option_schema,
-    "configuration",
-    ["generated", "configuration"],
-    "generated/configuration.cue"
-  )
+  # Build the final data structure with both configuration and group metadata
+  friendly_name = "configuration"
+  config_map_path = ["generated", "configuration"]
+  cue_relative_path = "generated/configuration.cue"
+
+  # Set up the structure for the value based on the configuration map path
+  data = {}
+  last = data
+  config_map_path.each do |segment|
+    last[segment] = {} if last[segment].nil?
+    last = last[segment]
+  end
+
+  # Add both the configuration schema and the group metadata
+  last['configuration'] = top_level_config_schema
+  last['groups'] = group_metadata
+
+  config_map_path.prepend('config-schema-base')
+  tmp_file_prefix = config_map_path.join('-')
+  final_json = to_pretty_json(data)
+
+  # Write the resolved schema as JSON
+  json_output_file = write_to_temp_file(["config-schema-#{tmp_file_prefix}-", '.json'], final_json)
+  @logger.info "[✓]   Wrote #{friendly_name} schema to '#{json_output_file}'. (#{final_json.length} bytes)"
+
+  # Import it as Cue
+  @logger.info "[*] Importing #{friendly_name} schema as Cue file..."
+  cue_output_file = "website/cue/reference/#{cue_relative_path}"
+  unless system(@cue_binary_path, 'import', '-f', '-o', cue_output_file, '-p', 'metadata', json_output_file)
+    @logger.error "[!]   Failed to import #{friendly_name} schema as valid Cue."
+    exit 1
+  end
+  @logger.info "[✓]   Imported #{friendly_name} schema to '#{cue_output_file}'."
 end
 
 if ARGV.empty?
@@ -1792,12 +1898,7 @@ def render_and_import_generated_global_option_schema(root_schema, global_options
   end
 end
 
-# At last, we generate the global options configuration.
-global_options = root_schema['definitions'].filter_map do |key, definition|
-  component_type = get_schema_metadata(definition, 'docs::component_type')
-  component_name = get_schema_metadata(definition, 'docs::component_name')
-  { component_name => key } if component_type == "global_option"
-end
-.reduce { |acc, item| nested_merge(acc, item) }
-
-render_and_import_generated_global_option_schema(root_schema, global_options)
+# Finally, generate the top-level Vector configuration schema. We extract ALL top-level config fields directly from the
+# ConfigBuilder struct (defined in src/config/builder.rs) by processing its allOf schemas. ConfigBuilder is the single
+# source of truth for what's actually allowed at the top level of Vector's configuration file.
+render_and_import_generated_top_level_config_schema(root_schema)

src/config/api.rs

@@ -4,7 +4,7 @@ use url::Url;
 use vector_lib::configurable::configurable_component;
 
 /// API options.
-#[configurable_component(global_option("api"))]
+#[configurable_component]
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 #[serde(default, deny_unknown_fields)]
 pub struct Options {

src/config/builder.rs

@@ -25,7 +25,6 @@ pub struct ConfigBuilder {
     pub api: api::Options,
 
     #[configurable(derived)]
-    #[configurable(metadata(docs::hidden))]
     #[serde(default)]
     pub schema: schema::Options,
 
@@ -34,32 +33,39 @@ pub struct ConfigBuilder {
     pub healthchecks: HealthcheckOptions,
 
     /// All configured enrichment tables.
+    #[configurable(metadata(docs::additional_props_description = "An enrichment table."))]
     #[serde(default)]
     pub enrichment_tables: IndexMap<ComponentKey, EnrichmentTableOuter<String>>,
 
     /// All configured sources.
+    #[configurable(metadata(docs::additional_props_description = "A source."))]
     #[serde(default)]
     pub sources: IndexMap<ComponentKey, SourceOuter>,
 
     /// All configured sinks.
+    #[configurable(metadata(docs::additional_props_description = "A sink."))]
     #[serde(default)]
     pub sinks: IndexMap<ComponentKey, SinkOuter<String>>,
 
     /// All configured transforms.
+    #[configurable(metadata(docs::additional_props_description = "A transform."))]
     #[serde(default)]
     pub transforms: IndexMap<ComponentKey, TransformOuter<String>>,
 
     /// All configured unit tests.
+    #[configurable(metadata(docs::hidden))]
     #[serde(default)]
     pub tests: Vec<TestDefinition<String>>,
 
     /// Optional configuration provider to use.
     ///
     /// Configuration providers allow sourcing configuration information from a source other than
     /// the typical configuration files that must be passed to Vector.
+    #[configurable(metadata(docs::hidden))]
     pub provider: Option<Providers>,
 
     /// All configured secrets backends.
+    #[configurable(metadata(docs::additional_props_description = "A secret backend."))]
     #[serde(default)]
     pub secret: IndexMap<ComponentKey, SecretBackends>,
 

src/config/mod.rs

@@ -270,7 +270,7 @@ impl Config {
 }
 
 /// Healthcheck options.
-#[configurable_component(global_option("healthchecks"))]
+#[configurable_component]
 #[derive(Clone, Copy, Debug)]
 #[serde(default)]
 pub struct HealthcheckOptions {

src/config/schema.rs

@@ -3,19 +3,41 @@ use vector_lib::{config::LogNamespace, configurable::configurable_component};
 pub(crate) use crate::schema::Definition;
 
 /// Schema options.
+///
+/// **Note:** The `enabled` and `validation` options are experimental and should only be enabled if you
+/// understand the limitations. While the infrastructure exists for schema tracking and validation, the
+/// full vision of automatic semantic field mapping and comprehensive schema enforcement was never fully
+/// realized.
+///
+/// If you encounter issues with these features, please [report them here](https://github.com/vectordotdev/vector/issues/new?template=bug.yml).
 #[configurable_component]
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 #[serde(default, deny_unknown_fields)]
 pub struct Options {
-    /// Whether or not schema is enabled.
+    /// When enabled, Vector tracks the schema (field types and structure) of events as they flow
+    /// from sources through transforms to sinks. This allows Vector to understand what data each
+    /// component receives and produces.
     #[serde(default = "default_enabled")]
     pub enabled: bool,
 
-    /// Whether or not schema validation is enabled.
+    /// When enabled, Vector validates that events flowing into each sink match the schema
+    /// requirements of that sink. If a sink requires certain fields or types that are missing
+    /// from the incoming events, Vector will report an error during configuration validation.
+    ///
+    /// This helps catch pipeline configuration errors early, before runtime.
     #[serde(default = "default_validation")]
     pub validation: bool,
 
-    /// Whether or not to enable log namespacing.
+    /// Controls how metadata is stored in log events.
+    ///
+    /// When set to `false` (legacy mode), metadata fields like `host`, `timestamp`, and `source_type`
+    /// are stored as top-level fields alongside your log data.
+    ///
+    /// When set to `true` (Vector namespace mode), metadata is stored in a separate metadata namespace,
+    /// keeping it distinct from your actual log data.
+    ///
+    /// See the [Log Namespacing guide](/guides/level-up/log_namespace/) for detailed information
+    /// about when to use Vector namespace mode and how to migrate from legacy mode.
     pub log_namespace: Option<bool>,
 }
 

src/enrichment_tables/mod.rs

@@ -35,7 +35,7 @@ pub mod mmdb;
 /// condition. We don't recommend using a condition that uses only date range searches.
 ///
 ///
-#[configurable_component(global_option("enrichment_tables"))]
+#[configurable_component]
 #[derive(Clone, Debug)]
 #[serde(tag = "type", rename_all = "snake_case")]
 #[enum_dispatch(EnrichmentTableConfig)]
@@ -67,6 +67,21 @@ pub enum EnrichmentTables {
     Mmdb(mmdb::MmdbConfig),
 }
 
+// Manual NamedComponent impl required because enum_dispatch doesn't support it yet.
+impl vector_lib::configurable::NamedComponent for EnrichmentTables {
+    fn get_component_name(&self) -> &'static str {
+        match self {
+            Self::File(config) => config.get_component_name(),
+            #[cfg(feature = "enrichment-tables-memory")]
+            Self::Memory(config) => config.get_component_name(),
+            #[cfg(feature = "enrichment-tables-geoip")]
+            Self::Geoip(config) => config.get_component_name(),
+            #[cfg(feature = "enrichment-tables-mmdb")]
+            Self::Mmdb(config) => config.get_component_name(),
+        }
+    }
+}
+
 impl GenerateConfig for EnrichmentTables {
     fn generate_config() -> toml::Value {
         toml::Value::try_from(Self::File(file::FileConfig {

src/providers/http.rs

@@ -21,6 +21,7 @@ use crate::{
 #[derive(Clone, Debug)]
 pub struct RequestConfig {
     /// HTTP headers to add to the request.
+    #[configurable(metadata(docs::additional_props_description = "An HTTP header."))]
     #[serde(default)]
     pub headers: IndexMap<String, String>,
 }

src/secrets/mod.rs

@@ -51,7 +51,7 @@ mod test;
 /// Secrets are loaded when Vector starts or if Vector receives a `SIGHUP` signal triggering its
 /// configuration reload process.
 #[allow(clippy::large_enum_variant)]
-#[configurable_component(global_option("secret"))]
+#[configurable_component]
 #[derive(Clone, Debug)]
 #[enum_dispatch(SecretBackend)]
 #[serde(tag = "type", rename_all = "snake_case")]
@@ -79,6 +79,20 @@ pub enum SecretBackends {
     Test(test::TestBackend),
 }
 
+// Manual NamedComponent impl required because enum_dispatch doesn't support it yet.
+impl vector_lib::configurable::NamedComponent for SecretBackends {
+    fn get_component_name(&self) -> &'static str {
+        match self {
+            Self::File(config) => config.get_component_name(),
+            Self::Directory(config) => config.get_component_name(),
+            Self::Exec(config) => config.get_component_name(),
+            #[cfg(feature = "secrets-aws-secrets-manager")]
+            Self::AwsSecretsManager(config) => config.get_component_name(),
+            Self::Test(config) => config.get_component_name(),
+        }
+    }
+}
+
 impl GenerateConfig for SecretBackends {
     fn generate_config() -> toml::Value {
         toml::Value::try_from(Self::File(file::FileBackend {

website/content/en/docs/reference/configuration/api.md

@@ -0,0 +1,14 @@
+---
+title: API configuration reference
+short: API
+weight: 6
+show_toc: true
+---
+
+This page documents the configuration for Vector's observability API.
+
+The API enables you to query Vector's topology, metrics, and health information through a GraphQL endpoint, as well as access an interactive GraphQL playground for development.
+
+{{< config-cross-links group="api" >}}
+
+{{< config/group group="api" >}}

website/content/en/docs/reference/configuration/global-options.md

@@ -3,8 +3,11 @@ title: Global options reference
 short: Global options
 weight: 4
 aliases: ["/docs/reference/global-options"]
+show_toc: true
 ---
 
-## Global configuration parameters
+This page documents global configuration options that apply to Vector as a whole, such as data directories, timezone settings, logging configuration, and more.
 
-{{< config/global >}}
+{{< config-cross-links group="global_options" >}}
+
+{{< config/group group="global_options" >}}