enhancement(vrl): consolidate all VRL functions into vector-vrl-functions crate

Cargo.toml

@@ -200,6 +200,7 @@ vector-config-common = { path = "lib/vector-config-common" }
 vector-config-macros = { path = "lib/vector-config-macros" }
 vector-common-macros = { path = "lib/vector-common-macros" }
 vector-lib = { path = "lib/vector-lib", default-features = false, features = ["vrl"] }
+vector-vrl-functions = { path = "lib/vector-vrl/functions" }
 vrl = { git = "https://github.com/vectordotdev/vrl.git", branch = "main", features = ["arbitrary", "cli", "test", "test_framework"] }
 mock_instant = { version = "0.6" }
 serial_test = { version = "3.2" }
@@ -227,7 +228,7 @@ vector-lib.workspace = true
 vector-config.workspace = true
 vector-config-common.workspace = true
 vector-config-macros.workspace = true
-vector-vrl-functions = { path = "lib/vector-vrl/functions" }
+vector-vrl-functions.workspace = true
 loki-logproto = { path = "lib/loki-logproto", optional = true }
 
 # Tokio / Futures
@@ -654,7 +655,7 @@ sources-aws_s3 = ["aws-core", "dep:aws-sdk-sqs", "dep:aws-sdk-s3", "dep:async-co
 sources-aws_sqs = ["aws-core", "dep:aws-sdk-sqs"]
 sources-datadog_agent = ["sources-utils-http-encoding", "protobuf-build", "dep:prost"]
 sources-demo_logs = ["dep:fakedata"]
-sources-dnstap = ["sources-utils-net-tcp", "dep:base64", "dep:hickory-proto", "dep:dnsmsg-parser", "dep:dnstap-parser", "protobuf-build", "dep:prost"]
+sources-dnstap = ["sources-utils-net-tcp", "dep:base64", "dep:hickory-proto", "dep:dnsmsg-parser", "dep:dnstap-parser", "protobuf-build", "dep:prost", "vector-vrl-functions/dnstap"]
 sources-docker_logs = ["docker"]
 sources-eventstoredb_metrics = []
 sources-exec = []

changelog.d/vrl_functions_consolidation.enhancement.md

@@ -0,0 +1,4 @@
+Vector-specific VRL functions are now available everywhere. Previously some functions were not
+available inside codec VRL transforms and in the VRL cli (via `vector vrl`).
+
+authors: thomasqueirozb

clippy.toml

@@ -5,6 +5,7 @@ allow-unwrap-in-tests = true
 # https://rust-lang.github.io/rust-clippy/master/index.html#disallowed_method
 disallowed-methods = [
   { path = "std::io::Write::write", reason = "This doesn't handle short writes, use `write_all` instead." },
+  { path = "vrl::stdlib::all", reason = "Use `vector_vrl_functions::all()` instead for consistency across all Vector VRL functions." },
 ]
 
 disallowed-types = [

lib/codecs/Cargo.toml

@@ -46,6 +46,7 @@ vector-common = { path = "../vector-common", default-features = false }
 vector-config = { path = "../vector-config", default-features = false }
 vector-config-macros = { path = "../vector-config-macros", default-features = false }
 vector-core = { path = "../vector-core", default-features = false, features = ["vrl"] }
+vector-vrl-functions.workspace = true
 
 [dev-dependencies]
 futures.workspace = true

lib/codecs/src/decoding/format/vrl.rs

@@ -59,7 +59,7 @@ impl VrlDeserializerConfig {
 
         match compile_vrl(
             &self.vrl.source,
-            &vrl::stdlib::all(),
+            &vector_vrl_functions::all(),
             &state,
             CompileConfig::default(),
         ) {

lib/dnstap-parser/Cargo.toml

@@ -15,7 +15,10 @@ hickory-proto.workspace = true
 prost.workspace = true
 snafu.workspace = true
 tracing.workspace = true
-vector-lib.workspace = true
+vector-config.workspace = true
+vector-common = { path = "../vector-common" }
+vector-lookup = { path = "../vector-lookup", features = ["test"] }
+vector-core = { path = "../vector-core" }
 vrl.workspace = true
 paste.workspace = true
 

lib/dnstap-parser/src/internal_events.rs

@@ -1,5 +1,5 @@
 use tracing::warn;
-use vector_lib::{
+use vector_common::{
     NamedInternalEvent,
     internal_event::{InternalEvent, error_stage, error_type},
 };

lib/dnstap-parser/src/parser.rs

@@ -16,10 +16,8 @@ use hickory_proto::{
 };
 use prost::Message;
 use snafu::Snafu;
-use vector_lib::{
-    Error, Result, emit,
-    event::{LogEvent, Value},
-};
+use vector_common::{Error, Result, internal_event::emit};
+use vector_core::event::{LogEvent, Value};
 use vrl::{owned_value_path, path};
 
 #[allow(warnings, clippy::all, clippy::pedantic, clippy::nursery)]
@@ -38,10 +36,8 @@ use dnstap_proto::{
     Dnstap, Message as DnstapMessage, SocketFamily, SocketProtocol,
     message::Type as DnstapMessageType,
 };
-use vector_lib::{
-    config::log_schema,
-    lookup::{PathPrefix, lookup_v2::ValuePath},
-};
+use vector_core::config::log_schema;
+use vector_lookup::{PathPrefix, lookup_v2::ValuePath};
 
 use crate::{internal_events::DnstapParseWarning, schema::DNSTAP_VALUE_PATHS};
 
@@ -151,13 +147,13 @@ impl DnstapParser {
                 && let Err(err) =
                     DnstapParser::parse_dnstap_message(event, &root, message, parsing_options)
             {
-                emit!(DnstapParseWarning { error: &err });
+                emit(DnstapParseWarning { error: &err });
                 need_raw_data = true;
                 DnstapParser::insert(event, &root, &DNSTAP_VALUE_PATHS.error, err.to_string());
             }
         } else {
-            emit!(DnstapParseWarning {
-                error: format!("Unknown dnstap data type: {dnstap_data_type_id}")
+            emit(DnstapParseWarning {
+                error: format!("Unknown dnstap data type: {dnstap_data_type_id}"),
             });
             need_raw_data = true;
         }

lib/dnstap-parser/src/schema.rs

@@ -1,6 +1,6 @@
 use std::{collections::BTreeMap, sync::LazyLock};
 
-use vector_lib::lookup::{OwnedValuePath, owned_value_path};
+use vector_lookup::{OwnedValuePath, owned_value_path};
 use vrl::{
     btreemap,
     value::{
@@ -135,8 +135,8 @@ impl DnstapEventSchema {
     /// Schema definition for fields stored in the root.
     fn root_schema_definition(
         &self,
-        schema: vector_lib::schema::Definition,
-    ) -> vector_lib::schema::Definition {
+        schema: vector_core::schema::Definition,
+    ) -> vector_core::schema::Definition {
         schema
             .optional_field(&DNSTAP_VALUE_PATHS.server_identity, Kind::bytes(), None)
             .optional_field(&DNSTAP_VALUE_PATHS.server_version, Kind::bytes(), None)
@@ -152,8 +152,8 @@ impl DnstapEventSchema {
     /// Schema definition from the message.
     pub fn message_schema_definition(
         &self,
-        schema: vector_lib::schema::Definition,
-    ) -> vector_lib::schema::Definition {
+        schema: vector_core::schema::Definition,
+    ) -> vector_core::schema::Definition {
         schema
             .optional_field(&DNSTAP_VALUE_PATHS.socket_family, Kind::bytes(), None)
             .optional_field(&DNSTAP_VALUE_PATHS.socket_protocol, Kind::bytes(), None)
@@ -179,8 +179,8 @@ impl DnstapEventSchema {
     /// The schema definition for a dns tap message.
     pub fn schema_definition(
         &self,
-        schema: vector_lib::schema::Definition,
-    ) -> vector_lib::schema::Definition {
+        schema: vector_core::schema::Definition,
+    ) -> vector_core::schema::Definition {
         self.root_schema_definition(self.message_schema_definition(schema))
     }
 }

lib/dnstap-parser/src/vrl_functions/parse_dnstap.rs

@@ -1,6 +1,6 @@
 use base64::prelude::{BASE64_STANDARD, Engine as _};
 use dnsmsg_parser::dns_message_parser::DnsParserOptions;
-use vector_lib::event::LogEvent;
+use vector_core::event::LogEvent;
 use vrl::prelude::*;
 
 use crate::{parser::DnstapParser, schema::DnstapEventSchema};

lib/vector-common-macros/src/internal_event.rs

@@ -27,7 +27,7 @@ pub fn derive_impl_named_internal_event(item: TokenStream) -> TokenStream {
     let pkg_name = std::env::var("CARGO_PKG_NAME").unwrap_or_default();
     let internal_event_path = if pkg_name == "vector-common" {
         quote! { crate::internal_event }
-    } else if pkg_name.starts_with("vector-") {
+    } else if pkg_name.starts_with("vector-") || pkg_name == "dnstap-parser" {
         // Most vector-* crates depend on vector-common but not vector-lib
         quote! { ::vector_common::internal_event }
     } else {

lib/vector-vrl/cli/Cargo.toml

@@ -8,5 +8,5 @@ license = "MPL-2.0"
 
 [dependencies]
 clap.workspace = true
-vector-vrl-functions = { path = "../functions" }
+vector-vrl-functions.workspace = true
 vrl.workspace = true

lib/vector-vrl/cli/src/main.rs

@@ -2,8 +2,6 @@ use clap::Parser;
 use vrl::cli::{Opts, cmd::cmd};
 
 fn main() {
-    let mut functions = vrl::stdlib::all();
-    functions.extend(vector_vrl_functions::all());
-
+    let functions = vector_vrl_functions::all();
     std::process::exit(cmd(&Opts::parse(), functions));
 }

lib/vector-vrl/functions/Cargo.toml

@@ -8,3 +8,9 @@ license = "MPL-2.0"
 
 [dependencies]
 vrl.workspace = true
+enrichment = { path = "../../enrichment" }
+dnstap-parser = { path = "../../dnstap-parser", optional = true }
+
+[features]
+default = []
+dnstap = ["dep:dnstap-parser"]

lib/vector-vrl/functions/src/lib.rs

@@ -1,3 +1,12 @@
+//! Central location for all VRL functions used in Vector.
+//!
+//! This crate provides a single source of truth for the complete set of VRL functions
+//! available throughout Vector, combining:
+//! - Standard VRL library functions (`vrl::stdlib::all`)
+//! - Vector-specific functions (`vector_vrl::secret_functions`)
+//! - Enrichment table functions (`enrichment::vrl_functions`)
+//! - DNS tap parsing functions (optional, with `dnstap` feature)
+
 #![deny(warnings)]
 
 use vrl::{compiler::Function, path::OwnedTargetPath};
@@ -16,11 +25,26 @@ pub enum MetadataKey {
 
 pub const LEGACY_METADATA_KEYS: [&str; 2] = ["datadog_api_key", "splunk_hec_token"];
 
-pub fn all() -> Vec<Box<dyn Function>> {
+/// Returns Vector-specific secret functions.
+pub fn secret_functions() -> Vec<Box<dyn Function>> {
     vec![
         Box::new(set_semantic_meaning::SetSemanticMeaning) as _,
         Box::new(get_secret::GetSecret) as _,
         Box::new(remove_secret::RemoveSecret) as _,
         Box::new(set_secret::SetSecret) as _,
     ]
 }
+
+/// Returns all VRL functions available in Vector.
+#[allow(clippy::disallowed_methods)]
+pub fn all() -> Vec<Box<dyn Function>> {
+    let functions = vrl::stdlib::all()
+        .into_iter()
+        .chain(secret_functions())
+        .chain(enrichment::vrl_functions());
+
+    #[cfg(feature = "dnstap")]
+    let functions = functions.chain(dnstap_parser::vrl_functions());
+
+    functions.collect()
+}

lib/vector-vrl/tests/Cargo.toml

@@ -7,10 +7,9 @@ publish = false
 
 [dependencies]
 chrono-tz.workspace = true
-dnstap-parser = { path = "../../dnstap-parser" }
+vector-vrl-functions = { workspace = true, features = ["dnstap"] }
 enrichment = { path = "../../enrichment" }
 vrl.workspace = true
-vector-vrl-functions = { path = "../../vector-vrl/functions" }
 
 clap.workspace = true
 glob.workspace = true

lib/vector-vrl/tests/src/main.rs

@@ -4,7 +4,7 @@
 mod docs;
 mod test_enrichment;
 
-use std::{env, path::PathBuf};
+use std::{collections::HashSet, env, path::PathBuf};
 
 use chrono_tz::Tz;
 use clap::Parser;
@@ -96,15 +96,10 @@ fn main() {
         timezone: cmd.timezone(),
     };
 
-    let mut functions = vrl::stdlib::all();
-    functions.extend(vector_vrl_functions::all());
-    functions.extend(dnstap_parser::vrl_functions());
-    functions.extend(enrichment::vrl_functions());
-
     run_tests(
         tests,
         &cfg,
-        &functions,
+        &vector_vrl_functions::all(),
         || {
             let mut config = CompileConfig::default();
             let enrichment_table = test_enrichment::test_enrichment_table();
@@ -122,21 +117,29 @@ pub fn test_dir() -> PathBuf {
 fn test_glob_pattern() -> String {
     test_dir().join("**/*.vrl").to_str().unwrap().to_string()
 }
+
+#[allow(clippy::disallowed_methods)]
 fn get_tests(cmd: &Cmd) -> Vec<Test> {
+    // Don't test vrl stdlib functions examples since they are already tested in VRL and some will
+    // fail to compile since they are missing required source files such as proto definitions.
+    let ignore_examples_from_functions: HashSet<String> = vrl::stdlib::all()
+        .into_iter()
+        .map(|f| format!("functions/{}", f.identifier()))
+        .collect();
+
+    let tests_from_functions = get_tests_from_functions(vector_vrl_functions::all());
+    let tests_from_functions = tests_from_functions
+        .into_iter()
+        .filter(|test| !ignore_examples_from_functions.contains(&test.category));
+
     glob(test_glob_pattern().as_str())
         .expect("valid pattern")
         .filter_map(|entry| {
             let path = entry.ok()?;
             Some(Test::from_path(&path))
         })
         .chain(docs::tests(cmd.ignore_cue))
-        .chain(get_tests_from_functions(
-            vector_vrl_functions::all()
-                .into_iter()
-                .chain(enrichment::vrl_functions())
-                .chain(dnstap_parser::vrl_functions())
-                .collect(),
-        ))
+        .chain(tests_from_functions)
         .filter(|test| {
             should_run(
                 &format!("{}/{}", test.category, test.name),

lib/vector-vrl/web-playground/Cargo.toml

@@ -17,7 +17,7 @@ vrl.workspace = true
 serde.workspace = true
 web-sys = { version = "0.3", features = ["Window", "Performance"] }
 gloo-utils = { version = "0.2", features = ["serde"] }
-vector-vrl-functions = { path = "../functions" }
+vector-vrl-functions.workspace = true
 enrichment = { path = "../../enrichment" }
 # Required per https://docs.rs/getrandom/latest/getrandom/#webassembly-support
 getrandom = { version = "0.2.15", features = ["js"] }

lib/vector-vrl/web-playground/src/lib.rs

@@ -77,9 +77,7 @@ fn compile(
     mut input: Input,
     tz_str: Option<String>,
 ) -> Result<VrlCompileResult, VrlDiagnosticResult> {
-    let mut functions = vrl::stdlib::all();
-    functions.extend(vector_vrl_functions::all());
-    functions.extend(enrichment::vrl_functions());
+    let functions = vector_vrl_functions::all();
 
     let event = &mut input.event;
     let state = TypeState::default();

src/cli.rs

@@ -369,11 +369,7 @@ impl SubCommand {
             #[cfg(feature = "top")]
             Self::Top(t) => top::cmd(t).await,
             Self::Validate(v) => validate::validate(v, color).await,
-            Self::Vrl(s) => {
-                let mut functions = vrl::stdlib::all();
-                functions.extend(vector_vrl_functions::all());
-                vrl::cli::cmd::cmd(s, functions)
-            }
+            Self::Vrl(s) => vrl::cli::cmd::cmd(s, vector_vrl_functions::all()),
         }
     }
 }