enhancement(vrl): consolidate all VRL functions into vector-vrl-functions crate (#24402)

* Create all_vrl_functions

* Create and use vector-vrl-all

* Move dnstap-parser off of vector-lib

* Use vector-vrl-all in lib/codecs

* refactor: use vector-vrl-all as workspace dependency

* docs: add crate-level documentation for vector-vrl-all

* Use LazyLock to cache VRL function list

* Revert "Use LazyLock to cache VRL function list"

This reverts commit 04616c2.

* Add back changelog

* cargo fmt

* Skip testing vrl stdlib functions' examples

* Fix broken examples

* Move VRL function consolidation logic into vector_vrl_functions::all()

* Cargo fmt

* Add ignore for disallowed_methods

Author: thomasqueirozb

Cargo.lock

@@ -200,6 +200,7 @@ vector-config-common = { path = "lib/vector-config-common" }
 vector-config-macros = { path = "lib/vector-config-macros" }
 vector-common-macros = { path = "lib/vector-common-macros" }
 vector-lib = { path = "lib/vector-lib", default-features = false, features = ["vrl"] }
+vector-vrl-functions = { path = "lib/vector-vrl/functions" }
 vrl = { git = "https://github.com/vectordotdev/vrl.git", branch = "main", features = ["arbitrary", "cli", "test", "test_framework"] }
 mock_instant = { version = "0.6" }
 serial_test = { version = "3.2" }
@@ -227,7 +228,7 @@ vector-lib.workspace = true
 vector-config.workspace = true
 vector-config-common.workspace = true
 vector-config-macros.workspace = true
-vector-vrl-functions = { path = "lib/vector-vrl/functions" }
+vector-vrl-functions.workspace = true
 loki-logproto = { path = "lib/loki-logproto", optional = true }
 
 # Tokio / Futures
@@ -654,7 +655,7 @@ sources-aws_s3 = ["aws-core", "dep:aws-sdk-sqs", "dep:aws-sdk-s3", "dep:async-co
 sources-aws_sqs = ["aws-core", "dep:aws-sdk-sqs"]
 sources-datadog_agent = ["sources-utils-http-encoding", "protobuf-build", "dep:prost"]
 sources-demo_logs = ["dep:fakedata"]
-sources-dnstap = ["sources-utils-net-tcp", "dep:base64", "dep:hickory-proto", "dep:dnsmsg-parser", "dep:dnstap-parser", "protobuf-build", "dep:prost"]
+sources-dnstap = ["sources-utils-net-tcp", "dep:base64", "dep:hickory-proto", "dep:dnsmsg-parser", "dep:dnstap-parser", "protobuf-build", "dep:prost", "vector-vrl-functions/dnstap"]
 sources-docker_logs = ["docker"]
 sources-eventstoredb_metrics = []
 sources-exec = []

Cargo.toml

@@ -0,0 +1,4 @@
+Vector-specific VRL functions are now available everywhere. Previously some functions were not
+available inside codec VRL transforms and in the VRL cli (via `vector vrl`).
+
+authors: thomasqueirozb

changelog.d/vrl_functions_consolidation.enhancement.md

@@ -5,6 +5,7 @@ allow-unwrap-in-tests = true
 # https://rust-lang.github.io/rust-clippy/master/index.html#disallowed_method
 disallowed-methods = [
   { path = "std::io::Write::write", reason = "This doesn't handle short writes, use `write_all` instead." },
+  { path = "vrl::stdlib::all", reason = "Use `vector_vrl_functions::all()` instead for consistency across all Vector VRL functions." },
 ]
 
 disallowed-types = [

clippy.toml

@@ -46,6 +46,7 @@ vector-common = { path = "../vector-common", default-features = false }
 vector-config = { path = "../vector-config", default-features = false }
 vector-config-macros = { path = "../vector-config-macros", default-features = false }
 vector-core = { path = "../vector-core", default-features = false, features = ["vrl"] }
+vector-vrl-functions.workspace = true
 
 [dev-dependencies]
 futures.workspace = true

lib/codecs/Cargo.toml

@@ -59,7 +59,7 @@ impl VrlDeserializerConfig {
 
         match compile_vrl(
             &self.vrl.source,
-            &vrl::stdlib::all(),
+            &vector_vrl_functions::all(),
             &state,
             CompileConfig::default(),
         ) {

lib/codecs/src/decoding/format/vrl.rs

@@ -15,7 +15,10 @@ hickory-proto.workspace = true
 prost.workspace = true
 snafu.workspace = true
 tracing.workspace = true
-vector-lib.workspace = true
+vector-config.workspace = true
+vector-common = { path = "../vector-common" }
+vector-lookup = { path = "../vector-lookup", features = ["test"] }
+vector-core = { path = "../vector-core" }
 vrl.workspace = true
 paste.workspace = true
 

lib/dnstap-parser/Cargo.toml

@@ -1,5 +1,5 @@
 use tracing::warn;
-use vector_lib::{
+use vector_common::{
     NamedInternalEvent,
     internal_event::{InternalEvent, error_stage, error_type},
 };

lib/dnstap-parser/src/internal_events.rs

@@ -16,10 +16,8 @@ use hickory_proto::{
 };
 use prost::Message;
 use snafu::Snafu;
-use vector_lib::{
-    Error, Result, emit,
-    event::{LogEvent, Value},
-};
+use vector_common::{Error, Result, internal_event::emit};
+use vector_core::event::{LogEvent, Value};
 use vrl::{owned_value_path, path};
 
 #[allow(warnings, clippy::all, clippy::pedantic, clippy::nursery)]
@@ -38,10 +36,8 @@ use dnstap_proto::{
     Dnstap, Message as DnstapMessage, SocketFamily, SocketProtocol,
     message::Type as DnstapMessageType,
 };
-use vector_lib::{
-    config::log_schema,
-    lookup::{PathPrefix, lookup_v2::ValuePath},
-};
+use vector_core::config::log_schema;
+use vector_lookup::{PathPrefix, lookup_v2::ValuePath};
 
 use crate::{internal_events::DnstapParseWarning, schema::DNSTAP_VALUE_PATHS};
 
@@ -151,13 +147,13 @@ impl DnstapParser {
                 && let Err(err) =
                     DnstapParser::parse_dnstap_message(event, &root, message, parsing_options)
             {
-                emit!(DnstapParseWarning { error: &err });
+                emit(DnstapParseWarning { error: &err });
                 need_raw_data = true;
                 DnstapParser::insert(event, &root, &DNSTAP_VALUE_PATHS.error, err.to_string());
             }
         } else {
-            emit!(DnstapParseWarning {
-                error: format!("Unknown dnstap data type: {dnstap_data_type_id}")
+            emit(DnstapParseWarning {
+                error: format!("Unknown dnstap data type: {dnstap_data_type_id}"),
             });
             need_raw_data = true;
         }

lib/dnstap-parser/src/parser.rs

@@ -1,6 +1,6 @@
 use std::{collections::BTreeMap, sync::LazyLock};
 
-use vector_lib::lookup::{OwnedValuePath, owned_value_path};
+use vector_lookup::{OwnedValuePath, owned_value_path};
 use vrl::{
     btreemap,
     value::{
@@ -135,8 +135,8 @@ impl DnstapEventSchema {
     /// Schema definition for fields stored in the root.
     fn root_schema_definition(
         &self,
-        schema: vector_lib::schema::Definition,
-    ) -> vector_lib::schema::Definition {
+        schema: vector_core::schema::Definition,
+    ) -> vector_core::schema::Definition {
         schema
             .optional_field(&DNSTAP_VALUE_PATHS.server_identity, Kind::bytes(), None)
             .optional_field(&DNSTAP_VALUE_PATHS.server_version, Kind::bytes(), None)
@@ -152,8 +152,8 @@ impl DnstapEventSchema {
     /// Schema definition from the message.
     pub fn message_schema_definition(
         &self,
-        schema: vector_lib::schema::Definition,
-    ) -> vector_lib::schema::Definition {
+        schema: vector_core::schema::Definition,
+    ) -> vector_core::schema::Definition {
         schema
             .optional_field(&DNSTAP_VALUE_PATHS.socket_family, Kind::bytes(), None)
             .optional_field(&DNSTAP_VALUE_PATHS.socket_protocol, Kind::bytes(), None)
@@ -179,8 +179,8 @@ impl DnstapEventSchema {
     /// The schema definition for a dns tap message.
     pub fn schema_definition(
         &self,
-        schema: vector_lib::schema::Definition,
-    ) -> vector_lib::schema::Definition {
+        schema: vector_core::schema::Definition,
+    ) -> vector_core::schema::Definition {
         self.root_schema_definition(self.message_schema_definition(schema))
     }
 }