Merge branch 'master' into add-ch-arrow-codec

Author: benjamin-awd

.github/workflows/integration.yml

@@ -46,15 +46,15 @@ jobs:
 
   build-test-runner:
     needs: changes
-    if: |
-      ${{
-        github.event_name == 'workflow_dispatch' ||
-        (github.event_name == 'merge_group' &&
-         (needs.changes.outputs.dependencies == 'true' ||
-          needs.changes.outputs.integration-yml == 'true' ||
-          needs.changes.outputs.int-tests-any == 'true' ||
-          needs.changes.outputs.e2e-tests-any == 'true'))
-      }}
+    if: ${{
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'merge_group' &&
+       (needs.changes.outputs.dependencies == 'true' ||
+        needs.changes.outputs.integration-yml == 'true' ||
+        needs.changes.outputs.int-tests-any == 'true' ||
+        needs.changes.outputs.e2e-tests-any == 'true'))
+     }}
+
     uses: ./.github/workflows/build-test-runner.yml
     with:
       commit_sha: ${{ github.sha }}
@@ -65,7 +65,7 @@ jobs:
       - changes
       - build-test-runner
 
-    if: ${{ !failure() && !cancelled() && (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch') }}
+    if: ${{ !failure() && !cancelled() && needs.build-test-runner.result == 'success' && (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch') }}
     strategy:
       matrix:
         # TODO: Add "splunk" back once https://github.com/vectordotdev/vector/issues/23474 is fixed.
@@ -133,7 +133,7 @@ jobs:
     needs:
       - changes
       - build-test-runner
-    if: ${{ !failure() && !cancelled() && (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch') }}
+    if: ${{ !failure() && !cancelled() && needs.build-test-runner.result == 'success' && (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch') }}
     strategy:
       matrix:
         service: [ "datadog-logs", "datadog-metrics", "opentelemetry-logs", "opentelemetry-metrics" ]

Cargo.toml

@@ -732,6 +732,7 @@ transforms-logs = [
   "transforms-exclusive-route",
   "transforms-sample",
   "transforms-throttle",
+  "transforms-trace_to_log"
 ]
 transforms-metrics = [
   "transforms-aggregate",
@@ -761,6 +762,7 @@ transforms-exclusive-route = []
 transforms-sample = ["transforms-impl-sample"]
 transforms-tag_cardinality_limit = ["dep:bloomy", "dep:hashbrown"]
 transforms-throttle = ["dep:governor"]
+transforms-trace_to_log = []
 
 # Implementations of transforms
 transforms-impl-sample = []

changelog.d/24081_memory_tables_in_tests.fix.md

@@ -0,0 +1,3 @@
+Fixed an issue in vector tests where memory enrichment tables would report missing components errors.
+
+authors: esensar Quad9DNS

changelog.d/add-trace-to-log-transform.feature.md

@@ -0,0 +1,3 @@
+Introduced `trace_to_log` transform that allows converting traces to logs.
+
+authors: huevosabio

changelog.d/internal_logs_rate_limiting.fix.md

@@ -0,0 +1,4 @@
+The `internal_logs` source now captures all internal Vector logs without rate limiting. Previously, repeated log messages were silently
+dropped.
+
+authors: pront

lib/codecs/src/encoding/serializer.rs

@@ -17,9 +17,20 @@ use super::format::{
 };
 #[cfg(feature = "opentelemetry")]
 use super::format::{OtlpSerializer, OtlpSerializerConfig};
-use super::framing::{
-    CharacterDelimitedEncoderConfig, FramingConfig, LengthDelimitedEncoderConfig,
-    VarintLengthDelimitedEncoderConfig,
+use super::{
+    chunking::Chunker,
+    format::{
+        AvroSerializer, AvroSerializerConfig, AvroSerializerOptions, CefSerializer,
+        CefSerializerConfig, CsvSerializer, CsvSerializerConfig, GelfSerializer,
+        GelfSerializerConfig, JsonSerializer, JsonSerializerConfig, LogfmtSerializer,
+        LogfmtSerializerConfig, NativeJsonSerializer, NativeJsonSerializerConfig, NativeSerializer,
+        NativeSerializerConfig, ProtobufSerializer, ProtobufSerializerConfig, RawMessageSerializer,
+        RawMessageSerializerConfig, TextSerializer, TextSerializerConfig,
+    },
+    framing::{
+        CharacterDelimitedEncoderConfig, FramingConfig, LengthDelimitedEncoderConfig,
+        VarintLengthDelimitedEncoderConfig,
+    },
 };
 
 /// Serializer configuration.

lib/vector-core/src/event/trace.rs

@@ -131,6 +131,12 @@ impl From<LogEvent> for TraceEvent {
     }
 }
 
+impl From<TraceEvent> for LogEvent {
+    fn from(trace: TraceEvent) -> Self {
+        trace.0
+    }
+}
+
 impl From<ObjectMap> for TraceEvent {
     fn from(map: ObjectMap) -> Self {
         Self(map.into())

src/config/unit_test/mod.rs

@@ -432,6 +432,15 @@ async fn build_unit_test(
         .collect::<Vec<_>>();
     valid_components.extend(unexpanded_transforms);
 
+    // Enrichment tables consume inputs but are referenced dynamically in VRL transforms
+    // (via get_enrichment_table_record). Since we can't statically analyze VRL usage,
+    // we conservatively include all enrichment table inputs as valid components.
+    config_builder
+        .enrichment_tables
+        .iter()
+        .filter_map(|(key, c)| c.as_sink(key).map(|(_, sink)| sink.inputs))
+        .for_each(|i| valid_components.extend(i.into_iter()));
+
     // Remove all transforms that are not relevant to the current test
     config_builder.transforms = config_builder
         .transforms

src/sinks/splunk_hec/common/acknowledgements.rs

@@ -1,8 +1,8 @@
 use hyper::Body;
 use serde::{Deserialize, Serialize};
-use std::io::Write;
 use std::{
     collections::HashMap,
+    io::Write,
     num::{NonZeroU8, NonZeroU64},
     sync::Arc,
     time::Duration,
@@ -11,14 +11,14 @@ use tokio::sync::{mpsc::Receiver, oneshot::Sender};
 use vector_lib::{configurable::configurable_component, event::EventStatus};
 
 use super::service::{HttpRequestBuilder, MetadataFields};
-use crate::sinks::util::Compressor;
 use crate::{
     config::AcknowledgementsConfig,
     http::HttpClient,
     internal_events::{
         SplunkIndexerAcknowledgementAPIError, SplunkIndexerAcknowledgementAckAdded,
         SplunkIndexerAcknowledgementAcksRemoved,
     },
+    sinks::util::Compressor,
 };
 
 /// Splunk HEC acknowledgement configuration.

src/sources/internal_logs.rs

@@ -213,6 +213,8 @@ mod tests {
     use vector_lib::{event::Value, lookup::OwnedTargetPath};
     use vrl::value::kind::Collection;
 
+    use serial_test::serial;
+
     use super::*;
     use crate::{
         event::Event,
@@ -234,6 +236,7 @@ mod tests {
     // cases because `consume_early_buffer` (called within the
     // `start_source` helper) panics when called more than once.
     #[tokio::test]
+    #[serial]
     async fn receives_logs() {
         trace::init(false, false, "debug", 10);
         trace::reset_early_buffer();
@@ -341,6 +344,43 @@ mod tests {
         rx
     }
 
+    // NOTE: This test requires #[serial] because it directly interacts with global tracing state.
+    // This is a pre-existing limitation around tracing initialization in tests.
+    #[tokio::test]
+    #[serial]
+    async fn repeated_logs_are_not_rate_limited() {
+        trace::init(false, false, "info", 10);
+        trace::reset_early_buffer();
+
+        let rx = start_source().await;
+
+        // Generate 20 identical log messages with the same component_id
+        for _ in 0..20 {
+            info!(component_id = "test", "Repeated test message.");
+        }
+
+        sleep(Duration::from_millis(50)).await;
+        let events = collect_ready(rx).await;
+
+        // Filter to only our test messages
+        let test_events: Vec<_> = events
+            .iter()
+            .filter(|e| {
+                e.as_log()
+                    .get("message")
+                    .map(|m| m.to_string_lossy() == "Repeated test message.")
+                    .unwrap_or(false)
+            })
+            .collect();
+
+        // We should receive all 20 messages, no rate limiting.
+        assert_eq!(
+            test_events.len(),
+            20,
+            "internal_logs source should capture all repeated messages without rate limiting"
+        );
+    }
+
     #[test]
     fn output_schema_definition_vector_namespace() {
         let config = InternalLogsConfig::default();