How to completely collect multi-line logs, like filebeat

[uglyliu]

My configuration is as follows

data_dir = "/data/svr/vector/data"
timezone = "local"

[api]
enabled = false
address = "[::]:8686"
playground = false

[sources.file1]
include = [ "/data/log/**/*.log" ]
type = "file"
ignore_checkpoints = true
read_from = "end"

  [sources.file1.fingerprint]
  strategy = "device_and_inode"

  [sources.file1.multiline]
  start_pattern = '^\[0-9]{4}-[0-9]{2}-[0-9]{2}'
  mode = "continue_past"
  condition_pattern = '^\[0-9]{4}-[0-9]{2}-[0-9]{2}'
  timeout_ms = 1000

[transforms.add_host]
inputs = ["file1"]
type = "remap"
source = """
.pod_ip = "${POD_IP}"
.host_ip = "${HOST_IP}"

"""


[sources.internal_metrics]
type = "internal_metrics"

[sinks.prom_exporter]
type = "prometheus_exporter"
inputs = [ "internal_metrics" ]
address = "[::]:9092"

[sinks.stdout]
type = "console"
inputs = ["add_host"]
[sinks.stdout.encoding]
  codec = "json"

When I execute, the log output by the console is as follows，Not treating the complete log as one line，Sorry, how should I modify the configuration?

{"file":"/data/log/user-center/user-center-20240317.log","host":"ip-10-65-1-163.ap-northeast-1.compute.internal","host_ip":"122.168.1.1","message":"2022-04-01 10:15:31.234 - ERROR - An error occurred: NullPointerException","pod_ip":"192.168.1.1","source_type":"file","timestamp":"2024-03-17T02:43:04.398117897Z"}
{"file":"/data/log/user-center/user-center-20240317.log","host":"ip-10-65-1-163.ap-northeast-1.compute.internal","host_ip":"122.168.1.1","message":"\tat com.example.MyClass.method1(MyClass.java:25)","pod_ip":"192.168.1.1","source_type":"file","timestamp":"2024-03-17T02:43:04.398160037Z"}
{"file":"/data/log/user-center/user-center-20240317.log","host":"ip-10-65-1-163.ap-northeast-1.compute.internal","host_ip":"122.168.1.1","message":"\tat com.example.MyClass.method2(MyClass.java:35)","pod_ip":"192.168.1.1","source_type":"file","timestamp":"2024-03-17T02:43:04.398163307Z"}
{"file":"/data/log/user-center/user-center-20240317.log","host":"ip-10-65-1-163.ap-northeast-1.compute.internal","host_ip":"122.168.1.1","message":"\tat com.example.MyClass.method2(MyClass.java:35)","pod_ip":"192.168.1.1","source_type":"file","timestamp":"2024-03-17T02:43:04.398165848Z"}

[uglyliu]

it is work out ,my whole config

data_dir = "/data/svr/vector/data"
timezone = "local"

[api]
enabled = false
address = "[::]:8686"
playground = false

[sources.file1]
include = [ "/data/log/**/*.log" ]
type = "file"
ignore_checkpoints = true
read_from = "end"

  [sources.file1.fingerprint]
  strategy = "device_and_inode"

  [sources.file1.multiline]
  start_pattern = "^[0-9]{4}-[0-9]{2}-[0-9]{2}"
  mode = "halt_before"
  condition_pattern = "^[0-9]{4}-[0-9]{2}-[0-9]{2}"
  timeout_ms = 1000

[transforms.add_host]
type = "remap"
inputs = ["file1"]
source = '''
.pod_ip = "${POD_IP}"
.host_ip = "${HOST_IP}"
.message2 = parse_regex!(.message, r'(?s)^(?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) (?P<log>.*)')
.message = .message2.timestamp + " " + "pod_ip=" + .pod_ip + " " + "host_ip=" + .host_ip + " " + .message2.log
'''

[sources.internal_metrics]
type = "internal_metrics"

[sinks.prom_exporter]
type = "prometheus_exporter"
inputs = [ "internal_metrics" ]
address = "[::]:9092"

[sinks.vector1]
type = "vector"
inputs = [ "add_host" ]
address = "10.65.1.248:8688"
version = "2"