Skip to content

Commit 785cc5d

Browse files
evanxg852000evanxg852000
authored
Updated using-vector tutorial (#3534)
* updated using-vector tutorial * few improvements * uncomment and fix index name * delete unused index-config file
1 parent 7aafbd8 commit 785cc5d

File tree

3 files changed

+106
-125
lines changed

3 files changed

+106
-125
lines changed

‎config/tutorials/vector-otel-logs/index-config.yaml‎

Lines changed: 0 additions & 36 deletions
This file was deleted.

‎config/tutorials/vector-otel-logs/vector.toml‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,24 @@
11
[sources.generate_syslog]
22
type = "demo_logs"
33
format = "syslog"
4-
count = 100
4+
count = 100000
5+
interval = 0.001
56

67
[transforms.remap_syslog]
78
inputs = [ "generate_syslog"]
89
type = "remap"
910
source = '''
1011
structured = parse_syslog!(.message)
11-
.timestamp, err = to_unix_timestamp(structured.timestamp)
12-
.body = .message
13-
del(.message)
14-
.resource.source_type = .source_type
15-
.resource.host.hostname = structured.hostname
16-
.resource.service.name = structured.appname
12+
.timestamp_nanos, err = to_unix_timestamp(structured.timestamp, unit: "nanoseconds")
13+
.body = structured
14+
.service_name = structured.appname
15+
.resource_attributes.source_type = .source_type
16+
.resource_attributes.host.hostname = structured.hostname
17+
.resource_attributes.service.name = structured.appname
1718
.attributes.syslog.procid = structured.procid
1819
.attributes.syslog.facility = structured.facility
1920
.attributes.syslog.version = structured.version
20-
del(.source_type)
21-
.severity = if includes(["emerg", "err", "crit", "alert"], structured.severity) {
21+
.severity_text = if includes(["emerg", "err", "crit", "alert"], structured.severity) {
2222
"ERROR"
2323
} else if structured.severity == "warning" {
2424
"WARN"
@@ -29,7 +29,10 @@ source = '''
2929
} else {
3030
structured.severity
3131
}
32-
.name = structured.msgid
32+
.scope_name = structured.msgid
33+
del(.message)
34+
del(.timestamp)
35+
del(.source_type)
3336
'''
3437

3538
[sinks.emit_syslog]
@@ -39,8 +42,8 @@ encoding.codec = "json"
3942

4043
[sinks.quickwit_logs]
4144
type = "http"
45+
method = "post"
4246
inputs = ["remap_syslog"]
4347
encoding.codec = "json"
4448
framing.method = "newline_delimited"
45-
uri = "http://host.docker.internal:7280/api/v1/otel-logs/bulk"
46-
49+
uri = "http://127.0.0.1:7280/api/v1/otel-logs-v0_6/ingest"

‎docs/log-management/send-logs/using-vector.md‎

Lines changed: 91 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -34,88 +34,96 @@ mkdir qwdata
3434
```bash
3535
# Create Quickwit data dir.
3636
mkdir qwdata
37-
docker run --rm -v $(pwd)/qwdata:/quickwit/qwdata -p 127.0.0.1:7280:7280 quickwit/quickwit run
37+
docker run --rm -v $(pwd)/qwdata:/quickwit/qwdata -p 7280:7280 quickwit/quickwit run
3838
```
3939

4040
</TabItem>
4141

4242
</Tabs>
4343

44-
## Create an index for logs
44+
## Taking advantage of Quickwit's native support for logs
4545

46-
Let's embrace the OpenTelemetry standard and create an index compatible with its [logs data model](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md).
46+
Let's embrace the OpenTelemetry standard and take advantage of Quickwit features. With the native support for OpenTelemetry standards, Quickwit already comes with an index called `otel-logs_v0_6` that is compatible with the OpenTelemetry [logs data model](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md). This means we can start pushing log data without any prior usual index setup.
4747

48-
```yaml title="index-config.yaml"
49-
#
50-
# Index config file for receiving logs in OpenTelemetry format.
51-
# Link: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md
52-
#
48+
Here is the OpenTelemetry index configuration for reference.
5349

50+
```yaml title="otel-logs_v0_6"
5451
version: 0.6
5552

56-
index_id: vector-otel-logs
53+
index_id: otel-logs-v0_6
5754

5855
doc_mapping:
56+
mode: strict
5957
field_mappings:
60-
- name: timestamp
58+
- name: timestamp_nanos
6159
type: datetime
62-
input_formats:
63-
- unix_timestamp
64-
output_format: unix_timestamp_secs
60+
input_formats: [unix_timestamp]
61+
output_format: unix_timestamp_nanos
62+
indexed: false
6563
fast: true
66-
- name: severity
64+
precision: milliseconds
65+
- name: observed_timestamp_nanos
66+
type: datetime
67+
input_formats: [unix_timestamp]
68+
output_format: unix_timestamp_nanos
69+
- name: service_name
70+
type: text
71+
tokenizer: raw
72+
- name: severity_text
6773
type: text
6874
tokenizer: raw
6975
fast: true
76+
- name: severity_number
77+
type: u64
78+
fast: true
7079
- name: body
71-
type: text
72-
tokenizer: default
73-
record: position
80+
type: json
7481
- name: attributes
7582
type: json
76-
- name: resource
83+
tokenizer: raw
84+
fast: true
85+
- name: dropped_attributes_count
86+
type: u64
87+
indexed: false
88+
- name: trace_id
89+
type: bytes
90+
- name: span_id
91+
type: bytes
92+
- name: trace_flags
93+
type: u64
94+
indexed: false
95+
- name: resource_attributes
7796
type: json
78-
timestamp_field: timestamp
97+
tokenizer: raw
98+
fast: true
99+
- name: resource_dropped_attributes_count
100+
type: u64
101+
indexed: false
102+
- name: scope_name
103+
type: text
104+
indexed: false
105+
- name: scope_version
106+
type: text
107+
indexed: false
108+
- name: scope_attributes
109+
type: json
110+
indexed: false
111+
- name: scope_dropped_attributes_count
112+
type: u64
113+
indexed: false
79114

80-
search_settings:
81-
default_search_fields: [severity, body]
115+
timestamp_field: timestamp_nanos
82116

83117
indexing_settings:
84-
commit_timeout_secs: 10
85-
```
86-
87-
First download the YAML file:
88-
89-
```bash
90-
curl -o vector-otel-logs.yaml https://raw.githubusercontent.com/quickwit-oss/quickwit/main/config/tutorials/vector-otel-logs/index-config.yaml
91-
```
92-
93-
And then create the index with `cURL` or the `CLI`:
94-
95-
<Tabs>
96-
97-
<TabItem value="curl" label="cURL">
118+
commit_timeout_secs: 5
98119

99-
```bash
100-
curl -XPOST http://localhost:7280/api/v1/indexes -H "content-type: application/yaml" --data-binary @vector-otel-logs.yaml
101-
```
102-
103-
</TabItem>
104-
105-
<TabItem value="cli" label="CLI">
106-
107-
```bash
108-
./quickwit index create --index-config vector-otel-logs.yaml
120+
search_settings:
121+
default_search_fields: [body.message]
109122
```
110123
111-
</TabItem>
112-
113-
</Tabs>
114-
115-
116124
## Setup Vector
117125
118-
Our sink here will be Quickwit ingest API `http://127.0.0.1:7280/api/v1/otel-logs/ingest`.
126+
Our sink here will be Quickwit ingest API `http://127.0.0.1:7280/api/v1/otel-logs-v0_6/ingest`.
119127
To keep it simple in this tutorial, we will use a log source called `demo_logs` that generates logs in a given format. Let's choose the common `syslog` format
120128
(Vector does not generate logs in the OpenTelemetry format directly!) and use the transform feature to map the `syslog` format into the OpenTelemetry format.
121129

@@ -132,17 +140,16 @@ inputs = [ "generate_syslog"]
132140
type = "remap"
133141
source = '''
134142
structured = parse_syslog!(.message)
135-
.timestamp, err = to_unix_timestamp(structured.timestamp, unit: "milliseconds")
136-
.body = .message
137-
del(.message)
138-
.resource.source_type = .source_type
139-
.resource.host.hostname = structured.hostname
140-
.resource.service.name = structured.appname
143+
.timestamp_nanos, err = to_unix_timestamp(structured.timestamp, unit: "nanoseconds")
144+
.body = structured
145+
.service_name = structured.appname
146+
.resource_attributes.source_type = .source_type
147+
.resource_attributes.host.hostname = structured.hostname
148+
.resource_attributes.service.name = structured.appname
141149
.attributes.syslog.procid = structured.procid
142150
.attributes.syslog.facility = structured.facility
143151
.attributes.syslog.version = structured.version
144-
del(.source_type)
145-
.severity = if includes(["emerg", "err", "crit", "alert"], structured.severity) {
152+
.severity_text = if includes(["emerg", "err", "crit", "alert"], structured.severity) {
146153
"ERROR"
147154
} else if structured.severity == "warning" {
148155
"WARN"
@@ -153,38 +160,45 @@ source = '''
153160
} else {
154161
structured.severity
155162
}
156-
.name = structured.msgid
163+
.scope_name = structured.msgid
164+
del(.message)
165+
del(.timestamp)
166+
del(.source_type)
157167
'''
158168
159169
# useful to see the logs in the terminal
160-
#[sinks.emit_syslog]
161-
#inputs = ["remap_syslog"]
162-
#type = "console"
163-
#encoding.codec = "json"
170+
# [sinks.emit_syslog]
171+
# inputs = ["remap_syslog"]
172+
# type = "console"
173+
# encoding.codec = "json"
164174
165175
[sinks.quickwit_logs]
166176
type = "http"
167177
method = "post"
168178
inputs = ["remap_syslog"]
169179
encoding.codec = "json"
170180
framing.method = "newline_delimited"
171-
uri = "http://host.docker.internal:7280/api/v1/vector-otel-logs/ingest"
181+
uri = "http://127.0.0.1:7280/api/v1/otel-logs-v0_6/ingest"
172182
```
183+
Download the above Vector config file.
173184

174-
Now let's start Vector to start send logs to Quickwit.
185+
```bash
186+
curl -o vector.toml https://raw.githubusercontent.com/quickwit-oss/quickwit/main/config/tutorials/vector-otel-logs/vector.toml
187+
```
188+
189+
Now let's start Vector so that we can start sending logs to Quickwit.
175190

176191
```bash
177-
docker run -v $(pwd)/vector.toml:/etc/vector/vector.toml:ro -p 8383:8383 --add-host=host.docker.internal:host-gateway timberio/vector:0.25.0-distroless-libc
192+
docker run -v $(pwd)/vector.toml:/etc/vector/vector.toml:ro -p 8383:8383 --net=host timberio/vector:0.25.0-distroless-libc
178193
```
179194

180195
## Search logs
181196

182197
Quickwit is now ingesting logs coming from Vector and you can search them either with `curl` or by using the UI:
183-
- `curl -XGET http://127.0.0.1:7280/api/v1/vector-otel-logs/search\?query\=severity:ERROR`
184-
- Open your browser at `http://127.0.0.1:7280/ui/search?query=severity:ERROR&index_id=vector-otel-logs&max_hits=10` and play with it!
185-
198+
- `curl -XGET http://127.0.0.1:7280/api/v1/otel-logs-v0_6/search?query=severity_text:ERROR`
199+
- Open your browser at `http://127.0.0.1:7280/ui/search?query=severity_text:ERROR&index_id=otel-logs-v0_6&max_hits=10` and play with it!
186200

187-
## Compute aggregation on severity
201+
## Compute aggregation on severity_text
188202

189203
For aggregations, we can't use yet Quickwit UI but we can use cURL.
190204

@@ -197,13 +211,13 @@ Let's craft a nice aggregation query to count how many `INFO`, `DEBUG`, `WARN`,
197211
"aggs": {
198212
"count_per_minute": {
199213
"histogram": {
200-
"field": "timestamp",
214+
"field": "timestamp_nanos",
201215
"interval": 60000000
202216
},
203217
"aggs": {
204-
"severity_count": {
218+
"severity_text_count": {
205219
"terms": {
206-
"field": "severity"
220+
"field": "severity_text"
207221
}
208222
}
209223
}
@@ -213,9 +227,9 @@ Let's craft a nice aggregation query to count how many `INFO`, `DEBUG`, `WARN`,
213227
```
214228

215229
```bash
216-
curl -XPOST -H "Content-Type: application/json" http://127.0.0.1:7280/api/v1/vector-otel-logs/search --data @aggregation-query.json
230+
curl -XPOST -H "Content-Type: application/json" http://127.0.0.1:7280/api/v1/otel-logs-v0_6/search --data @aggregation-query.json
217231
```
218232

219-
## Further improvements
233+
## Going further
220234

221-
Coming soon: deploy Vector + Quickwit on your infrastructure, use Grafana to query Quickwit, and more!
235+
Now you can also deploy Grafana and connect to Quickwit as data source for query, dashboard, alerts and more!

0 commit comments

Comments
 (0)