Skip to content

Commit e543a36

Browse files
rcastleyrcastley
committed
Added master YAML files - completed workshop version
1 parent 3b6f381 commit e543a36

File tree

2 files changed

+256
-0
lines changed

2 files changed

+256
-0
lines changed

‎content/en/ninja-workshops/10-advanced-otel/master-agent.yaml‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
########################### This section holds all the
2+
## Configuration section ## configurations that can be
3+
########################### used in this OpenTelemetry Collector
4+
extensions: # Array of Extensions
5+
health_check: # Configures the health check extension
6+
endpoint: 0.0.0.0:13133 # Endpoint to collect health check data
7+
file_storage/checkpoint: # Extension Type/Name
8+
directory: "./checkpoint-dir" # Define directory
9+
create_directory: true # Create directory
10+
timeout: 1s # Timeout for file operations
11+
compaction: # Compaction settings
12+
on_start: true # Start compaction at Collector startup
13+
# Define compaction directory
14+
directory: "./checkpoint-dir/tmp"
15+
# Max. size limit before compaction occurs
16+
max_transaction_size: 65536
17+
18+
receivers: # Array of Receivers
19+
hostmetrics: # Receiver Type
20+
collection_interval: 3600s # Scrape metrics every hour
21+
scrapers: # Array of hostmetric scrapers
22+
cpu: # Scraper for cpu metrics
23+
otlp: # Receiver Type
24+
protocols: # list of Protocols used
25+
http: # This wil enable the HTTP Protocol
26+
endpoint: "0.0.0.0:4318" # Endpoint for incoming telemetry data
27+
filelog/quotes: # Receiver Type/Name
28+
include: ./quotes.log # The file to read log data from
29+
include_file_path: true # Include file path in the log data
30+
include_file_name: false # Exclude file name from the log data
31+
resource: # Add custom resource attributes
32+
com.splunk.source: ./quotes.log # Source of the log data
33+
com.splunk.sourcetype: quotes # Source type of the log data
34+
35+
exporters: # Array of Exporters
36+
debug: # Exporter Type
37+
verbosity: detailed # Enabled detailed debug output
38+
file: # Exporter Type
39+
path: "./agent.out" # Save path (OTLP JSON)
40+
append: false # Overwrite the file each time
41+
otlphttp: # Exporter Type
42+
endpoint: "http://localhost:5318" # Gateway OTLP endpoint
43+
headers: # Headers to add to the HTTPcall
44+
X-SF-Token: "ACCESS_TOKEN" # Splunk ACCESS_TOKEN header
45+
retry_on_failure: # Retry on failure settings
46+
enabled: true # Enables retrying
47+
sending_queue: # Sending queue settings
48+
enabled: true # Enables Sending queue
49+
num_consumers: 10 # Number of consumers
50+
queue_size: 10000 # Maximum queue size
51+
# File storage extension
52+
storage: file_storage/checkpoint
53+
54+
processors: # Array of Processors
55+
memory_limiter: # Limits memory usage by Collectors pipeline
56+
check_interval: 2s # Interval to check memory usage
57+
limit_mib: 512 # Memory limit in MiB
58+
resourcedetection: # Processor Type
59+
detectors: [system] # Detect system resource information
60+
override: true # Overwrites existing attributes
61+
resource/add_mode: # Processor Type/Name
62+
attributes: # Array of attributes and modifications
63+
- action: insert # Action is to insert a key
64+
key: otelcol.service.mode # Key name
65+
value: "agent" # Key value
66+
batch: # Processor Type
67+
metadata_keys: [X-SF-Token] # Array of metadata keys to batch
68+
attributes/update: # Processor Type/Name
69+
actions: # List of actions
70+
- key: user.phone_number # Target key
71+
action: update # Replace value with "UNKNOWN NUMBER"
72+
value: "UNKNOWN NUMBER"
73+
- key: user.email # Hash the email value
74+
action: hash
75+
- key: user.account_password # Remove the password
76+
action: delete
77+
redaction/redact: # Processor Type/Name
78+
allow_all_keys: true # If false, only allowed keys will be retained
79+
blocked_values: # List of regex patterns to hash
80+
- '\b4[0-9]{3}[\s-]?[0-9]{4}[\s-]?[0-9]{4}[\s-]?[0-9]{4}\b' # Visa card
81+
- '\b5[1-5][0-9]{2}[\s-]?[0-9]{4}[\s-]?[0-9]{4}[\s-]?[0-9]{4}\b' # MasterCard
82+
summary: debug # Show debug details about redaction
83+
transform/logs: # Processor Type/Name
84+
log_statements: # Log Processing Statements
85+
- context: resource # Log Context
86+
statements: # List of attribute keys to keep
87+
- keep_keys(attributes, ["com.splunk.sourcetype", "host.name", "otelcol.service.mode"])
88+
- context: log # Log Context
89+
statements: # Transform Statements Array
90+
- set(cache, ParseJSON(body)) where IsMatch(body, "^\\{")
91+
- flatten(cache, "")
92+
- merge_maps(attributes, cache, "upsert")
93+
- set(severity_text, attributes["level"])
94+
- set(severity_number, 1) where severity_text == "TRACE"
95+
- set(severity_number, 5) where severity_text == "DEBUG"
96+
- set(severity_number, 9) where severity_text == "INFO"
97+
- set(severity_number, 13) where severity_text == "WARN"
98+
- set(severity_number, 17) where severity_text == "ERROR"
99+
- set(severity_number, 21) where severity_text == "FATAL"
100+
101+
########################### This section controls what
102+
### Activation Section ### configurations will be used
103+
########################### by this OpenTelemetry Collector
104+
service: # Services configured for this Collector
105+
extensions: # Enabled extensions
106+
- health_check
107+
- file_storage/checkpoint # Enabled extensions for this collector
108+
109+
pipelines: # Array of configured pipelines
110+
traces:
111+
receivers:
112+
- otlp
113+
processors:
114+
- memory_limiter # Memory Limiter processor
115+
- attributes/update # Update, hash, and remove attributes
116+
- redaction/redact # Redact sensitive fields using regex
117+
- resourcedetection # Adds system attributes to the data
118+
- resource/add_mode # Adds collector mode metadata
119+
- batch
120+
exporters:
121+
- debug
122+
- otlphttp
123+
metrics:
124+
receivers:
125+
- otlp
126+
# - hostmetrics
127+
processors:
128+
- memory_limiter # Memory Limiter processor
129+
- resourcedetection # Adds system attributes to the data
130+
- resource/add_mode # Adds collector mode metadata
131+
- batch
132+
exporters:
133+
- debug
134+
- otlphttp
135+
logs:
136+
receivers:
137+
- otlp
138+
- filelog/quotes
139+
processors:
140+
- memory_limiter # Memory Limiter processor
141+
- resourcedetection # Adds system attributes to the data
142+
- resource/add_mode # Adds collector mode metadata
143+
- transform/logs
144+
- batch
145+
exporters:
146+
- debug
147+
- otlphttp

‎content/en/ninja-workshops/10-advanced-otel/master-gateway.yaml‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
########################### This section holds all the
2+
## Configuration section ## configurations that can be
3+
########################### used in this OpenTelemetry Collector
4+
extensions: # Array of Extensions
5+
health_check: # Configures the health check extension
6+
endpoint: 0.0.0.0:14133 # Port changed to prevent conflict with agent!!!
7+
8+
receivers:
9+
otlp: # Receiver Type
10+
protocols: # list of Protocols used
11+
http: # This wil enable the HTTP Protocol
12+
endpoint: "0.0.0.0:5318" # Port changed to prevent conflict with agent!!!
13+
include_metadata: true # Needed for token pass through mode
14+
15+
exporters: # Array of Exporters
16+
debug: # Exporter Type
17+
verbosity: detailed # Enabled detailed debug output
18+
file/traces: # Exporter Type/Name
19+
path: "./gateway-traces.out" # Path where data will be saved in OTLP json format
20+
append: false # Overwrite the file each time
21+
file/metrics: # Exporter Type/Name
22+
path: "./gateway-metrics.out" # Path where data will be saved in OTLP json format
23+
append: false # Overwrite the file each time
24+
file/logs: # Exporter Type/Name
25+
path: "./gateway-logs.out" # Path where data will be saved in OTLP json format
26+
append: false # Overwrite the file each time
27+
file/traces/standard: # Exporter for regular traces
28+
path: "./gateway-traces-standard.out" # Path for saving trace data
29+
append: false # Overwrite the file each time
30+
file/traces/security: # Exporter for security traces
31+
path: "./gateway-traces-security.out" # Path for saving trace data
32+
append: false # Overwrite the file each time
33+
34+
connectors:
35+
routing:
36+
default_pipelines: [traces/standard] # Default pipeline if no rule matches
37+
error_mode: ignore # Ignore errors in routing
38+
table: # Define routing rules
39+
# Routes spans to a target pipeline if the resourceSpan attribute matches the rule
40+
- statement: route() where attributes["deployment.environment"] == "security_applications"
41+
pipelines: [traces/security] # Target pipeline
42+
43+
processors: # Array of Processors
44+
memory_limiter: # Limits memory usage by Collectors pipeline
45+
check_interval: 2s # Interval to check memory usage
46+
limit_mib: 512 # Memory limit in MiB
47+
batch: # Processor to Batch data before sending
48+
metadata_keys: # Include token in batches
49+
- X-SF-Token # Batch data grouped by Token
50+
resource/add_mode: # Processor Type/Name
51+
attributes: # Array of Attributes and modifications
52+
- action: upsert # Action taken is to `insert' or 'update' a key
53+
key: otelcol.service.mode # key Name
54+
value: "gateway" # Key Value
55+
filter/health: # Defines a filter processor
56+
error_mode: ignore # Ignore errors
57+
traces: # Filtering rules for traces
58+
span: # Exclude spans named "/_healthz"
59+
- 'name == "/_healthz"'
60+
########################### This section controls what
61+
### Activation Section ### configuration will be used
62+
########################### by the OpenTelemetry Collector
63+
service: # Services configured for this Collector
64+
extensions: [health_check] # Enabled extensions for this collector
65+
pipelines: # Array of configured pipelines
66+
traces:
67+
receivers:
68+
- otlp # OTLP Receiver
69+
processors: []
70+
exporters:
71+
- routing # Debug Exporter
72+
traces/standard: # Default pipeline for unmatched spans
73+
receivers:
74+
- routing # Receive data from the routing connector
75+
processors:
76+
- memory_limiter # Limits memory usage
77+
- resource/add_mode # Adds collector mode metadata
78+
exporters:
79+
- debug # Debug exporter
80+
- file/traces/standard # File exporter for unmatched spans
81+
traces/security: # New Security Traces/Spans Pipeline
82+
receivers:
83+
- routing # Routing Connector, Only receives data from Connector
84+
processors:
85+
- memory_limiter # Memory Limiter Processor
86+
- resource/add_mode # Adds collector mode metadata
87+
exporters:
88+
- debug # Debug Exporter
89+
- file/traces/security # File Exporter for spans matching rule
90+
metrics:
91+
receivers:
92+
- otlp # OTLP Receiver
93+
processors:
94+
- memory_limiter # Memory Limiter processor
95+
- resource/add_mode
96+
- batch
97+
exporters:
98+
- debug # Debug Exporter
99+
- file/metrics
100+
logs:
101+
receivers:
102+
- otlp # OTLP Receiver
103+
processors:
104+
- memory_limiter # Memory Limiter processor
105+
- resource/add_mode
106+
- batch
107+
exporters:
108+
- debug # Debug Exporter
109+
- file/logs

0 commit comments

Comments
 (0)