You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
Copy file name to clipboardExpand all lines: articles/operator-insights/ingestion-agent-configuration-reference.md
+13-16Lines changed: 13 additions & 16 deletions
Display the source diff
Display the rich diff
Original file line number
Diff line number
Diff line change
@@ -8,6 +8,7 @@ ms.service: operator-insights
8
8
ms.topic: conceptual
9
9
ms.date: 12/06/2023
10
10
---
11
+
11
12
# Configuration reference for Azure Operator Insights ingestion agent
12
13
13
14
This reference provides the complete set of configuration for the [Azure Operator Insights ingestion agent](ingestion-agent-overview.md), listing all fields with explanatory comments.
@@ -22,12 +23,12 @@ This reference shows two pipelines: one with an MCC EDR source and one with an S
22
23
23
24
```yaml
24
25
# A unique identifier for this agent instance. Reserved URL characters must be percent-encoded. It's included in the upload path to the Data Product's input storage account.
25
-
agent_id: agent01
26
+
agent_id: agent01
26
27
# Config for secrets providers. We support reading secrets from Azure Key Vault and from the VM's local filesystem.
27
28
# Multiple secret providers can be defined and each must be given a unique name, which is referenced later in the config.
28
29
# A secret provider of type `key_vault` which contains details required to connect to the Azure Key Vault and allow connection to the Data Product's input storage account. This is always required.
29
30
# A secret provider of type `file_system`, which specifies a directory on the VM where secrets are stored. For example for an SFTP pull source, for storing credentials for connecting to an SFTP server.
30
-
secret_providers:
31
+
secret_providers:
31
32
- name: data_product_keyvault_mi
32
33
key_vault:
33
34
vault_name: contoso-dp-kv
@@ -50,16 +51,12 @@ pipelines:
50
51
# Pipeline IDs must be unique for a given agent instance. Any URL reserved characters must be percent-encoded.
51
52
- id: mcc-edrs
52
53
source:
53
-
mcc_edrs:
54
-
<mcc edrs source configuration>
55
-
sink:
56
-
<sink configuration>
54
+
mcc_edrs: <mcc edrs source configuration>
55
+
sink: <sink configuration>
57
56
- id: contoso-logs
58
57
source:
59
-
sftp_pull:
60
-
<sftp pull source configuration>
61
-
sink:
62
-
<sink configuration>
58
+
sftp_pull: <sftp pull source configuration>
59
+
sink: <sink configuration>
63
60
```
64
61
65
62
## Sink configuration
@@ -73,7 +70,7 @@ sink:
73
70
# Optional A string giving an optional base path to use in the container in the Data Product's input storage account. Reserved URL characters must be percent-encoded. See the Data Product for what value, if any, is required.
74
71
base_path: base-path
75
72
sas_token:
76
-
# This must reference a secret provider configured above.
73
+
# This must reference a secret provider configured above.
77
74
secret_provider: data_product_keyvault_mi
78
75
# The name of a secret in the corresponding provider.
79
76
# This will be the name of a secret in the Key Vault.
@@ -102,13 +99,13 @@ source:
102
99
mcc_edrs:
103
100
# The maximum amount of data to buffer in memory before uploading. Units are B, KiB, MiB, GiB, etc.
104
101
message_queue_capacity: 32 MiB
105
-
# Quick check on the maximum RAM that the agent should use.
106
-
# This is a guide to check the other tuning parameters, rather than a hard limit.
102
+
# Quick check on the maximum RAM that the agent should use.
103
+
# This is a guide to check the other tuning parameters, rather than a hard limit.
107
104
maximum_overall_capacity: 1216 MiB
108
105
listener:
109
106
# The TCP port to listen on. Must match the port MCC is configured to send to. Defaults to 36001.
110
107
port: 36001
111
-
# EDRs greater than this size are dropped. Subsequent EDRs continue to be processed.
108
+
# EDRs greater than this size are dropped. Subsequent EDRs continue to be processed.
112
109
# This condition likely indicates MCC sending larger than expected EDRs. MCC is not normally expected
113
110
# to send EDRs larger than the default size. If EDRs are being dropped because of this limit,
114
111
# investigate and confirm that the EDRs are valid, and then increase this value. Units are B, KiB, MiB, GiB, etc.
@@ -118,7 +115,7 @@ source:
118
115
# corrupt EDRs to Azure. You should not need to change this value. Units are B, KiB, MiB, GiB, etc.
119
116
hard_maximum_message_size: 100000 B
120
117
batching:
121
-
# The maximum size of a single blob (file) to store in the Data Product's input storage account.
118
+
# The maximum size of a single blob (file) to store in the Data Product's input storage account.
122
119
maximum_blob_size: 128 MiB. Units are B, KiB, MiB, GiB, etc.
123
120
# The maximum time to wait when no data is received before uploading pending batched data to the Data Product's input storage account. Examples: 30s, 10m, 1h, 1d.
124
121
blob_rollover_period: 5m
@@ -158,7 +155,7 @@ source:
158
155
# Optional. A regular expression to specify which files in the base_path folder should be ingested. If not specified, the agent will attempt to ingest all files in the base_path folder (subject to exclude_pattern, settling_time and exclude_before_time).
159
156
include_pattern: "*\.csv$"
160
157
# Optional. A regular expression to specify any files in the base_path folder which should not be ingested. Takes priority over include_pattern, so files which match both regular expressions will not be ingested.
161
-
exclude_pattern: '\.backup$'
158
+
exclude_pattern: '\.backup$'
162
159
# A duration, such as "10s", "5m", "1h".. During an upload run, any files last modified within the settling time are not selected for upload, as they may still be being modified.
163
160
settling_time: 1m
164
161
# Optional. A datetime that adheres to the RFC 3339 format. Any files last modified before this datetime will be ignored.
0 commit comments