Skip to content

Commit 35f1b6d

Browse files
cbandycbandy
committed
Provide a way to enable resource detectors
Resource detectors are a simple way to gather resource attributes from the environment automatically. They are especially useful in managed or cloud environments. Issue: PGO-2193
1 parent 6dbbf9b commit 35f1b6d

17 files changed

+268
-9
lines changed

‎config/crd/bases/postgres-operator.crunchydata.com_pgadmins.yaml‎

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1605,6 +1605,38 @@ spec:
16051605
description: Config is the place for users to configure exporters
16061606
and provide files.
16071607
properties:
1608+
detectors:
1609+
description: |-
1610+
Resource detectors add identifying attributes to logs and metrics. These run in the order they are defined.
1611+
More info: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/processor/resourcedetectionprocessor#readme
1612+
items:
1613+
properties:
1614+
attributes:
1615+
additionalProperties:
1616+
type: boolean
1617+
description: |-
1618+
Attributes to use from this detector. Detectors usually add every attribute
1619+
they know automatically. Names omitted here behave according detector defaults.
1620+
maxProperties: 30
1621+
minProperties: 1
1622+
type: object
1623+
x-kubernetes-map-type: atomic
1624+
name:
1625+
description: 'Name of the resource detector to enable:
1626+
`aks`, `eks`, `gcp`, etc.'
1627+
maxLength: 20
1628+
minLength: 1
1629+
type: string
1630+
required:
1631+
- name
1632+
type: object
1633+
x-kubernetes-map-type: atomic
1634+
maxItems: 10
1635+
minItems: 1
1636+
type: array
1637+
x-kubernetes-list-map-keys:
1638+
- name
1639+
x-kubernetes-list-type: map
16081640
exporters:
16091641
description: |-
16101642
Exporters allows users to configure OpenTelemetry exporters that exist
@@ -1937,7 +1969,9 @@ spec:
19371969
- path
19381970
type: object
19391971
type: object
1972+
minItems: 1
19401973
type: array
1974+
x-kubernetes-list-type: atomic
19411975
type: object
19421976
image:
19431977
description: |-
@@ -1949,12 +1983,12 @@ spec:
19491983
collection.
19501984
properties:
19511985
exporters:
1952-
description: |-
1953-
Exporters allows users to specify which exporters they want to use in
1954-
the logs pipeline.
1986+
description: The names of exporters that should send logs.
19551987
items:
19561988
type: string
1989+
minItems: 1
19571990
type: array
1991+
x-kubernetes-list-type: set
19581992
retentionPeriod:
19591993
description: |-
19601994
How long to retain log files locally. An RFC 3339 duration or a number

‎config/crd/bases/postgres-operator.crunchydata.com_postgresclusters.yaml‎

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11181,6 +11181,38 @@ spec:
1118111181
description: Config is the place for users to configure exporters
1118211182
and provide files.
1118311183
properties:
11184+
detectors:
11185+
description: |-
11186+
Resource detectors add identifying attributes to logs and metrics. These run in the order they are defined.
11187+
More info: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/processor/resourcedetectionprocessor#readme
11188+
items:
11189+
properties:
11190+
attributes:
11191+
additionalProperties:
11192+
type: boolean
11193+
description: |-
11194+
Attributes to use from this detector. Detectors usually add every attribute
11195+
they know automatically. Names omitted here behave according detector defaults.
11196+
maxProperties: 30
11197+
minProperties: 1
11198+
type: object
11199+
x-kubernetes-map-type: atomic
11200+
name:
11201+
description: 'Name of the resource detector to enable:
11202+
`aks`, `eks`, `gcp`, etc.'
11203+
maxLength: 20
11204+
minLength: 1
11205+
type: string
11206+
required:
11207+
- name
11208+
type: object
11209+
x-kubernetes-map-type: atomic
11210+
maxItems: 10
11211+
minItems: 1
11212+
type: array
11213+
x-kubernetes-list-map-keys:
11214+
- name
11215+
x-kubernetes-list-type: map
1118411216
exporters:
1118511217
description: |-
1118611218
Exporters allows users to configure OpenTelemetry exporters that exist
@@ -11513,7 +11545,9 @@ spec:
1151311545
- path
1151411546
type: object
1151511547
type: object
11548+
minItems: 1
1151611549
type: array
11550+
x-kubernetes-list-type: atomic
1151711551
type: object
1151811552
image:
1151911553
description: |-
@@ -11525,12 +11559,12 @@ spec:
1152511559
collection.
1152611560
properties:
1152711561
exporters:
11528-
description: |-
11529-
Exporters allows users to specify which exporters they want to use in
11530-
the logs pipeline.
11562+
description: The names of exporters that should send logs.
1153111563
items:
1153211564
type: string
11565+
minItems: 1
1153311566
type: array
11567+
x-kubernetes-list-type: set
1153411568
retentionPeriod:
1153511569
description: |-
1153611570
How long to retain log files locally. An RFC 3339 duration or a number

‎internal/collector/config.go‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,35 @@ func NewConfig(spec *v1beta1.InstrumentationSpec) *Config {
114114
Pipelines: map[PipelineID]Pipeline{},
115115
}
116116

117+
// Create a resource detection processor according to the API spec.
118+
// When nothing is specified, the processor does nothing.
119+
{
120+
// https://pkg.go.dev/github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor#section-readme
121+
processor := map[string]any{"override": false, "timeout": "30s"}
122+
123+
if spec != nil && spec.Config != nil {
124+
names := make([]string, len(spec.Config.Detectors))
125+
for i, detector := range spec.Config.Detectors {
126+
names[i] = detector.Name
127+
128+
if len(detector.Attributes) > 0 {
129+
attributes := make(map[string]any, len(detector.Attributes))
130+
for k, v := range detector.Attributes {
131+
attributes[k] = map[string]any{"enabled": v}
132+
}
133+
processor[detector.Name] = map[string]any{
134+
"resource_attributes": attributes,
135+
}
136+
}
137+
}
138+
processor["detectors"] = names
139+
} else {
140+
processor["detectors"] = []string{}
141+
}
142+
143+
config.Processors[ResourceDetectionProcessor] = processor
144+
}
145+
117146
// If there are exporters defined in the spec, add them to the config.
118147
if spec != nil && spec.Config != nil && spec.Config.Exporters != nil {
119148
for k, v := range spec.Config.Exporters {

‎internal/collector/config_test.go‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ import (
99

1010
"gotest.tools/v3/assert"
1111

12+
"github.com/crunchydata/postgres-operator/internal/testing/cmp"
13+
"github.com/crunchydata/postgres-operator/internal/testing/require"
1214
"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
1315
)
1416

@@ -28,6 +30,10 @@ processors:
2830
batch/200ms:
2931
timeout: 200ms
3032
groupbyattrs/compact: {}
33+
resourcedetection:
34+
detectors: []
35+
override: false
36+
timeout: 30s
3137
receivers: {}
3238
service:
3339
extensions: []
@@ -56,12 +62,43 @@ processors:
5662
batch/200ms:
5763
timeout: 200ms
5864
groupbyattrs/compact: {}
65+
resourcedetection:
66+
detectors: []
67+
override: false
68+
timeout: 30s
5969
receivers: {}
6070
service:
6171
extensions: []
6272
pipelines: {}
6373
`)
6474
})
75+
76+
t.Run("Detectors", func(t *testing.T) {
77+
var spec *v1beta1.InstrumentationSpec
78+
require.UnmarshalInto(t, &spec, `{
79+
config: {
80+
detectors: [
81+
{ name: gcp },
82+
{ name: aks, attributes: { k8s.cluster.name: true } },
83+
],
84+
},
85+
}`)
86+
87+
result, err := NewConfig(spec).ToYAML()
88+
assert.NilError(t, err)
89+
assert.Assert(t, cmp.Contains(result, `
90+
resourcedetection:
91+
aks:
92+
resource_attributes:
93+
k8s.cluster.name:
94+
enabled: true
95+
detectors:
96+
- gcp
97+
- aks
98+
override: false
99+
timeout: 30s
100+
`))
101+
})
65102
}
66103

67104
func TestGenerateLogrotateConfig(t *testing.T) {

‎internal/collector/naming.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ const Prometheus = "prometheus"
1212
const PGBouncerMetrics = "metrics/pgbouncer"
1313
const PostgresMetrics = "metrics/postgres"
1414
const PatroniMetrics = "metrics/patroni"
15+
const ResourceDetectionProcessor = "resourcedetection"
1516

1617
const SqlQuery = "sqlquery"
1718

‎internal/collector/patroni.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ func EnablePatroniLogging(ctx context.Context,
118118
Processors: []ComponentID{
119119
"resource/patroni",
120120
"transform/patroni_logs",
121+
ResourceDetectionProcessor,
121122
SubSecondBatchProcessor,
122123
CompactingProcessor,
123124
},

‎internal/collector/patroni_test.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,10 @@ processors:
5555
- action: insert
5656
key: k8s.pod.name
5757
value: ${env:K8S_POD_NAME}
58+
resourcedetection:
59+
detectors: []
60+
override: false
61+
timeout: 30s
5862
transform/patroni_logs:
5963
log_statements:
6064
- context: log
@@ -89,6 +93,7 @@ service:
8993
processors:
9094
- resource/patroni
9195
- transform/patroni_logs
96+
- resourcedetection
9297
- batch/200ms
9398
- groupbyattrs/compact
9499
receivers:
@@ -142,6 +147,10 @@ processors:
142147
- action: insert
143148
key: k8s.pod.name
144149
value: ${env:K8S_POD_NAME}
150+
resourcedetection:
151+
detectors: []
152+
override: false
153+
timeout: 30s
145154
transform/patroni_logs:
146155
log_statements:
147156
- context: log
@@ -176,6 +185,7 @@ service:
176185
processors:
177186
- resource/patroni
178187
- transform/patroni_logs
188+
- resourcedetection
179189
- batch/200ms
180190
- groupbyattrs/compact
181191
receivers:

‎internal/collector/pgadmin.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ func EnablePgAdminLogging(ctx context.Context, spec *v1beta1.InstrumentationSpec
102102
Processors: []ComponentID{
103103
"resource/pgadmin",
104104
"transform/pgadmin_log",
105+
ResourceDetectionProcessor,
105106
SubSecondBatchProcessor,
106107
CompactingProcessor,
107108
},
@@ -114,6 +115,7 @@ func EnablePgAdminLogging(ctx context.Context, spec *v1beta1.InstrumentationSpec
114115
Processors: []ComponentID{
115116
"resource/pgadmin",
116117
"transform/pgadmin_log",
118+
ResourceDetectionProcessor,
117119
SubSecondBatchProcessor,
118120
CompactingProcessor,
119121
},

‎internal/collector/pgadmin_test.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,10 @@ collector.yaml: |
6363
- action: insert
6464
key: k8s.pod.name
6565
value: ${env:K8S_POD_NAME}
66+
resourcedetection:
67+
detectors: []
68+
override: false
69+
timeout: 30s
6670
transform/pgadmin_log:
6771
log_statements:
6872
- context: log
@@ -99,6 +103,7 @@ collector.yaml: |
99103
processors:
100104
- resource/pgadmin
101105
- transform/pgadmin_log
106+
- resourcedetection
102107
- batch/200ms
103108
- groupbyattrs/compact
104109
receivers:
@@ -109,6 +114,7 @@ collector.yaml: |
109114
processors:
110115
- resource/pgadmin
111116
- transform/pgadmin_log
117+
- resourcedetection
112118
- batch/200ms
113119
- groupbyattrs/compact
114120
receivers:
@@ -175,6 +181,10 @@ collector.yaml: |
175181
- action: insert
176182
key: k8s.pod.name
177183
value: ${env:K8S_POD_NAME}
184+
resourcedetection:
185+
detectors: []
186+
override: false
187+
timeout: 30s
178188
transform/pgadmin_log:
179189
log_statements:
180190
- context: log
@@ -211,6 +221,7 @@ collector.yaml: |
211221
processors:
212222
- resource/pgadmin
213223
- transform/pgadmin_log
224+
- resourcedetection
214225
- batch/200ms
215226
- groupbyattrs/compact
216227
receivers:
@@ -221,6 +232,7 @@ collector.yaml: |
221232
processors:
222233
- resource/pgadmin
223234
- transform/pgadmin_log
235+
- resourcedetection
224236
- batch/200ms
225237
- groupbyattrs/compact
226238
receivers:

‎internal/collector/pgbackrest.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ func NewConfigForPgBackrestRepoHostPod(
107107
Processors: []ComponentID{
108108
"resource/pgbackrest",
109109
"transform/pgbackrest_logs",
110+
ResourceDetectionProcessor,
110111
SubSecondBatchProcessor,
111112
CompactingProcessor,
112113
},

0 commit comments

Comments
 (0)