Skip to content

Commit 25bc1d7

Browse files
vparfonovvparfonov
committed
Add support for configuring the maximum message length (in bytes) after merging partial log lines in the kubernetes_logs source
1 parent f0d00a4 commit 25bc1d7

23 files changed

+720
-57
lines changed

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,7 @@ test-functional-benchmarker-vector: bin/functional-benchmarker
232232
@out=$$(RELATED_IMAGE_VECTOR=$(IMAGE_LOGGING_VECTOR) bin/functional-benchmarker --image=$(IMAGE_LOGGING_VECTOR) --artifact-dir=/tmp/benchmark-test-vector 2>&1); if [ "$$?" != "0" ] ; then echo "$$out"; exit 1; fi
233233

234234
.PHONY: test-unit
235-
test-unit: test-forwarder-generator test-unit-api
235+
test-unit: test-unit-api
236236
RELATED_IMAGE_VECTOR=$(IMAGE_LOGGING_VECTOR) \
237237
RELATED_IMAGE_LOG_FILE_METRIC_EXPORTER=$(IMAGE_LOGFILEMETRICEXPORTER) \
238238
go test -coverprofile=test.cov -race ./api/... ./internal/... `go list ./test/... | grep -Ev 'test/(e2e|functional|framework|client|helpers)'`

‎api/observability/v1/input_types.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ limitations under the License.
1515
package v1
1616

1717
import (
18+
"k8s.io/apimachinery/pkg/api/resource"
1819
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1920
)
2021

@@ -96,6 +97,12 @@ type ContainerInputTuningSpec struct {
9697
// +kubebuilder:validation:Optional
9798
// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Per-Container Rate Limit"
9899
RateLimitPerContainer *LimitSpec `json:"rateLimitPerContainer,omitempty"`
100+
101+
// MaxMessageSize The maximum message length in bytes that a single log event can be when all
102+
// partial log lines are merged. Messages exceeding this limit are dropped.
103+
// +kubebuilder:validation:Optional
104+
// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Max Message Size"
105+
MaxMessageSize *resource.Quantity `json:"maxMessageSize,omitempty"`
99106
}
100107

101108
// ApplicationSource defines the type of ApplicationSource log source to use.
@@ -184,6 +191,16 @@ var (
184191
}
185192
)
186193

194+
// InfrastructureInputTuningSpec is the infrastructure input tuning spec, for now available only for container sources
195+
type InfrastructureInputTuningSpec struct {
196+
197+
// Container is the input tuning spec for container sources
198+
//
199+
// +kubebuilder:validation:Optional
200+
// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Input Tuning"
201+
Container *ContainerInputTuningSpec `json:"container,omitempty"`
202+
}
203+
187204
// Infrastructure enables infrastructure logs.
188205
// Sources of these logs:
189206
// * container workloads deployed to namespaces: default, kube*, openshift*
@@ -195,6 +212,12 @@ type Infrastructure struct {
195212
// +kubebuilder:validation:Optional
196213
// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Log Sources"
197214
Sources []InfrastructureSource `json:"sources,omitempty"`
215+
216+
// Tuning is the infrastructure input tuning spec, for now available only for container sources
217+
//
218+
// +kubebuilder:validation:Optional
219+
// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Container Input Tuning"
220+
Tuning *InfrastructureInputTuningSpec `json:"tuning,omitempty"`
198221
}
199222

200223
// AuditSource defines which type of audit log source is used.

‎api/observability/v1/zz_generated.deepcopy.go‎

Lines changed: 30 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎bundle/manifests/cluster-logging.clusterserviceversion.yaml‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ metadata:
8282
categories: OpenShift Optional, Logging & Tracing
8383
certified: "false"
8484
containerImage: quay.io/openshift-logging/cluster-logging-operator:latest
85-
createdAt: "2025-07-29T15:25:15Z"
85+
createdAt: "2025-12-12T17:14:22Z"
8686
description: The Red Hat OpenShift Logging Operator for OCP provides a means for
8787
configuring and managing log collection and forwarding.
8888
features.operators.openshift.io/cnf: "false"
@@ -357,6 +357,11 @@ spec:
357357
sources
358358
displayName: Input Tuning
359359
path: inputs[0].application.tuning
360+
- description: |-
361+
MaxMessageSize The maximum message length in bytes that a single log event can be when all
362+
partial log lines are merged. Messages exceeding this limit are dropped.
363+
displayName: Max Message Size
364+
path: inputs[0].application.tuning.maxMessageSize
360365
- description: |-
361366
RateLimitPerContainer is the limit applied to each container
362367
by this input. This limit is applied per collector deployment.
@@ -385,6 +390,30 @@ spec:
385390
This field is optional and omission results in the collection of all infrastructure sources.
386391
displayName: Log Sources
387392
path: inputs[0].infrastructure.sources
393+
- description: Tuning is the infrastructure input tuning spec, for now available
394+
only for container sources
395+
displayName: Container Input Tuning
396+
path: inputs[0].infrastructure.tuning
397+
- description: Container is the input tuning spec for container sources
398+
displayName: Input Tuning
399+
path: inputs[0].infrastructure.tuning.container
400+
- description: |-
401+
MaxMessageSize The maximum message length in bytes that a single log event can be when all
402+
partial log lines are merged. Messages exceeding this limit are dropped.
403+
displayName: Max Message Size
404+
path: inputs[0].infrastructure.tuning.container.maxMessageSize
405+
- description: |-
406+
RateLimitPerContainer is the limit applied to each container
407+
by this input. This limit is applied per collector deployment.
408+
displayName: Per-Container Rate Limit
409+
path: inputs[0].infrastructure.tuning.container.rateLimitPerContainer
410+
- description: |-
411+
MaxRecordsPerSecond is the maximum number of log records
412+
allowed per input/output in a pipeline
413+
displayName: Max Records Per Second
414+
path: inputs[0].infrastructure.tuning.container.rateLimitPerContainer.maxRecordsPerSecond
415+
x-descriptors:
416+
- urn:alm:descriptor:com.tectonic.ui:number
388417
- description: Name used to refer to the input of a `pipeline`.
389418
displayName: Input Name
390419
path: inputs[0].name

‎bundle/manifests/observability.openshift.io_clusterlogforwarders.yaml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -629,6 +629,15 @@ spec:
629629
description: Tuning is the container input tuning spec for
630630
this container sources
631631
properties:
632+
maxMessageSize:
633+
anyOf:
634+
- type: integer
635+
- type: string
636+
description: |-
637+
MaxMessageSize The maximum message length in bytes that a single log event can be when all
638+
partial log lines are merged. Messages exceeding this limit are dropped.
639+
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
640+
x-kubernetes-int-or-string: true
632641
rateLimitPerContainer:
633642
description: |-
634643
RateLimitPerContainer is the limit applied to each container
@@ -680,6 +689,41 @@ spec:
680689
- node
681690
type: string
682691
type: array
692+
tuning:
693+
description: Tuning is the infrastructure input tuning spec,
694+
for now available only for container sources
695+
properties:
696+
container:
697+
description: Container is the input tuning spec for
698+
container sources
699+
properties:
700+
maxMessageSize:
701+
anyOf:
702+
- type: integer
703+
- type: string
704+
description: |-
705+
MaxMessageSize The maximum message length in bytes that a single log event can be when all
706+
partial log lines are merged. Messages exceeding this limit are dropped.
707+
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
708+
x-kubernetes-int-or-string: true
709+
rateLimitPerContainer:
710+
description: |-
711+
RateLimitPerContainer is the limit applied to each container
712+
by this input. This limit is applied per collector deployment.
713+
properties:
714+
maxRecordsPerSecond:
715+
description: |-
716+
MaxRecordsPerSecond is the maximum number of log records
717+
allowed per input/output in a pipeline
718+
exclusiveMinimum: true
719+
format: int64
720+
minimum: 0
721+
type: integer
722+
required:
723+
- maxRecordsPerSecond
724+
type: object
725+
type: object
726+
type: object
683727
type: object
684728
name:
685729
description: Name used to refer to the input of a `pipeline`.

‎config/crd/bases/observability.openshift.io_clusterlogforwarders.yaml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -629,6 +629,15 @@ spec:
629629
description: Tuning is the container input tuning spec for
630630
this container sources
631631
properties:
632+
maxMessageSize:
633+
anyOf:
634+
- type: integer
635+
- type: string
636+
description: |-
637+
MaxMessageSize The maximum message length in bytes that a single log event can be when all
638+
partial log lines are merged. Messages exceeding this limit are dropped.
639+
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
640+
x-kubernetes-int-or-string: true
632641
rateLimitPerContainer:
633642
description: |-
634643
RateLimitPerContainer is the limit applied to each container
@@ -680,6 +689,41 @@ spec:
680689
- node
681690
type: string
682691
type: array
692+
tuning:
693+
description: Tuning is the infrastructure input tuning spec,
694+
for now available only for container sources
695+
properties:
696+
container:
697+
description: Container is the input tuning spec for
698+
container sources
699+
properties:
700+
maxMessageSize:
701+
anyOf:
702+
- type: integer
703+
- type: string
704+
description: |-
705+
MaxMessageSize The maximum message length in bytes that a single log event can be when all
706+
partial log lines are merged. Messages exceeding this limit are dropped.
707+
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
708+
x-kubernetes-int-or-string: true
709+
rateLimitPerContainer:
710+
description: |-
711+
RateLimitPerContainer is the limit applied to each container
712+
by this input. This limit is applied per collector deployment.
713+
properties:
714+
maxRecordsPerSecond:
715+
description: |-
716+
MaxRecordsPerSecond is the maximum number of log records
717+
allowed per input/output in a pipeline
718+
exclusiveMinimum: true
719+
format: int64
720+
minimum: 0
721+
type: integer
722+
required:
723+
- maxRecordsPerSecond
724+
type: object
725+
type: object
726+
type: object
683727
type: object
684728
name:
685729
description: Name used to refer to the input of a `pipeline`.

‎config/manifests/bases/cluster-logging.clusterserviceversion.yaml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -280,6 +280,11 @@ spec:
280280
sources
281281
displayName: Input Tuning
282282
path: inputs[0].application.tuning
283+
- description: |-
284+
MaxMessageSize The maximum message length in bytes that a single log event can be when all
285+
partial log lines are merged. Messages exceeding this limit are dropped.
286+
displayName: Max Message Size
287+
path: inputs[0].application.tuning.maxMessageSize
283288
- description: |-
284289
RateLimitPerContainer is the limit applied to each container
285290
by this input. This limit is applied per collector deployment.
@@ -308,6 +313,30 @@ spec:
308313
This field is optional and omission results in the collection of all infrastructure sources.
309314
displayName: Log Sources
310315
path: inputs[0].infrastructure.sources
316+
- description: Tuning is the infrastructure input tuning spec, for now available
317+
only for container sources
318+
displayName: Container Input Tuning
319+
path: inputs[0].infrastructure.tuning
320+
- description: Container is the input tuning spec for container sources
321+
displayName: Input Tuning
322+
path: inputs[0].infrastructure.tuning.container
323+
- description: |-
324+
MaxMessageSize The maximum message length in bytes that a single log event can be when all
325+
partial log lines are merged. Messages exceeding this limit are dropped.
326+
displayName: Max Message Size
327+
path: inputs[0].infrastructure.tuning.container.maxMessageSize
328+
- description: |-
329+
RateLimitPerContainer is the limit applied to each container
330+
by this input. This limit is applied per collector deployment.
331+
displayName: Per-Container Rate Limit
332+
path: inputs[0].infrastructure.tuning.container.rateLimitPerContainer
333+
- description: |-
334+
MaxRecordsPerSecond is the maximum number of log records
335+
allowed per input/output in a pipeline
336+
displayName: Max Records Per Second
337+
path: inputs[0].infrastructure.tuning.container.rateLimitPerContainer.maxRecordsPerSecond
338+
x-descriptors:
339+
- urn:alm:descriptor:com.tectonic.ui:number
311340
- description: Name used to refer to the input of a `pipeline`.
312341
displayName: Input Name
313342
path: inputs[0].name

0 commit comments

Comments
 (0)