Skip to content

vmagent: support per remoteWrite queues configuration #2139

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AndrewChubatiuk wants to merge 1 commit into
base: master
Choose a base branch
from
Open

vmagent: support per remoteWrite queues configuration #2139

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions api/operator/v1beta1/vmagent_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -381,6 +381,9 @@ type VMAgentRemoteWriteSpec struct {
ProxyURL *string `json:"proxyURL,omitempty"`
// AWS describes params specific to AWS cloud
AWS *AWS `json:"aws,omitempty"`
// The number of concurrent queues
// +optional
Queues *int32 `json:"queues,omitempty"`
}

// AsConfigMapKey key for kubernetes configmap
Expand Down
5 changes: 5 additions & 0 deletions api/operator/v1beta1/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions config/crd/overlay/crd.descriptionless.yaml
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions config/crd/overlay/crd.yaml
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion docs/CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ aliases:
## tip

* Dependency: [vmoperator](https://docs.victoriametrics.com/operator/): Updated default versions for VM apps to [v1.142.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.142.0) version
* FEATURE: [vmauth](https://docs.victoriametrics.com/operator/resources/vmauth): previously VMAuth could read configuration only from predefined locations; now VMAuth supports arbitrary filesystem access configuration, allowing users to reference required files directly and reducing configuration workarounds. See [#899](https://github.com/VictoriaMetrics/operator/issues/899).
* FEATURE: [vmauth](https://docs.victoriametrics.com/operator/resources/vmauth/): previously VMAuth could read configuration only from predefined locations; now VMAuth supports arbitrary filesystem access configuration, allowing users to reference required files directly and reducing configuration workarounds. See [#899](https://github.com/VictoriaMetrics/operator/issues/899).
* FEATURE: [vmagent](https://docs.victoriametrics.com/operator/resources/vmagent/): support per remote write queues configuration. See [#2138](https://github.com/VictoriaMetrics/operator/issues/2138).

* BUGFIX: [converter](https://docs.victoriametrics.com/operator/integrations/prometheus/#objects-conversion): disable all prometheus controllers if CRD group was not found. See [#2838](https://github.com/VictoriaMetrics/helm-charts/issues/2838).
* BUGFIX: [vmdistributed](https://docs.victoriametrics.com/operator/resources/vmdistributed/): change default load balancing policy for write requests from `first_available` to `least_loaded`. This should allow to evenly distribute write load across all VMAgents.
Expand Down
1 change: 1 addition & 0 deletions docs/api.md
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

28 changes: 24 additions & 4 deletions internal/controller/operator/factory/vmagent/vmagent.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -915,11 +915,18 @@ func sortMap(m map[string]string) []item {

func buildRemoteWriteArgs(cr *vmv1beta1.VMAgent, ac *build.AssetsCache) ([]string, error) {
maxDiskUsage := defaultMaxDiskUsage
if cr.Spec.RemoteWriteSettings != nil && cr.Spec.RemoteWriteSettings.MaxDiskUsagePerURL != nil {
maxDiskUsage = cr.Spec.RemoteWriteSettings.MaxDiskUsagePerURL.String()
queues := "0"
if cr.Spec.RemoteWriteSettings != nil {
if cr.Spec.RemoteWriteSettings.MaxDiskUsagePerURL != nil {
maxDiskUsage = cr.Spec.RemoteWriteSettings.MaxDiskUsagePerURL.String()
}
if cr.Spec.RemoteWriteSettings.Queues != nil {
queues = strconv.Itoa(int(*cr.Spec.RemoteWriteSettings.Queues))
}
}

var args []string
var hasAnyQueuesSet bool
var hasAnyDiskUsagesSet bool
var storageLimit int64

Expand Down Expand Up @@ -969,6 +976,7 @@ func buildRemoteWriteArgs(cr *vmv1beta1.VMAgent, ac *build.AssetsCache) ([]strin
var relabelKeys []string

var maxDiskUsagesPerRW []string
var queuesPerRW []string

if storageLimit > 0 && maxDiskUsage == defaultMaxDiskUsage {
// conditionally change default value of maxDiskUsage
Expand Down Expand Up @@ -1017,6 +1025,12 @@ func buildRemoteWriteArgs(cr *vmv1beta1.VMAgent, ac *build.AssetsCache) ([]strin
}
bearerTokenFile.Add(strconv.Quote(value), i)
}
if rw.Queues != nil {
queuesPerRW = append(queuesPerRW, strconv.Itoa(int(*rw.Queues)))
hasAnyQueuesSet = true
Copy link
Copy Markdown
Contributor

@cubic-dev-ai cubic-dev-ai Bot May 6, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1: hasAnyQueuesSet is set too broadly, which can drop -remoteWrite.queues entirely when explicit per-URL values equal the global queue value.

Prompt for AI agents 
Check if this issue is valid — if so, understand the root cause and fix it. At internal/controller/operator/factory/vmagent/vmagent.go, line 1030:

<comment>`hasAnyQueuesSet` is set too broadly, which can drop `-remoteWrite.queues` entirely when explicit per-URL values equal the global queue value.</comment>

<file context>
@@ -1017,6 +1025,12 @@ func buildRemoteWriteArgs(cr *vmv1beta1.VMAgent, ac *build.AssetsCache) ([]strin
 			}
+			if rw.Queues != nil {
+				queuesPerRW = append(queuesPerRW, strconv.Itoa(int(*rw.Queues)))
+				hasAnyQueuesSet = true
+			} else {
+				queuesPerRW = append(queuesPerRW, queues)
</file context>
Fix with Cubic

Copy link
Copy Markdown
Contributor Author

@AndrewChubatiuk AndrewChubatiuk May 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

there's no need in per url values in this case

Copy link
Copy Markdown
Contributor

@cubic-dev-ai cubic-dev-ai Bot May 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Got it, thanks for clarifying.

} else {
queuesPerRW = append(queuesPerRW, queues)
}
var relabelConfig *vmv1beta1.CommonRelabelParams
if rw.UrlRelabelConfig != nil || len(rw.InlineUrlRelabelConfig) > 0 {
relabelConfig = &vmv1beta1.CommonRelabelParams{
Expand Down Expand Up @@ -1089,9 +1103,15 @@ func buildRemoteWriteArgs(cr *vmv1beta1.VMAgent, ac *build.AssetsCache) ([]strin
maxDiskUsagePerURL.Add(usage, i)
}
}
rwQueues := build.NewFlag("-remoteWrite.queues", queues)
if hasAnyQueuesSet {
for i, q := range queuesPerRW {
rwQueues.Add(q, i)
}
}

totalCount := len(remoteTargets)
args = build.AppendFlagsToArgs(args, totalCount, url, authUser, bearerTokenFile)
args = build.AppendFlagsToArgs(args, totalCount, url, authUser, bearerTokenFile, rwQueues)
args = build.RelabelArgsTo(args, "remoteWrite.urlRelabelConfig", relabelKeys, relabelConfigs...)
args = build.AppendFlagsToArgs(args, totalCount, tlsInsecure, sendTimeout, proxyURL)
args = build.AppendFlagsToArgs(args, totalCount, tlsServerName, tlsKeys, tlsCerts, tlsCAs)
Expand All @@ -1109,7 +1129,7 @@ func buildRemoteWriteArgs(cr *vmv1beta1.VMAgent, ac *build.AssetsCache) ([]strin
if rws.FlushInterval != nil {
args = append(args, fmt.Sprintf("-remoteWrite.flushInterval=%s", *rws.FlushInterval))
}
if rws.Queues != nil {
if rws.Queues != nil && !hasAnyQueuesSet {
args = append(args, fmt.Sprintf("-remoteWrite.queues=%d", *rws.Queues))
}
if rws.ShowURL != nil {
Expand Down
30 changes: 30 additions & 0 deletions internal/controller/operator/factory/vmagent/vmagent_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -892,6 +892,36 @@ func TestBuildRemoteWriteArgs(t *testing.T) {
},
})

// queues
f(opts{
cr: &vmv1beta1.VMAgent{
ObjectMeta: metav1.ObjectMeta{
Name: "default-vmagent",
Namespace: "default",
},
Spec: vmv1beta1.VMAgentSpec{
RemoteWrite: []vmv1beta1.VMAgentRemoteWriteSpec{
{
URL: "localhost:8429",
Queues: ptr.To[int32](5),
},
{
URL: "localhost:8430",
},
},
RemoteWriteSettings: &vmv1beta1.VMAgentRemoteWriteSettings{
Queues: ptr.To[int32](10),
},
},
},
want: []string{
`-remoteWrite.maxDiskUsagePerURL=1073741824`,
`-remoteWrite.url=localhost:8429,localhost:8430`,
`-remoteWrite.queues=5,10`,
`-remoteWrite.tmpDataPath=/tmp/vmagent-remotewrite-data`,
},
})

// test insecure with key only
f(opts{
cr: &vmv1beta1.VMAgent{
Expand Down
Loading