464 logs infra (#478)

## Motivation and Context
Logs will help in fixing issues quickly, thereby reducing the downtime
and MTTD.

## How Has This Been Tested?
End to end local setup is tested

## Breaking Changes
No

## Types of changes
<!-- What types of changes does your code introduce? Put an `x` in all
the boxes that apply: -->
- [ ] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
- [ ] Documentation update

## Checklist
<!-- Go over all the following points, and put an `x` in all the boxes
that apply. -->
- [ ] I have read the [MCP
Documentation](https://modelcontextprotocol.io)
- [x] My code follows the repository's style guidelines
- [x] New and existing tests pass locally
- [ ] I have added appropriate error handling
- [ ] I have added or updated documentation as needed

## Additional context
- It handles container logs for mcp-registry pods.
- It supports k8sattributes resources which helps in identifying from
where the logs are coming.
- Supports victorialogs and it's plugin in grafana.

<img width="1129" height="383" alt="Screenshot 2025-09-15 at 1 43 51 AM"
src="https://github.com/user-attachments/assets/a560c541-721d-415c-a0a8-8cbe008c3af6"
/>

<img width="1134" height="703" alt="Screenshot 2025-09-15 at 1 44 34 AM"
src="https://github.com/user-attachments/assets/8b5b4132-9fc3-4b8e-8027-27c7787070f1"
/>

<img width="1134" height="727" alt="Screenshot 2025-09-15 at 1 46 10 AM"
src="https://github.com/user-attachments/assets/bcdb4d9d-cea1-4d88-8dd5-fa4cdb51031f"
/>

---------

Co-authored-by: Radoslav Dimitrov <[email protected]>

Author: pree-dew

deploy/pkg/k8s/monitoring.go

@@ -105,10 +105,281 @@ func DeployMonitoringStack(ctx *pulumi.Context, cluster *providers.ProviderInfo,
 		return err
 	}
 
+	// Deploy VictoriaLogs for log storage
+	err = deployVictoriaLogs(ctx, cluster, ns, environment)
+	if err != nil {
+		return err
+	}
+
+	// Deploy OpenTelemetry Collector DaemonSet
+	err = deployOtelCollectorDaemonSet(ctx, cluster, ns, environment)
+	if err != nil {
+		return err
+	}
+
 	// Deploy Grafana
 	return deployGrafana(ctx, cluster, ns, environment, ingressNginx)
 }
 
+// deployVictoriaLogs deploys VictoriaLogs for log storage
+func deployVictoriaLogs(ctx *pulumi.Context, cluster *providers.ProviderInfo, ns *corev1.Namespace, environment string) error {
+	// Deploy VictoriaLogs using Helm chart
+	_, err := helm.NewChart(ctx, "victoria-logs", helm.ChartArgs{
+		Chart:     pulumi.String("victoria-logs-single"),
+		Version:   pulumi.String("0.11.8"),
+		Namespace: ns.Metadata.Name().Elem(),
+		FetchArgs: helm.FetchArgs{
+			Repo: pulumi.String("https://victoriametrics.github.io/helm-charts/"),
+		},
+		Values: pulumi.Map{
+			"server": pulumi.Map{
+				"retentionPeriod": pulumi.String("15d"),
+				"resources": pulumi.Map{
+					"requests": pulumi.Map{
+						"memory": pulumi.String("256Mi"),
+						"cpu":    pulumi.String("100m"),
+					},
+					"limits": pulumi.Map{
+						"memory": pulumi.String("2Gi"),
+						"cpu":    pulumi.String("1000m"),
+					},
+				},
+				"persistence": pulumi.Map{
+					"enabled": pulumi.Bool(true),
+					"size":    pulumi.String("20Gi"),
+				},
+			},
+		},
+	}, pulumi.Provider(cluster.Provider))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// deployOtelCollectorDaemonSet deploys OpenTelemetry Collector using Helm chart
+func deployOtelCollectorDaemonSet(ctx *pulumi.Context, cluster *providers.ProviderInfo, ns *corev1.Namespace, environment string) error {
+	// Deploy OpenTelemetry Collector using Helm chart
+	_, err := helm.NewChart(ctx, "opentelemetry-collector", helm.ChartArgs{
+		Chart:     pulumi.String("opentelemetry-collector"),
+		Version:   pulumi.String("0.133.0"),
+		Namespace: ns.Metadata.Name().Elem(),
+		FetchArgs: helm.FetchArgs{
+			Repo: pulumi.String("https://open-telemetry.github.io/opentelemetry-helm-charts"),
+		},
+		Values: pulumi.Map{
+			"mode": pulumi.String("daemonset"),
+			"image": pulumi.Map{
+				"repository": pulumi.String("otel/opentelemetry-collector-contrib"),
+				"tag":        pulumi.String("0.133.0"),
+			},
+			"clusterRole": pulumi.Map{
+				"create": pulumi.Bool(true),
+				"rules": pulumi.Array{
+					pulumi.Map{
+						"apiGroups": pulumi.StringArray{pulumi.String("")},
+						"resources": pulumi.StringArray{
+							pulumi.String("pods"),
+							pulumi.String("pods/log"),
+							pulumi.String("nodes"),
+							pulumi.String("namespaces"),
+						},
+						"verbs": pulumi.StringArray{
+							pulumi.String("get"),
+							pulumi.String("list"),
+							pulumi.String("watch"),
+						},
+					},
+				},
+			},
+			"config": pulumi.Map{
+				"receivers": pulumi.Map{
+					"filelog": pulumi.Map{
+						"include":           pulumi.StringArray{pulumi.String("/var/log/pods/default_mcp-registry*/*/*.log")},
+						"exclude":           pulumi.StringArray{pulumi.String("/var/log/pods/*/*-collector-*/*.log")},
+						"start_at":          pulumi.String("end"),
+						"include_file_path": pulumi.Bool(true),
+						"include_file_name": pulumi.Bool(false),
+						"operators": pulumi.Array{
+							pulumi.Map{
+								"type":       pulumi.String("regex_parser"),
+								"id":         pulumi.String("extract_metadata_from_filepath"),
+								"regex":      pulumi.String(`^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]{36})\/(?P<container_name>[^\._]+)\/(?P<restart_count>\d+)\.log`),
+								"parse_from": pulumi.String("attributes[\"log.file.path\"]"),
+								"cache": pulumi.Map{
+									"size": pulumi.Int(128),
+								},
+							},
+							pulumi.Map{
+								"type": pulumi.String("move"),
+								"from": pulumi.String("attributes.container_name"),
+								"to":   pulumi.String("resource[\"k8s.container.name\"]"),
+							},
+							pulumi.Map{
+								"type": pulumi.String("move"),
+								"from": pulumi.String("attributes.namespace"),
+								"to":   pulumi.String("resource[\"k8s.namespace.name\"]"),
+							},
+							pulumi.Map{
+								"type": pulumi.String("move"),
+								"from": pulumi.String("attributes.pod_name"),
+								"to":   pulumi.String("resource[\"k8s.pod.name\"]"),
+							},
+							pulumi.Map{
+								"type": pulumi.String("move"),
+								"from": pulumi.String("attributes.restart_count"),
+								"to":   pulumi.String("resource[\"k8s.container.restart_count\"]"),
+							},
+							pulumi.Map{
+								"type": pulumi.String("move"),
+								"from": pulumi.String("attributes.uid"),
+								"to":   pulumi.String("resource[\"k8s.pod.uid\"]"),
+							},
+						},
+					},
+				},
+				"processors": pulumi.Map{
+					"batch": pulumi.Map{},
+					"k8sattributes": pulumi.Map{
+						"auth_type":   pulumi.String("serviceAccount"),
+						"passthrough": pulumi.Bool(false),
+						"filter": pulumi.Map{
+							"node_from_env_var": pulumi.String("KUBERNETES_NODE_NAME"),
+						},
+						"extract": pulumi.Map{
+							"metadata": pulumi.StringArray{
+								pulumi.String("k8s.pod.name"),
+								pulumi.String("k8s.pod.uid"),
+								pulumi.String("k8s.deployment.name"),
+								pulumi.String("k8s.namespace.name"),
+								pulumi.String("k8s.node.name"),
+								pulumi.String("k8s.pod.start_time"),
+								pulumi.String("k8s.cluster.uid"),
+							},
+							"labels": pulumi.Array{
+								pulumi.Map{
+									"tag_name": pulumi.String("app"),
+									"key":      pulumi.String("app"),
+									"from":     pulumi.String("pod"),
+								},
+							},
+						},
+						"pod_association": pulumi.Array{
+							pulumi.Map{
+								"sources": pulumi.Array{
+									pulumi.Map{
+										"from": pulumi.String("resource_attribute"),
+										"name": pulumi.String("k8s.pod.name"),
+									},
+									pulumi.Map{
+										"from": pulumi.String("resource_attribute"),
+										"name": pulumi.String("k8s.namespace.name"),
+									},
+								},
+							},
+						},
+					},
+				},
+				"exporters": pulumi.Map{
+					"otlphttp/victorialogs": pulumi.Map{
+						"logs_endpoint": pulumi.String("http://victoria-logs-victoria-logs-single-server:9428/insert/opentelemetry/v1/logs"),
+						"headers": pulumi.Map{
+							"VL-Msg-Field":     pulumi.String("body"),
+							"VL-Time-Field":    pulumi.String("timestamp"),
+							"VL-Stream-Fields": pulumi.String("k8s.namespace.name,k8s.pod.name,k8s.container.name,log.iostream"),
+						},
+						"timeout": pulumi.String("10s"),
+						"retry_on_failure": pulumi.Map{
+							"enabled":          pulumi.Bool(true),
+							"initial_interval": pulumi.String("5s"),
+							"max_interval":     pulumi.String("30s"),
+							"max_elapsed_time": pulumi.String("300s"),
+						},
+						"sending_queue": pulumi.Map{
+							"enabled":       pulumi.Bool(true),
+							"num_consumers": pulumi.Int(10),
+							"queue_size":    pulumi.Int(50),
+						},
+					},
+				},
+				"service": pulumi.Map{
+					"pipelines": pulumi.Map{
+						"logs": pulumi.Map{
+							"receivers":  pulumi.StringArray{pulumi.String("filelog")},
+							"processors": pulumi.StringArray{pulumi.String("batch"), pulumi.String("k8sattributes")},
+							"exporters":  pulumi.StringArray{pulumi.String("otlphttp/victorialogs")},
+						},
+					},
+				},
+			},
+			"extraVolumes": pulumi.Array{
+				pulumi.Map{
+					"name": pulumi.String("varlogpods"),
+					"hostPath": pulumi.Map{
+						"path": pulumi.String("/var/log/pods"),
+					},
+				},
+				pulumi.Map{
+					"name": pulumi.String("varlibdockercontainers"),
+					"hostPath": pulumi.Map{
+						"path": pulumi.String("/var/lib/docker/containers"),
+					},
+				},
+			},
+			"extraVolumeMounts": pulumi.Array{
+				pulumi.Map{
+					"name":      pulumi.String("varlogpods"),
+					"mountPath": pulumi.String("/var/log/pods"),
+					"readOnly":  pulumi.Bool(true),
+				},
+				pulumi.Map{
+					"name":      pulumi.String("varlibdockercontainers"),
+					"mountPath": pulumi.String("/var/lib/docker/containers"),
+					"readOnly":  pulumi.Bool(true),
+				},
+			},
+			"extraEnvs": pulumi.Array{
+				pulumi.Map{
+					"name": pulumi.String("KUBERNETES_NODE_NAME"),
+					"valueFrom": pulumi.Map{
+						"fieldRef": pulumi.Map{
+							"fieldPath": pulumi.String("spec.nodeName"),
+						},
+					},
+				},
+			},
+			"resources": pulumi.Map{
+				"requests": pulumi.Map{
+					"memory": pulumi.String("200Mi"),
+					"cpu":    pulumi.String("100m"),
+				},
+				"limits": pulumi.Map{
+					"memory": pulumi.String("400Mi"),
+					"cpu":    pulumi.String("200m"),
+				},
+			},
+			"tolerations": pulumi.Array{
+				pulumi.Map{
+					"key":      pulumi.String("node-role.kubernetes.io/master"),
+					"operator": pulumi.String("Exists"),
+					"effect":   pulumi.String("NoSchedule"),
+				},
+				pulumi.Map{
+					"key":      pulumi.String("node-role.kubernetes.io/control-plane"),
+					"operator": pulumi.String("Exists"),
+					"effect":   pulumi.String("NoSchedule"),
+				},
+			},
+		},
+	}, pulumi.Provider(cluster.Provider))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func deployGrafana(ctx *pulumi.Context, cluster *providers.ProviderInfo, ns *corev1.Namespace, environment string, ingressNginx *helm.Chart) error {
 	conf := config.New(ctx, "mcp-registry")
 	grafanaSecret, err := corev1.NewSecret(ctx, "grafana-secrets", &corev1.SecretArgs{
@@ -149,7 +420,7 @@ func deployGrafana(ctx *pulumi.Context, cluster *providers.ProviderInfo, ns *cor
 		return err
 	}
 
-	// Create VictoriaMetrics datasource
+	// Create VictoriaMetrics and VictoriaLogs datasources
 	datasourcesConfig := map[string]interface{}{
 		"apiVersion": 1,
 		"datasources": []map[string]interface{}{
@@ -160,6 +431,15 @@ func deployGrafana(ctx *pulumi.Context, cluster *providers.ProviderInfo, ns *cor
 				"access":    "proxy",
 				"isDefault": true,
 			},
+			{
+				"name":   "VictoriaLogs",
+				"type":   "victoriametrics-logs-datasource",
+				"url":    "http://victoria-logs-victoria-logs-single-server:9428",
+				"access": "proxy",
+				"jsonData": map[string]interface{}{
+					"maxLines": 1000,
+				},
+			},
 		},
 	}
 
@@ -187,6 +467,9 @@ func deployGrafana(ctx *pulumi.Context, cluster *providers.ProviderInfo, ns *cor
 		},
 		Namespace: ns.Metadata.Name().Elem(),
 		Values: pulumi.Map{
+			"plugins": pulumi.Array{
+				pulumi.String("victoriametrics-logs-datasource"),
+			},
 			"extraConfigmapMounts": pulumi.Array{
 				pulumi.Map{
 					"name":      pulumi.String("grafana-datasources"),