product-kubearchive-logging: add WAL settings and PVC retention policy

Signed-off-by: obetsun <[email protected]>

rh-pre-commit.version: 2.3.2
rh-pre-commit.check-secrets: ENABLED

Author: olegbet

components/vector-kubearchive-log-collector/development/kustomization.yaml

@@ -39,6 +39,19 @@ patches:
       version: v1
       kind: PodDisruptionBudget
       labelSelector: app.kubernetes.io/name=loki
+  - patch: |
+        - op: replace
+          path: /spec/persistentVolumeReclaimPolicy/whenDeleted
+          value: Delete
+          path: /spec/persistentVolumeReclaimPolicy/whenScaled
+          value: Delete
+    target:
+      group: apps
+      version: v1
+      kind: StatefulSet
+      name: loki-ingester
+      namespace: product-kubearchive-logging
+      labelSelector: app.kubernetes.io/name=loki,app.kubernetes.io/component=ingester
 
 generators:
 - vector-helm-generator.yaml

components/vector-kubearchive-log-collector/development/loki-helm-dev-values.yaml

@@ -107,10 +107,20 @@ loki:
     autoforget_unhealthy: true
     chunk_encoding: snappy
     chunk_target_size: 3145728
-    chunk_idle_period: 5m
+    chunk_idle_period: 1h
     max_chunk_age: 2h
     chunk_retain_period: 1h
     flush_op_timeout: 10m
+    lifecycler:
+      ring:
+        heartbeat_timeout: 10m
+    # WAL Configuration - for preventing OOM
+    wal:
+      enabled: true
+      dir: /var/loki/wal
+      checkpoint_duration: 5m          # Create checkpoints every 5 minutes
+      flush_on_shutdown: true          # Ensure data is flushed before shutdown
+      replay_memory_ceiling: 4GB       # 75% of 6Gi limit - PREVENTS OOM during replay
   server:
     grpc_server_max_recv_msg_size: 15728640  # 15MB
     grpc_server_max_send_msg_size: 15728640
@@ -141,12 +151,12 @@ ingester:
     enabled: false
   resources:
     requests:
-      cpu: 50m  # Reduce from 100m
-      memory: 256Mi  # Increase from 128Mi to prevent OOM
+      cpu: 500m  # Reduce from 100m
+      memory: 4Gi  # Increase from 128Mi to prevent OOM
     limits:
-      memory: 512Mi  # Increase from 256Mi to prevent OOM
+      memory: 6Gi  # Increase from 256Mi to prevent OOM
   persistence:
-    enabled: false  # Disable persistence for dev to avoid storage class issues
+    enabled: true  # Disable persistence for dev to avoid storage class issues
     size: 10Gi
   affinity: {}
   podAntiAffinity:

components/vector-kubearchive-log-collector/production/stone-prod-p02/kustomization.yaml

@@ -14,6 +14,21 @@ commonAnnotations:
 resources:
 - ../base
 
+patches:
+  - patch: |
+        - op: replace
+          path: /spec/persistentVolumeReclaimPolicy/whenDeleted
+          value: Delete
+          path: /spec/persistentVolumeReclaimPolicy/whenScaled
+          value: Delete
+    target:
+      group: apps
+      version: v1
+      kind: StatefulSet
+      name: loki-ingester
+      namespace: product-kubearchive-logging
+      labelSelector: app.kubernetes.io/name=loki,app.kubernetes.io/component=ingester
+
 generators:
 - vector-helm-generator.yaml
 - loki-helm-generator.yaml

components/vector-kubearchive-log-collector/production/stone-prod-p02/loki-helm-prod-values.yaml

@@ -75,11 +75,20 @@ loki:
   ingester:
     autoforget_unhealthy: true
     chunk_target_size: 8388608        # 8MB
-    chunk_idle_period: 5m
+    chunk_idle_period: 1h
     max_chunk_age: 2h
     chunk_encoding: snappy            # Compress data (reduces S3 transfer size)
     chunk_retain_period: 1h           # Keep chunks in memory after flush
     flush_op_timeout: 10m             # Add timeout for S3 operations
+    lifecycler:
+      ring:
+        heartbeat_timeout: 10m
+    wal:
+      enabled: true
+      dir: /var/loki/wal
+      checkpoint_duration: 5m
+      flush_on_shutdown: true
+      replay_memory_ceiling: 4GB
   server:
     grpc_server_max_recv_msg_size: 15728640  # 15MB
     grpc_server_max_send_msg_size: 15728640