Modify configurations in the logging module

Author: linnjia-aws

nomad/origin/logs-collector.hcl

@@ -39,7 +39,7 @@ job "logs-collector" {
       config {
         network_mode = "host"
         image        = "timberio/vector:0.34.X-alpine"
-
+        auth_soft_fail = true
         ports = [
           "health",
           "logs",
@@ -72,30 +72,77 @@ data_dir = "alloc/data/vector/"
 enabled = true
 address = "0.0.0.0:44313"
 
-[sources.envd]
+[sources.http_server]
 type = "http_server"
 address = "0.0.0.0:30006"
-encoding = "json"
+encoding = "ndjson"
 path_key = "_path"
 
-[transforms.add_source_envd]
+[transforms.add_source_http_server]
 type = "remap"
-inputs = ["envd"]
+inputs = ["http_server"]
 source = """
 del(."_path")
-.service = "envd"
 .sandboxID = .instanceID
+.timestamp = parse_timestamp(.timestamp, format: "%+") ?? now()
+# Normalize keys
+if exists(.sandbox_id) {
+  .sandboxID = .sandbox_id
+}
+if exists(.build_id) {
+  .buildID = .build_id
+}
+if exists(.env_id) {
+  .envID = .env_id
+}
+if exists(.team_id) {
+  .teamID = .team_id
+}
+if exists(."template.id") {
+  .templateID = ."template.id"
+  del(."template.id")
+}
+if exists(."sandbox.id") {
+  .sandboxID = ."sandbox.id"
+  del(."sandbox.id")
+}
+if exists(."build.id") {
+  .buildID = ."build.id"
+  del(."build.id")
+}
+if exists(."env.id") {
+  .envID = ."env.id"
+  del(."env.id")
+}
+if exists(."team.id") {
+  .teamID = ."team.id"
+  del(."team.id")
+}
+
+# Apply defaults if not already set
 if !exists(.envID) {
   .envID = "unknown"
 }
 if !exists(.category) {
   .category = "default"
 }
+if !exists(.teamID) {
+  .teamID = "unknown"
+}
+if !exists(.sandboxID) {
+  .sandboxID = "unknown"
+}
+if !exists(.buildID) {
+  .buildID = "unknown"
+}
+if !exists(.service) {
+  .service = "envd"
+}
 """
 
 [transforms.internal_routing]
 type = "route"
-inputs = [ "add_source_envd" ]
+inputs = [ "add_source_http_server" ]
 
 [transforms.internal_routing.route]
 internal = '.internal == true'
@@ -107,23 +154,33 @@ source = '''
 del(.internal)
 '''
 
+# Enable debugging of logs to the console
+# [sinks.console_loki]
+# type = "console"
+# inputs = ["remove_internal"]
+# encoding.codec = "json"
+
 [sinks.local_loki_logs]
 type = "loki"
 inputs = [ "remove_internal" ]
 endpoint = "http://loki.service.consul:3100"
 encoding.codec = "json"
+# This is recommended behavior for Loki 2.4.0 and newer and is default in Vector 0.39.0 and newer
+# https://vector.dev/docs/reference/configuration/sinks/loki/#out_of_order_action
+# https://vector.dev/releases/0.39.0/
+out_of_order_action = "accept"
 
 [sinks.local_loki_logs.labels]
 source = "logs-collector"
 service = "{{ service }}"
 teamID = "{{ teamID }}"
 envID = "{{ envID }}"
+buildID = "{{ buildID }}"
 sandboxID = "{{ sandboxID }}"
 category = "{{ category }}"
 
-
         EOH
       }
     }
   }
-}
+}

nomad/origin/loki.hcl

@@ -21,7 +21,7 @@ job "loki" {
         path     = "/ready"
         interval = "20s"
         timeout  = "2s"
-        port     = "loki"
+        port     = "3100"
       }
     }
 
@@ -31,7 +31,7 @@ job "loki" {
       config {
         network_mode = "host"
         image = "grafana/loki:2.9.8"
-
+        auth_soft_fail = true
         args = [
           "-config.file",
           "local/loki-config.yml",
@@ -67,38 +67,40 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/tsdb-shipper-active
     cache_location: /loki/tsdb-shipper-cache
-    cache_ttl: 24h
+    cache_ttl: 1h
     shared_store: aws
 
 chunk_store_config:
   chunk_cache_config:
     embedded_cache:
       enabled: true
-      max_size_mb: 512
-      ttl: 1h
+      max_size_mb: 2048
+      ttl: 30m
 
 query_range:
   align_queries_with_step: true
   cache_results: true
-  max_retries: 5
+  max_retries: 2
   results_cache:
     cache:
       embedded_cache:
         enabled: true
-        max_size_mb: 256
-        ttl: 1h
+        max_size_mb: 2048
+        ttl: 30m
 
 ingester_client:
   grpc_client_config:
     max_recv_msg_size: 104857600  # 100 Mb
     max_send_msg_size: 104857600  # 100 Mb
 
 ingester:
-  chunk_idle_period: 30m
+  chunk_idle_period: 10m
   chunk_encoding: snappy
+  max_chunk_age: 15m
   chunk_target_size: 1048576  # 1MB
   wal:
     dir: /loki/wal
+    enabled: false
     flush_on_shutdown: true
 
 schema_config:
@@ -127,7 +129,8 @@ limits_config:
   per_stream_rate_limit: "80MB"
   per_stream_rate_limit_burst: "240MB"
   max_streams_per_user: 0
-  max_global_streams_per_user: 10000
+  split_queries_by_interval: 30m
+  max_global_streams_per_user: 0
   unordered_writes: true
   reject_old_samples_max_age: 168h
 EOF