Merge pull request #104 from opencrvs/fix-apm-service-name

fix: Improved Pod metrics visibility

Author: adskyiproger

.github/TEMPLATES/secret-mapping-deps.yml

@@ -25,24 +25,6 @@ postgres-admin-user:
   - POSTGRES_USER
   - POSTGRES_PASSWORD
 
-# SMTP_HOST
-
-# SMTP_PORT
-
-# SMTP_USERNAME
-
-# SMTP_PASSWORD
-
-# SMTP_SECURE
-
-# Whether or not your SMTP port requires TLS
-
-# ALERT_EMAIL
-
-# Email address or Slack channel address to send system technical alerts to.
-
-# SENDER_EMAIL_ADDRESS
-
 
 # REPLICAS
 

.github/TEMPLATES/secret-mapping-opencrvs.yml

@@ -35,13 +35,14 @@ smtp-config:
   - SMTP_PORT
   - SMTP_SECURE
   - SMTP_USERNAME
+  - ALERT_EMAIL
+
+# TODO:
 
 # CONTENT_SECURITY_POLICY_WILDCARD
 
 # This string is supplied to the clients and nginx config and ensures that the format of your domain above can be configurable for CORS purposes.
 
 # ACTIVATE_USERS
-
 # NOTIFICATION_TRANSPORT
-
 # A prop which can be used to configure either Email or SMS for staff and beneficiary comms or potentially both.

.github/workflows/k8s-reset-data.yml

@@ -67,7 +67,7 @@ jobs:
                 -s templates/postgres-on-update-core.yaml \
                 oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply -n ${namespace} --wait=true -f -;
             kubectl wait --for=condition=complete job/postgres-on-update-core -n ${namespace} --timeout=600s || true
-            kubectl logs job/postgres-on-update-core -f --all-containers=true -n ${namespace}; || true
+            kubectl logs job/postgres-on-update-core -f --all-containers=true -n ${namespace} || true
         - name: Re-run postgres-data-migration
           run: |
             kubectl delete job -n ${namespace} --ignore-not-found=true postgres-data-migration;

charts/dependencies/README.md

@@ -266,6 +266,18 @@ elasticsearch:
   use_default_credentials: false
 ```
 
+For backward compatibility `HTTP_POST2_ALERT_URL` environment variable needs to be added to elastalert configuration. All alerts will be send to country config service and forwarded to email address
+
+See example:
+```yaml
+elastalert:
+  env:
+    HTTP_POST2_ALERT_URL: http://countryconfig.opencrvs-dev.svc.cluster.local:3040/email
+```
+
+> NOTE: This behavior will be changed in future releases, see [#10608](https://github.com/opencrvs/opencrvs-core/issues/10608)
+
+
 ## Backup Configuration
 
 The dependencies chart includes a built-in backup feature that supports automated backups for internal components. Backups are stored on an external server via an SSH connection.

…dencies/files/beats/rollover-policy.json …iles/beats/filebeat-rollover-policy.jsoncharts/dependencies/files/beats/rollover-policy.json renamed to charts/dependencies/files/beats/filebeat-rollover-policy.json charts/dependencies/files/beats/rollover-policy.json renamed to charts/dependencies/files/beats/filebeat-rollover-policy.json

@@ -16,6 +16,12 @@ filebeat.autodiscover:
     - type: kubernetes
       hints.enabled: true
 
+filebeat.modules:
+  - module: system
+    syslog:
+      enabled: true
+    auth:
+      enabled: true
 #================================ Processors ===================================
 processors:
   - add_kubernetes_metadata: ~
@@ -66,7 +72,11 @@ processors:
             event.Put("level_name", levelName);
             event.Put("log.level", levelName);  // ECS standard
           }
-          
+        }
+  - script:
+      lang: javascript
+      source: >
+        function process(event) {
           // Kubernetes correlation
           var k8s = event.Get("kubernetes");
           if (k8s) {

charts/dependencies/files/beats/filebeat.yml

@@ -0,0 +1,52 @@
+{
+  "policy": {
+      "phases": {
+        "hot": {
+          "actions": {
+            "rollover": {
+              "max_size": "2GB",
+              "max_age": "1d"
+            },
+            "set_priority": {
+              "priority": 100
+            }
+          }
+        },
+        "warm": {
+          "min_age": "3d",
+          "actions": {
+            "allocate": {
+              "number_of_replicas": 0
+            },
+            "forcemerge": {
+              "max_num_segments": 1
+            },
+            "set_priority": {
+              "priority": 50
+            }
+          }
+        },
+        "cold": {
+          "min_age": "7d",
+          "actions": {
+            "allocate": {
+              "number_of_replicas": 0
+            },
+            "set_priority": {
+              "priority": 0
+            }
+          }
+        },
+        "delete": {
+          "min_age": "30d",
+          "actions": {
+            "delete": {}
+          }
+        }
+    },
+    "_meta": {
+      "managed": true,
+      "description": "built-in ILM policy using the hot and warm phases with a retention of 7 days"
+    }
+  }
+}

charts/dependencies/files/beats/metricbeat-rollover-policy.json

@@ -1,94 +1,42 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# OpenCRVS is also distributed under the terms of the Civil Registration
-# & Healthcare Disclaimer located at http://opencrvs.org/license.
-#
-# Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
----
-#-------------------------------- Autodiscovery -------------------------------
-# metricbeat.autodiscover:
-#   providers:
-#     - type: kubernetes
-#       node: ${NODE_IP}
-#       hints.enabled: true
+system.hostfs: /hostfs
 
 metricbeat.modules:
-  #------------------------------- System Module -------------------------------
-  - module: system
-    metricsets:
-      - cpu
-      - load
-      - memory
-      - network
-      - process
-      - process_summary
-      - core
-      - diskio
-      - socket
-    processes: ['.*']
-    process.include_top_n:
-      by_cpu: 5
-      by_memory: 5
-    period: 10s
-    cpu.metrics: ['percentages']
-    core.metrics: ['percentages']
-
-  - module: system
-    period: 1m
-    metricsets:
-      - filesystem
-      - fsstat
-    processors:
-      - drop_event.when.regexp:
-          system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)'
-
-  - module: system
-    period: 15m
-    metricsets:
-      - uptime
-
-  #------------------------------- Kubernetes Module -------------------------------
-  # Kubernetes pod metrics
-  - module: kubernetes
-    period: 10s
-    host: ${NODE_IP}
-    hosts: ["https://${NODE_IP}:10250"]
-    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-    ssl.verification_mode: "none"
-    metricsets: ["node", "pod", "container", "volume"]
-
-#================================ Processors ===================================
+- module: kubernetes
+  period: 10s
+  host: ${NODE_NAME}
+  hosts: ["https://${NODE_IP}:10250"]
+  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+  ssl.verification_mode: "none"
+  metricsets:
+    - node
+    - pod
+    - container
+    - volume
+    - system
+  add_metadata: true
+- module: system
+  metricsets:
+    - cpu
+    - memory
+    - network
+    - process
+    - diskio
+  period: 10s
+- module: system
+  metricsets:
+    - filesystem
+  period: 60s
+  processors:
+    - drop_event.when.regexp:
+        system.filesystem.mount_point: "^/hostfs/run"
 processors:
-  - add_fields:
-      target: orchestrator.cluster.name
-      fields:
-        name: opencrvs-k8s
-
+  - add_host_metadata: ~
   - add_kubernetes_metadata:
-      host: ${NODE_IP}
-  - add_locale:
-      format: offset
-  - add_host_metadata:
-      netinfo.enabled: true
-  # # Ensure proper field mapping for Infrastructure correlation
-  - script:
-      lang: javascript
-      source: >
-        function process(event) {
-          var k8s = event.Get("kubernetes");
-          if (k8s && k8s.pod && k8s.pod.name) {
-            event.Put("host.name", k8s.pod.name);
-            event.Put("host.hostname", k8s.pod.name);
-            if (k8s.container && k8s.container.name) {
-              event.Put("container.name", k8s.container.name);
-            }
-          if (k8s && k8s.labels && k8s.labels["service_name"]) {
-            event.Put("service.name", k8s.labels["service_name"]);
-          }
-          }
-        }
+      host: ${NODE_NAME}
+      # Enable more indexers for better pod correlation
+      default_indexers.enabled: true
+      default_matchers.enabled: true
+
 #========================== Elasticsearch output ===============================
 output.elasticsearch:
   hosts: ['${ELASTICSEARCH_HOST}']
@@ -97,7 +45,7 @@ output.elasticsearch:
 
 #============================== Dashboards =====================================
 setup.dashboards:
-  enabled: true
+  enabled: false
 
 #============================== Kibana =========================================
 setup.kibana:
@@ -120,4 +68,6 @@ setup.ilm.overwrite: true
 
 #============================== Logging ===============================
 logging.level: info
-logging.to_stderr: true
+logging.selectors: ["*"]
+logging.metrics.enabled: true
+logging.to_files: false

charts/dependencies/files/beats/metricbeat.yml

@@ -20,5 +20,7 @@ es_port: "${ES_HOST}"
 # es_password: <passed as environment variables>
 writeback_index: elastalert_status
 
+http_post2_url: "${HTTP_POST2_ALERT_URL}"
+
 alert_time_limit:
   days: 2

charts/dependencies/files/elastalert/elastalert.yaml

@@ -36,7 +36,7 @@ filter:
       minimum_should_match: 1
 
 alert: post2
-http_post2_url: 'http://countryconfig:3040/email'
+http_post2_url: '${HTTP_POST2_ALERT_URL}'
 http_post2_payload:
   subject: '{% raw %}{{DOMAIN}}{% endraw %} {{kibana.alert.context.metrics__alert__inventory__threshold.alertState}}: {{rule.name}} 🚨'
   html: 'Reason: {{kibana.alert.context.metrics__alert__inventory__threshold.reason}}. Login to https://kibana.{% raw %}{{DOMAIN}}{% endraw %} to view the alert.'