Merge branch 'main' into hotfix-hapi-fhir-tag-fixing

Author: arran-standish

documentation/packages/monitoring/README.md

@@ -4,14 +4,15 @@ description: A package for monitoring the platform services
 
 # Monitoring
 
-The monitoring package sets up services to monitor the entire deployed stack. This includes the state of the servers involved in the docker swarm, the docker containers themselves and particular applications such as Kafka.
+The monitoring package sets up services to monitor the entire deployed stack. This includes the state of the servers involved in the docker swarm, the docker containers themselves and particular applications such as Kafka. It also captures the logs from the various services.
 
 This monitoring package uses: 
 
 * Grafana: for dashboards
 * Prometheus: for recording metrics
 * Cadvisor: for reading docker container metrics 
-* Kafka: for saving a backup of metrics data
+* Loki: for storing logs
+* Node Exporter: for monitoring host machine metrics like CPU, memory etc
 
 To use the monitoring services, include the `monitoring` package id to your list of package ids when standing up the platform.
 
@@ -31,7 +32,20 @@ To use custom metrics for an application, first configure that application to pr
 <strong>        - prometheus-job-service=kafka
 </strong><strong>        - prometheus-address=kafka-minion:8080</strong></code></pre>
 
-`prometheus-job` lets Prometheus know to enable monitoring for this container and `prometheus-address` give the endpoint that the monitoring can access the metrics on. By default this is assumed to be at the path `/metrics` by Prometheus.
+`prometheus-job-service` lets Prometheus know to enable monitoring for this container and `prometheus-address` gives the endpoint that Prometheus can access the metrics on. By default this is assumed to be at the path `/metrics` by Prometheus.
+
+By using the `prometheus-job-service` label prometheus will only create a single target for your application even if it is replicated via service config in docker swarm. If you would like to monitor each replica separately (i.e. if metrics are only captured for that replica and not shared to some central location in the application cluster) you can instead used the `prometheus-job-task` label and Prometheus will create a target for each replica.
+
+A full list od supported labels are listed below:
+
+* `prometheus-job-service` - indicates this service should be monitored
+* `prometheus-job-task` - indicates each task in the replicated service should be monitored separately
+* `prometheus-address` - the service address Prometheus can scrape metrics from, can only be used with `prometheus-job-service`
+* `prometheus-scheme` - the scheme to use when scaping a task or service (e.g. http or https), defaults to http
+* `prometheus-metrics-path` - the path to the metrics endpoint on the target (defaults to /metrics)
+* `prometheus-port` - the port of the metrics endpoint. Only usable with `prometheus-job-task`, defaults to all exposed ports for the container if no label is present
+
+All services must also be on the `prometheus_public` network to be able to be seen by Prometheus for metrics scraping.
 
 ## Adding additional dashboards
 

interoperability-layer-openhim/docker-compose.yml

@@ -2,14 +2,15 @@ version: '3.9'
 
 services:
   openhim-core:
-    image: jembi/openhim-core:v8.0.1
+    image: jembi/openhim-core:v8.1.1
     networks:
       kafka:
       hapi-fhir:
       reverse-proxy:
       keycloak:
       public:
       default:
+      prometheus:
     environment:
       - mongo_url=${OPENHIM_MONGO_URL}
       - mongo_atnaUrl=${OPENHIM_MONGO_ATNAURL}
@@ -31,6 +32,10 @@ services:
         reservations:
           cpus: ${OPENHIM_CPU_RESERVE}
           memory: ${OPENHIM_MEMORY_RESERVE}
+      labels:
+        - prometheus-job-task=openhim
+        - prometheus-scheme=https
+        - prometheus-port=8080
 
   openhim-console:
     image: jembi/openhim-console:v1.18.2
@@ -74,4 +79,7 @@ networks:
   public:
     name: openhim_public
     external: true
+  prometheus:
+    name: prometheus_public
+    external: true
   default:

monitoring/docker-compose.yml

@@ -50,6 +50,10 @@ services:
         source: kminion-groups_rev1.json
       - target: /etc/grafana/provisioning/dashboards/applications/kminion-topic_rev1.json
         source: kminion-topic_rev1.json
+      - target: /etc/grafana/provisioning/dashboards/applications/openhim_nodejs_dashboard.json
+        source: openhim_nodejs_dashboard.json
+      - target: /etc/grafana/provisioning/dashboards/applications/openhim_transactions_dashboard.json
+        source: openhim_transactions_dashboard.json
       - target: /etc/grafana/provisioning/dashboards/containers/logging-universal-dashboard_rev1.json
         source: logging-universal-dashboard_rev1.json
     networks:
@@ -195,6 +199,16 @@ configs:
     name: kminion-topic_rev1.json-${kminion_topic_rev1_json_DIGEST:?err}
     labels:
       name: grafana
+  openhim_nodejs_dashboard.json:
+    file: ./grafana/dashboards/applications/openhim_nodejs_dashboard.json
+    name: openhim_nodejs_dashboard.json-${openhim_nodejs_dashboard_json_DIGEST:?err}
+    labels:
+      name: grafana
+  openhim_transactions_dashboard.json:
+    file: ./grafana/dashboards/applications/openhim_transactions_dashboard.json
+    name: openhim_transactions_dashboard.json-${openhim_transactions_dashboard_json_DIGEST:?err}
+    labels:
+      name: grafana
   logging-universal-dashboard_rev1.json:
     file: ./grafana/dashboards/containers/logging-universal-dashboard_rev1.json
     name: logging-universal-dashboard_rev1.json-${logging_universal_dashboard_rev1_json_DIGEST:?err}