Merge remote-tracking branch 'upstream/main' into 2025/refactor/pgBackup

Author: mrnicegyu11

services/logging/Makefile

@@ -7,11 +7,24 @@ STACK_NAME = $(notdir $(shell pwd))
 TEMP_COMPOSE=.stack.${STACK_NAME}.yaml
 REPO_BASE_DIR := $(shell git rev-parse --show-toplevel)
 
+define create-s3-bucket
+	# ensure bucket is available in S3...
+	set -o allexport; \
+	source .env; \
+	echo Creating bucket "$${S3_BUCKET_NAME_LOKI}";\
+	export S3_ACCESS_KEY=$${S3_ACCESS_KEY_LOKI}; \
+	export S3_SECRET_KEY=$${S3_SECRET_KEY_LOKI}; \
+	export STORAGE_DOMAIN=$${STORAGE_DOMAIN}; \
+	${REPO_BASE_DIR}/scripts/create-s3-bucket.bash "$${S3_BUCKET_NAME_LOKI}" && \
+	set +o allexport; \
+	# bucket is available in S3
+endef
+
 # TARGETS --------------------------------------------------
 include ${REPO_BASE_DIR}/scripts/common.Makefile
 
 .PHONY: up
-up: .init .env ${TEMP_COMPOSE}  ## Deploys graylog stack
+up: .init .env ${TEMP_COMPOSE}  ## Deploys logging stack
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE} ${STACK_NAME}
 	$(MAKE) configure
 
@@ -43,6 +56,7 @@ up-master: .init .env ${TEMP_COMPOSE}-master
 
 .PHONY: up-local ## Deploys graylog stack for local deployment
 up-local: .init .env ${TEMP_COMPOSE}-local  ## Deploys graylog stack for local cluster
+	@$(create-s3-bucket)
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-local ${STACK_NAME}
 	$(MAKE) configure
 

services/logging/fluentd/fluent.conf

@@ -30,6 +30,8 @@
   @type forward
   port 24224
   bind 0.0.0.0
+  # Add source hostname to records
+  source_hostname_key source_hostname
 </source>
 
 # Add additional metadata
@@ -38,10 +40,22 @@
   <record>
     hostname "#{Socket.gethostname}"
     fluentd_hostname "#{ENV['FLUENTD_HOSTNAME']}"
-    tag ${tag}
   </record>
 </filter>
 
+# Clean container names and set proper host field
+<filter docker.**>
+  @type record_transformer
+  enable_ruby true
+  <record>
+    # cleanup container names by removing leading slashes
+    container_name ${record["container_name"] ? record["container_name"].sub(/^\//, '') : record["container_name"]}
+    # Use source hostname from forward input as the host field for GELF
+    host ${record["source_hostname"] || record["source"] || record["_hostname"] || "unknown"}
+  </record>
+</filter>
+
+
 # Output to both Graylog (GELF) and Loki
 <match docker.**>
   @type copy
@@ -54,6 +68,12 @@
     protocol udp
     add_msec_time true
     flush_interval 5s
+    # Use the host field from record for GELF host field
+    use_record_host true
+    # Map the correct fields for Graylog
+    <format>
+      @type json
+    </format>
     <buffer>
       @type file
       path /fluentd/buffer/graylog

services/logging/template.env

@@ -16,10 +16,11 @@ GRAYLOG_LOG_MAX_DAYS_IN_STORAGE=${GRAYLOG_LOG_MAX_DAYS_IN_STORAGE}
 GRAYLOG_LOG_MIN_DAYS_IN_STORAGE=${GRAYLOG_LOG_MIN_DAYS_IN_STORAGE}
 PUBLIC_NETWORK=${PUBLIC_NETWORK}
 MONITORED_NETWORK=${MONITORED_NETWORK}
-LOKI_RETETION_PERIOD=${LOKI_RETETION_PERIOD}
+LOKI_RETENTION_PERIOD=${LOKI_RETENTION_PERIOD}
 S3_ENDPOINT_LOKI=${S3_ENDPOINT_LOKI}
 S3_ACCESS_KEY_LOKI=${S3_ACCESS_KEY_LOKI}
 S3_BUCKET_NAME_LOKI=${S3_BUCKET_NAME_LOKI}
 S3_FORCE_PATH_STYLE_LOKI=${S3_FORCE_PATH_STYLE_LOKI}
 S3_REGION_LOKI=${S3_REGION_LOKI}
 S3_SECRET_KEY_LOKI=${S3_SECRET_KEY_LOKI}
+STORAGE_DOMAIN=${STORAGE_DOMAIN}

services/monitoring/prometheus/prometheus.rules.yml

@@ -42,3 +42,8 @@ groups:
     rules:
       - record: osparc_autoscaling_machines_buffer
         expr: sum(swarm_node_info{instance=~"^ip-.*$"}) - ( sum(count_values("instance",node_boot_time_seconds{instance=~"^ip-.*$"})) OR clamp_max(absent(node_boot_time_seconds{instance=~"^ip-.*$"}),0) )
+  - name: osparc_ops_cpu_throttling_seconds
+    interval: 120s
+    rules:
+      - record: osparc_ops_cpu_throttling_seconds
+        expr: sum by (name) (rate(container_cpu_cfs_throttled_seconds_total[5m])) > 0.2