feat(observability): 增强服务实例标识与监控面板

JaD1ng · JaD1ng · commit cc9f91a18b24 · 2025-09-08T15:28:38.000+08:00
- 在Prometheus配置中添加service_id标签用于区分服务实例
- 为OTEL资源添加service_instance_id属性
- 改进Consul注册地址探测逻辑，优先使用容器网卡IP
- 更新Grafana仪表板以支持按实例筛选和展示
diff --git a/mock/s3/deployments/observability/grafana/dashboards/mock-s3-services-metrics.json b/mock/s3/deployments/observability/grafana/dashboards/mock-s3-services-metrics.json
@@ -28,6 +28,20 @@
           "text": "All",
           "value": "$__all"
         }
+      },
+      {
+        "name": "instance",
+        "type": "query",
+        "query": "label_values(system_cpu_usage_percent_percent{service=~\"$service\"}, instance)",
+        "refresh": 1,
+        "includeAll": true,
+        "allValue": ".*",
+        "multi": true,
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
+        }
       }
     ]
   },
@@ -38,8 +52,8 @@
       "type": "timeseries",
       "targets": [
         {
-          "expr": "system_cpu_usage_percent_percent{service=~\"$service\"}",
-          "legendFormat": "{{service}} CPU",
+          "expr": "system_cpu_usage_percent_percent{service=~\"$service\",instance=~\"$instance\"}",
+          "legendFormat": "{{service}} {{instance}} CPU",
           "refId": "A"
         }
       ],
@@ -74,8 +88,8 @@
       "type": "timeseries",
       "targets": [
         {
-          "expr": "system_memory_usage_percent_percent{service=~\"$service\"}",
-          "legendFormat": "{{service}} Memory",
+          "expr": "system_memory_usage_percent_percent{service=~\"$service\",instance=~\"$instance\"}",
+          "legendFormat": "{{service}} {{instance}} Memory",
           "refId": "A"
         }
       ],
@@ -110,7 +124,7 @@
       "type": "stat",
       "targets": [
         {
-          "expr": "system_cpu_usage_percent_percent{service=~\"$service\"}",
+          "expr": "system_cpu_usage_percent_percent{service=~\"$service\",instance=~\"$instance\"}",
           "legendFormat": "{{service}}",
           "refId": "A"
         }
@@ -158,7 +172,7 @@
       "type": "stat",
       "targets": [
         {
-          "expr": "system_memory_usage_percent_percent{service=~\"$service\"}",
+          "expr": "system_memory_usage_percent_percent{service=~\"$service\",instance=~\"$instance\"}",
           "legendFormat": "{{service}}",
           "refId": "A"
         }
@@ -206,8 +220,8 @@
       "type": "timeseries",
       "targets": [
         {
-          "expr": "system_network_qps_per_second{service=~\"$service\"}",
-          "legendFormat": "{{service}} QPS",
+          "expr": "system_network_qps_per_second{service=~\"$service\",instance=~\"$instance\"}",
+          "legendFormat": "{{service}} {{instance}} QPS",
           "refId": "A"
         }
       ],
@@ -241,7 +255,7 @@
       "type": "stat",
       "targets": [
         {
-          "expr": "system_machine_online_status{service=~\"$service\"}",
+          "expr": "system_machine_online_status{service=~\"$service\",instance=~\"$instance\"}",
           "legendFormat": "{{service}}",
           "refId": "A"
         }
@@ -351,28 +365,28 @@
       "type": "table",
       "targets": [
         {
-          "expr": "system_cpu_usage_percent_percent{service=~\"$service\"}",
+          "expr": "system_cpu_usage_percent_percent{service=~\"$service\",instance=~\"$instance\"}",
           "legendFormat": "",
           "refId": "A",
           "instant": true,
           "format": "table"
         },
         {
-          "expr": "system_memory_usage_percent_percent{service=~\"$service\"}",
+          "expr": "system_memory_usage_percent_percent{service=~\"$service\",instance=~\"$instance\"}",
           "legendFormat": "",
           "refId": "B",
           "instant": true,
           "format": "table"
         },
         {
-          "expr": "system_network_qps_per_second{service=~\"$service\"}",
+          "expr": "system_network_qps_per_second{service=~\"$service\",instance=~\"$instance\"}",
           "legendFormat": "",
           "refId": "C",
           "instant": true,
           "format": "table"
         },
         {
-          "expr": "system_machine_online_status{service=~\"$service\"}",
+          "expr": "system_machine_online_status{service=~\"$service\",instance=~\"$instance\"}",
           "legendFormat": "",
           "refId": "D",
           "instant": true,
@@ -389,14 +403,14 @@
           "options": {
             "excludeByName": {
               "Time": true,
-              "instance": true,
               "job": true,
               "node": true,
               "otel_scope_name": true,
               "__name__": true
             },
             "renameByName": {
               "service": "Service",
+              "instance": "Instance",
               "Value #A": "CPU (%)",
               "Value #B": "Memory (%)",
               "Value #C": "Network QPS",
@@ -494,8 +508,8 @@
       "type": "piechart",
       "targets": [
         {
-          "expr": "topk(5, system_cpu_usage_percent_percent{service=~\"$service\"})",
-          "legendFormat": "{{service}}",
+          "expr": "topk(5, system_cpu_usage_percent_percent{service=~\"$service\",instance=~\"$instance\"})",
+          "legendFormat": "{{service}} {{instance}}",
           "refId": "A"
         }
       ],
@@ -534,8 +548,8 @@
       "type": "heatmap",
       "targets": [
         {
-          "expr": "avg_over_time(system_cpu_usage_percent_percent{service=~\"$service\"}[5m])",
-          "legendFormat": "{{service}}",
+          "expr": "avg_over_time(system_cpu_usage_percent_percent{service=~\"$service\",instance=~\"$instance\"}[5m])",
+          "legendFormat": "{{service}} {{instance}}",
           "refId": "A"
         }
       ],
@@ -560,4 +574,4 @@
   },
   "schemaVersion": 36,
   "version": 1
-}
+}
diff --git a/mock/s3/deployments/observability/prometheus.yml b/mock/s3/deployments/observability/prometheus.yml
@@ -33,7 +33,9 @@ scrape_configs:
     relabel_configs:
       - source_labels: [__meta_consul_service]
         target_label: service
+      - source_labels: [__meta_consul_service_id]
+        target_label: service_id
       - source_labels: [__meta_consul_node]
         target_label: node
 
-# 存储配置 - 命令行参数将在docker-compose中配置
+# 存储配置 - 命令行参数将在docker-compose中配置
diff --git a/mock/s3/shared/observability/providers.go b/mock/s3/shared/observability/providers.go
@@ -1,11 +1,12 @@
 package observability
 
 import (
-	"context"
-	"fmt"
-	"mocks3/shared/observability/config"
+    "context"
+    "fmt"
+    "mocks3/shared/observability/config"
+    "os"
 
-	"go.opentelemetry.io/otel"
+    "go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"
 	"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
@@ -182,11 +183,21 @@ func (p *Providers) Shutdown(ctx context.Context) error {
 
 // createResource 创建OTEL资源
 func createResource(config *config.ObservabilityConfig) (*resource.Resource, error) {
-	return resource.New(context.Background(),
-		resource.WithAttributes(
-			semconv.ServiceName(config.ServiceName),
-			semconv.ServiceVersion(config.ServiceVersion),
-			semconv.DeploymentEnvironment(config.Environment),
-		),
-	)
+    // Derive a stable service instance id
+    instanceID := os.Getenv("INSTANCE_ID")
+    if instanceID == "" {
+        if h, err := os.Hostname(); err == nil && h != "" {
+            instanceID = h
+        } else {
+            instanceID = "unknown-instance"
+        }
+    }
+    return resource.New(context.Background(),
+        resource.WithAttributes(
+            semconv.ServiceName(config.ServiceName),
+            semconv.ServiceVersion(config.ServiceVersion),
+            semconv.DeploymentEnvironment(config.Environment),
+            semconv.ServiceInstanceID(instanceID),
+        ),
+    )
 }
diff --git a/mock/s3/shared/server/service_bootstrap.go b/mock/s3/shared/server/service_bootstrap.go
@@ -1,14 +1,15 @@
 package server
 
 import (
-	"context"
-	"fmt"
-	"mocks3/shared/observability"
-	"net/http"
-	"os"
-	"os/signal"
-	"syscall"
-	"time"
+    "context"
+    "fmt"
+    "mocks3/shared/observability"
+    "net/http"
+    "net"
+    "os"
+    "os/signal"
+    "syscall"
+    "time"
 
 	"github.com/gin-gonic/gin"
 	"mocks3/shared/middleware/consul"
@@ -173,7 +174,7 @@ func (sb *ServiceBootstrap) setupObservability() error {
 
 // setupConsulRegistration 设置Consul服务注册
 func (sb *ServiceBootstrap) setupConsulRegistration() error {
-	ctx := context.Background()
+    ctx := context.Background()
 
 	// 检查配置是否支持Consul
 	consulConfig, ok := sb.Config.(ConsulServiceConfig)
@@ -190,19 +191,23 @@ func (sb *ServiceBootstrap) setupConsulRegistration() error {
 
 	sb.ConsulClient = consulClient
 
-	// 注册服务到Consul
-	// 使用hostname作为注册地址，而不是绑定地址"0.0.0.0"
-	var registerAddress string
-	if sb.Config.GetHost() == "0.0.0.0" {
-		// 如果绑定地址是0.0.0.0，使用hostname进行注册
-		hostname, err := os.Hostname()
-		if err != nil {
-			return fmt.Errorf("failed to get hostname for Consul registration: %w", err)
-		}
-		registerAddress = hostname
-	} else {
-		registerAddress = sb.Config.GetHost()
-	}
+    // 注册服务到Consul
+    // 优先使用可达的容器/主机实例IP地址进行注册，确保多实例下目标唯一
+    var registerAddress string
+    if sb.Config.GetHost() == "0.0.0.0" {
+        // 允许通过环境变量覆盖对外公布地址
+        if envAddr := os.Getenv("ADVERTISE_ADDR"); envAddr != "" {
+            registerAddress = envAddr
+        } else {
+            ip, err := detectAdvertiseAddr()
+            if err != nil {
+                return fmt.Errorf("failed to detect advertise address: %w", err)
+            }
+            registerAddress = ip
+        }
+    } else {
+        registerAddress = sb.Config.GetHost()
+    }
 
 	err = consul.RegisterService(ctx, consulClient,
 		sb.Config.GetServiceName(),
@@ -212,12 +217,69 @@ func (sb *ServiceBootstrap) setupConsulRegistration() error {
 		return fmt.Errorf("failed to register service with Consul: %w", err)
 	}
 
-	sb.Logger.Info(ctx, "Service registered with Consul successfully",
-		observability.String("consul_addr", consulConfig.GetConsulAddress()),
-		observability.String("service_name", sb.Config.GetServiceName()),
-		observability.String("register_address", registerAddress))
+    sb.Logger.Info(ctx, "Service registered with Consul successfully",
+        observability.String("consul_addr", consulConfig.GetConsulAddress()),
+        observability.String("service_name", sb.Config.GetServiceName()),
+        observability.String("register_address", registerAddress))
 
-	return nil
+    return nil
+}
+
+// detectAdvertiseAddr 自动探测一个非回环的IPv4地址，优先选择常见容器网卡
+func detectAdvertiseAddr() (string, error) {
+    // 优先尝试常见的容器网卡名称
+    preferredIfaces := []string{"eth0", "ens3", "ens4", "en0"}
+    for _, name := range preferredIfaces {
+        ifi, err := net.InterfaceByName(name)
+        if err == nil && (ifi.Flags&net.FlagUp) != 0 {
+            addrs, err := ifi.Addrs()
+            if err == nil {
+                if ip := firstIPv4(addrs); ip != "" {
+                    return ip, nil
+                }
+            }
+        }
+    }
+
+    // 回退：遍历所有网卡，取第一个非回环且Up的IPv4
+    ifaces, err := net.Interfaces()
+    if err != nil {
+        return "", err
+    }
+    for _, ifi := range ifaces {
+        if (ifi.Flags&net.FlagUp) == 0 || (ifi.Flags&net.FlagLoopback) != 0 {
+            continue
+        }
+        addrs, err := ifi.Addrs()
+        if err != nil {
+            continue
+        }
+        if ip := firstIPv4(addrs); ip != "" {
+            return ip, nil
+        }
+    }
+    return "", fmt.Errorf("no non-loopback IPv4 address found")
+}
+
+func firstIPv4(addrs []net.Addr) string {
+    for _, a := range addrs {
+        var ip net.IP
+        switch v := a.(type) {
+        case *net.IPNet:
+            ip = v.IP
+        case *net.IPAddr:
+            ip = v.IP
+        }
+        if ip == nil {
+            continue
+        }
+        ip4 := ip.To4()
+        if ip4 == nil || ip4.IsLoopback() {
+            continue
+        }
+        return ip4.String()
+    }
+    return ""
 }
 
 // setupErrorInjection 设置错误注入中间件
