feat(config): 添加从D1拉取配置功能并优化同步机制

新增从D1数据库拉取配置的功能，包括后端API接口、前端UI按钮和配置同步逻辑。同时优化了D1同步机制，移除了定时同步循环，改为按需同步。添加了路径统计数据的同步功能。

Author: woodchen-ink

cloudflare-worker/package.json

@@ -8,7 +8,8 @@
     "deploy": "wrangler deploy",
     "tail": "wrangler tail",
     "d1:create": "wrangler d1 create proxy-go-data",
-    "d1:migrations": "wrangler d1 migrations apply proxy-go-data"
+    "d1:migrations": "wrangler d1 migrations apply proxy-go-data",
+    "d1:remote": "wrangler d1 migrations apply proxy-go-data --remote"
   },
   "keywords": [],
   "author": "",

internal/handler/config.go

@@ -26,6 +26,8 @@ func (h *ConfigHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		h.handleGetConfig(w, r)
 	case "/admin/api/config/save":
 		h.handleSaveConfig(w, r)
+	case "/admin/api/config/pull":
+		h.handlePullConfig(w, r)
 	default:
 		http.NotFound(w, r)
 	}
@@ -68,3 +70,21 @@ func (h *ConfigHandler) handleSaveConfig(w http.ResponseWriter, r *http.Request)
 	w.Write([]byte(`{"message": "配置已更新并生效"}`))
 }
 
+// handlePullConfig 处理从 D1 拉取配置请求
+func (h *ConfigHandler) handlePullConfig(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "方法不允许", http.StatusMethodNotAllowed)
+		return
+	}
+
+	// 从 D1 拉取配置
+	configData, err := h.configService.PullConfigFromD1()
+	if err != nil {
+		http.Error(w, "从 D1 拉取配置失败: "+err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(configData)
+}
+

internal/metrics/persistence.go

@@ -109,6 +109,14 @@ func (ms *MetricsStorage) SaveMetrics() error {
 		}
 	}
 
+	// 获取路径统计
+	pathStats := ms.getPathStatsArray()
+	if len(pathStats) > 0 {
+		if err := sync.SavePathStats(ctx, pathStats); err != nil {
+			log.Printf("[MetricsStorage] 保存路径统计失败: %v", err)
+		}
+	}
+
 	// 更新最后保存时间
 	ms.mutex.Lock()
 	ms.lastSaveTime = time.Now()
@@ -228,3 +236,36 @@ func (ms *MetricsStorage) getLatencyDistributionMap() map[string]int64 {
 		"gt1s":       atomic.LoadInt64(&ms.collector.latencyBuckets.gt1s),
 	}
 }
+
+// getPathStatsArray 获取路径统计数组（转换为 sync.PathStat 格式）
+func (ms *MetricsStorage) getPathStatsArray() []sync.PathStat {
+	pathMetrics := ms.collector.GetPathStats()
+	if len(pathMetrics) == 0 {
+		return nil
+	}
+
+	now := time.Now().UnixMilli()
+	result := make([]sync.PathStat, 0, len(pathMetrics))
+
+	for _, pm := range pathMetrics {
+		result = append(result, sync.PathStat{
+			Path:             pm.Path,
+			RequestCount:     pm.RequestCount,
+			ErrorCount:       pm.ErrorCount,
+			BytesTransferred: pm.BytesTransferred,
+			Status2xx:        pm.Status2xx,
+			Status3xx:        pm.Status3xx,
+			Status4xx:        pm.Status4xx,
+			Status5xx:        pm.Status5xx,
+			CacheHits:        pm.CacheHits,
+			CacheMisses:      pm.CacheMisses,
+			CacheHitRate:     pm.CacheHitRate,
+			BytesSaved:       pm.BytesSaved,
+			AvgLatency:       pm.AvgLatency,
+			LastAccessTime:   pm.LastAccessTime,
+			UpdatedAt:        now,
+		})
+	}
+
+	return result
+}

internal/service/config_service.go

@@ -1,10 +1,13 @@
 package service
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"net/url"
 	"proxy-go/internal/config"
+	"proxy-go/pkg/sync"
+	"time"
 )
 
 type ConfigService struct {
@@ -52,6 +55,53 @@ func (s *ConfigService) SaveConfig(newConfig *config.Config) error {
 	return nil
 }
 
+// PullConfigFromD1 从 D1 拉取配置并更新本地
+func (s *ConfigService) PullConfigFromD1() ([]byte, error) {
+	// 检查 D1 同步是否启用
+	if !sync.IsEnabled() {
+		return nil, fmt.Errorf("D1 同步未启用")
+	}
+
+	// 从 D1 下载配置
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	configData, usedLocal, err := sync.DownloadConfigOnly(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("从 D1 下载配置失败: %v", err)
+	}
+
+	if usedLocal {
+		return nil, fmt.Errorf("D1 配置为空，无法拉取")
+	}
+
+	// 将 map[string]any 转换为 Config 结构
+	configJson, err := json.Marshal(configData)
+	if err != nil {
+		return nil, fmt.Errorf("序列化配置失败: %v", err)
+	}
+
+	var newConfig config.Config
+	if err := json.Unmarshal(configJson, &newConfig); err != nil {
+		return nil, fmt.Errorf("解析配置失败: %v", err)
+	}
+
+	// 验证配置
+	if err := s.validateConfig(&newConfig); err != nil {
+		return nil, fmt.Errorf("配置验证失败: %v", err)
+	}
+
+	// 更新本地配置
+	if err := s.configManager.UpdateConfig(&newConfig); err != nil {
+		return nil, fmt.Errorf("更新本地配置失败: %v", err)
+	}
+
+	fmt.Printf("[Config] 从 D1 拉取配置成功: %d 个路径映射\n", len(newConfig.MAP))
+
+	// 返回配置数据
+	return configJson, nil
+}
+
 // validateConfig 验证配置
 func (s *ConfigService) validateConfig(cfg *config.Config) error {
 	if cfg == nil {

pkg/sync/d1_manager.go

@@ -56,8 +56,10 @@ func (m *D1Manager) Start(ctx context.Context) error {
 
 	log.Printf("[D1Sync] Skipping initial sync at startup (already downloaded config)")
 
-	// 启动定时同步（仅同步config.json）
-	go m.syncLoop(ctx)
+	// 注意: 不再需要定时同步循环
+	// - Config: 通过 ConfigUpdateCallback 在修改时立即同步
+	// - Path Stats/Metrics: 由 MetricsStorage 每 30 分钟自动同步
+	// - Banned IPs: 在封禁/解封时立即同步
 
 	return nil
 }
@@ -203,31 +205,6 @@ func (m *D1Manager) GetEventChannel() <-chan SyncEvent {
 	return m.eventChan
 }
 
-// syncLoop 同步循环
-func (m *D1Manager) syncLoop(ctx context.Context) {
-	ticker := time.NewTicker(10 * time.Minute)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-ticker.C:
-			if err := m.SyncNow(ctx); err != nil {
-				log.Printf("[D1Sync] Scheduled sync failed: %v", err)
-				m.sendEvent(SyncEvent{
-					Type:      SyncEventError,
-					Timestamp: time.Now(),
-					Message:   "Scheduled sync failed",
-					Error:     err,
-				})
-			}
-		case <-m.stopChan:
-			return
-		case <-ctx.Done():
-			return
-		}
-	}
-}
-
 // syncConfigToD1 同步配置到D1
 func (m *D1Manager) syncConfigToD1(ctx context.Context) error {
 	config, err := m.configLoader.LoadConfig()
@@ -318,6 +295,20 @@ func (m *D1Manager) LoadLatencyDistribution(ctx context.Context) (map[string]int
 	return result, nil
 }
 
+// SavePathStats 保存路径统计到 D1
+func (m *D1Manager) SavePathStats(ctx context.Context, stats []PathStat) error {
+	if len(stats) == 0 {
+		return nil
+	}
+
+	if err := m.storage.BatchUpsertPathStats(ctx, stats); err != nil {
+		return fmt.Errorf("failed to save path stats: %w", err)
+	}
+
+	log.Printf("[D1Sync] Saved %d path stats", len(stats))
+	return nil
+}
+
 // downloadConfigWithFallback 下载配置，如果远程不存在则上传本地配置
 // 返回: 远程配置数据（如果存在），是否使用了本地配置，错误
 func (m *D1Manager) downloadConfigWithFallback(ctx context.Context) (map[string]any, bool, error) {

pkg/sync/service.go

@@ -271,3 +271,15 @@ func LoadPathStats(ctx context.Context) ([]PathStat, error) {
 	// 调用 D1Client 的 GetPathStats 方法
 	return globalSyncService.manager.storage.GetPathStats(ctx, "")
 }
+
+// SavePathStats 保存路径统计到 D1
+func SavePathStats(ctx context.Context, stats []PathStat) error {
+	globalSyncMutex.RLock()
+	defer globalSyncMutex.RUnlock()
+
+	if globalSyncService == nil || !globalSyncService.isEnabled {
+		return nil
+	}
+
+	return globalSyncService.manager.SavePathStats(ctx, stats)
+}

web/app/dashboard/config/page.tsx

@@ -6,7 +6,7 @@ import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"
 import { useToast } from "@/components/ui/use-toast"
 import { useRouter } from "next/navigation"
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"
-import { Download, Upload } from "lucide-react"
+import { Download, RefreshCw } from "lucide-react"
 import {
   AlertDialog,
   AlertDialogAction,
@@ -624,84 +624,34 @@ export default function ConfigPage() {
     URL.revokeObjectURL(url)
   }
 
-  const importConfig = (event: React.ChangeEvent<HTMLInputElement>) => {
-    const file = event.target.files?.[0]
-    if (!file) return
-
-    const reader = new FileReader()
-    reader.onload = (e) => {
-      try {
-        const content = e.target?.result as string
-        const newConfig = JSON.parse(content)
-
-        // 验证配置结构
-        if (!newConfig.MAP || typeof newConfig.MAP !== 'object') {
-          throw new Error('配置文件缺少 MAP 字段或格式不正确')
-        }
-
-        if (!newConfig.Compression || 
-            typeof newConfig.Compression !== 'object' ||
-            !newConfig.Compression.Gzip ||
-            !newConfig.Compression.Brotli) {
-          throw new Error('配置文件压缩设置格式不正确')
-        }
+  const pullConfigFromD1 = async () => {
+    try {
+      const response = await fetch('/admin/api/config/pull', {
+        method: 'POST',
+      })
 
-        // 如果没有安全配置，添加默认配置
-        if (!newConfig.Security) {
-          newConfig.Security = {
-            IPBan: {
-              Enabled: false,
-              ErrorThreshold: 10,
-              WindowMinutes: 5,
-              BanDurationMinutes: 5,
-              CleanupIntervalMinutes: 1
-            }
-          }
-        }
+      if (!response.ok) {
+        const errorText = await response.text()
+        throw new Error(errorText || '从 D1 拉取配置失败')
+      }
 
-        // 验证路径映射
-        for (const [path, target] of Object.entries(newConfig.MAP)) {
-          if (!path.startsWith('/')) {
-            throw new Error(`路径 ${path} 必须以/开头`)
-          }
+      const pulledConfig = await response.json()
 
-          if (typeof target === 'string') {
-            try {
-              new URL(target)
-            } catch {
-              throw new Error(`路径 ${path} 的目标URL格式不正确`)
-            }
-          } else if (target && typeof target === 'object') {
-            const mapping = target as PathMapping
-            if (!mapping.DefaultTarget || typeof mapping.DefaultTarget !== 'string') {
-              throw new Error(`路径 ${path} 的默认目标格式不正确`)
-            }
-            try {
-              new URL(mapping.DefaultTarget)
-            } catch {
-              throw new Error(`路径 ${path} 的默认目标URL格式不正确`)
-            }
-          } else {
-            throw new Error(`路径 ${path} 的目标格式不正确`)
-          }
-        }
+      // 使用 setConfig 而不是 updateConfig，因为拉取的配置已经在服务端更新过了
+      isConfigFromApiRef.current = true
+      setConfig(pulledConfig)
 
-        // 使用setConfig而不是updateConfig，因为导入的配置不应触发自动保存
-        isConfigFromApiRef.current = true
-        setConfig(newConfig)
-        toast({
-          title: "成功",
-          description: "配置已导入",
-        })
-      } catch (error) {
-        toast({
-          title: "错误",
-          description: error instanceof Error ? error.message : "配置文件格式错误",
-          variant: "destructive",
-        })
-      }
+      toast({
+        title: "成功",
+        description: "已从 D1 拉取最新配置",
+      })
+    } catch (error) {
+      toast({
+        title: "错误",
+        description: error instanceof Error ? error.message : "从 D1 拉取配置失败",
+        variant: "destructive",
+      })
     }
-    reader.readAsText(file)
   }
 
   const handleEditPath = (path: string, target: PathMapping | string) => {
@@ -986,18 +936,10 @@ export default function ConfigPage() {
               <Download className="w-4 h-4 mr-2" />
               导出配置
             </Button>
-            <label>
-              <Button variant="outline" className="cursor-pointer">
-                <Upload className="w-4 h-4 mr-2" />
-                导入配置
-              </Button>
-              <input
-                type="file"
-                className="hidden"
-                accept=".json"
-                onChange={importConfig}
-              />
-            </label>
+            <Button onClick={pullConfigFromD1} variant="outline">
+              <RefreshCw className="w-4 h-4 mr-2" />
+              从 D1 拉取配置
+            </Button>
             {saving && (
               <div className="flex items-center text-sm text-muted-foreground">
                 <span className="animate-pulse mr-2">●</span>