update backup strategy

Author: sunwei

internal/application/backup.go

@@ -402,53 +402,42 @@ func (bs *BackupScheduler) backupCouchDB(tempDir string, info *CouchDBBackupInfo
 	return nil
 }
 
-// backupCouchDBDatabase 备份单个 CouchDB 数据库
+// backupCouchDBDatabase 备份单个 CouchDB 数据库（优化内存使用）
 func (bs *BackupScheduler) backupCouchDBDatabase(dir string, dbName string) (DatabaseBackupInfo, error) {
 	start := time.Now()
-
+	
 	// 获取数据库信息
 	dbInfo, err := bs.getCouchDBDatabaseInfo(dbName)
 	if err != nil {
 		return DatabaseBackupInfo{}, err
 	}
-
-	bs.log.Printf("Backing up: %s (%d docs)", dbName, dbInfo["doc_count"])
-
-	// 导出所有文档
-	docs, err := bs.exportCouchDBDocs(dbName)
-	if err != nil {
-		return DatabaseBackupInfo{}, err
-	}
-
-	// 构建备份数据
-	backup := map[string]interface{}{
-		"db_name":    dbName,
-		"update_seq": dbInfo["update_seq"],
-		"doc_count":  dbInfo["doc_count"],
-		"timestamp":  time.Now().Format(time.RFC3339),
-		"docs":       docs,
+	
+	// 修复类型转换问题
+	docCount := 0
+	if dc, ok := dbInfo["doc_count"].(float64); ok {
+		docCount = int(dc)
 	}
-
-	// 保存为 JSON 文件
+	
+	bs.log.Printf("Backing up: %s (%d docs)", dbName, docCount)
+	
+	// 直接流式写入文件，避免大量文档占用内存
 	outputPath := filepath.Join(dir, dbName+".json")
-	data, err := json.Marshal(backup)
+	
+	// 使用流式导出（避免 OOM）
+	fileSize, err := bs.streamExportCouchDBDocs(dbName, dbInfo, outputPath)
 	if err != nil {
-		return DatabaseBackupInfo{}, fmt.Errorf("failed to marshal backup: %w", err)
-	}
-
-	if err := os.WriteFile(outputPath, data, 0644); err != nil {
-		return DatabaseBackupInfo{}, fmt.Errorf("failed to write backup file: %w", err)
+		return DatabaseBackupInfo{}, err
 	}
-
+	
 	info := DatabaseBackupInfo{
-		DocCount:   int(dbInfo["doc_count"].(float64)),
+		DocCount:   docCount,
 		UpdateSeq:  fmt.Sprintf("%v", dbInfo["update_seq"]),
-		Size:       int64(len(data)),
+		Size:       fileSize,
 		BackupTime: time.Since(start).Milliseconds(),
 	}
-
-	bs.log.Printf("✓ %s backed up: %.2f KB", dbName, float64(info.Size)/1024)
-
+	
+	bs.log.Printf("✓ %s backed up: %.2f KB in %d ms", dbName, float64(info.Size)/1024, info.BackupTime)
+	
 	return info, nil
 }
 

internal/application/backup_helpers.go

@@ -643,42 +643,165 @@ func (bs *BackupScheduler) getCouchDBDatabaseInfo(dbName string) (map[string]int
 // exportCouchDBDocs 导出数据库所有文档
 func (bs *BackupScheduler) exportCouchDBDocs(dbName string) ([]interface{}, error) {
 	url := fmt.Sprintf("%s/%s/_all_docs?include_docs=true", bs.couchdbClient.GetURL(), dbName)
-
+	
 	req, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {
 		return nil, err
 	}
-
+	
 	bs.setBasicAuth(req)
-
+	
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return nil, err
 	}
 	defer resp.Body.Close()
-
+	
 	if resp.StatusCode != http.StatusOK {
 		body, _ := io.ReadAll(resp.Body)
 		return nil, fmt.Errorf("unexpected status code: %d, body: %s", resp.StatusCode, string(body))
 	}
-
+	
 	var result struct {
 		Rows []struct {
 			Doc interface{} `json:"doc"`
 		} `json:"rows"`
 	}
-
+	
 	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
 		return nil, err
 	}
-
+	
 	docs := make([]interface{}, 0, len(result.Rows))
 	for _, row := range result.Rows {
 		if row.Doc != nil {
 			docs = append(docs, row.Doc)
 		}
 	}
+	
+	return docs, nil
+}
 
+// streamExportCouchDBDocs 流式导出数据库文档（分批处理，避免 OOM）
+func (bs *BackupScheduler) streamExportCouchDBDocs(dbName string, dbInfo map[string]interface{}, outputPath string) (int64, error) {
+	// 创建输出文件
+	file, err := os.Create(outputPath)
+	if err != nil {
+		return 0, fmt.Errorf("failed to create output file: %w", err)
+	}
+	defer file.Close()
+	
+	// 写入文件头部（元数据）
+	docCount := 0
+	if dc, ok := dbInfo["doc_count"].(float64); ok {
+		docCount = int(dc)
+	}
+	
+	header := fmt.Sprintf(`{"db_name":"%s","update_seq":"%v","doc_count":%d,"timestamp":"%s","docs":[`,
+		dbName,
+		dbInfo["update_seq"],
+		docCount,
+		time.Now().Format(time.RFC3339),
+	)
+	
+	if _, err := file.WriteString(header); err != nil {
+		return 0, err
+	}
+	
+	// 分批导出文档，避免 OOM
+	// 每批处理 1000 个文档
+	batchSize := 1000
+	totalExported := 0
+	
+	for skip := 0; skip < docCount; skip += batchSize {
+		// 获取一批文档
+		limit := batchSize
+		if skip+limit > docCount {
+			limit = docCount - skip
+		}
+		
+		docs, err := bs.exportCouchDBDocsBatch(dbName, skip, limit)
+		if err != nil {
+			return 0, fmt.Errorf("failed to export batch at skip=%d: %w", skip, err)
+		}
+		
+		// 写入文档
+		for i, doc := range docs {
+			if totalExported > 0 || i > 0 {
+				file.WriteString(",")
+			}
+			
+			// 将文档序列化为 JSON 并写入
+			docJSON, err := json.Marshal(doc)
+			if err != nil {
+				bs.log.Warnf("Failed to marshal doc in %s: %v", dbName, err)
+				continue
+			}
+			
+			file.Write(docJSON)
+			totalExported++
+		}
+		
+		// 输出进度（每 10 批）
+		if (skip/batchSize)%10 == 0 && skip > 0 {
+			progress := float64(totalExported) / float64(docCount) * 100
+			bs.log.Printf("  Progress: %d/%d docs (%.1f%%)", totalExported, docCount, progress)
+		}
+	}
+	
+	// 写入文件尾部
+	file.WriteString("]}")
+	
+	// 获取文件大小
+	stat, _ := os.Stat(outputPath)
+	
+	if totalExported != docCount {
+		bs.log.Warnf("Expected %d docs, exported %d docs for %s", docCount, totalExported, dbName)
+	}
+	
+	return stat.Size(), nil
+}
+
+// exportCouchDBDocsBatch 分批导出文档
+func (bs *BackupScheduler) exportCouchDBDocsBatch(dbName string, skip, limit int) ([]interface{}, error) {
+	url := fmt.Sprintf("%s/%s/_all_docs?include_docs=true&skip=%d&limit=%d", 
+		bs.couchdbClient.GetURL(), dbName, skip, limit)
+	
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	
+	bs.setBasicAuth(req)
+	
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		return nil, fmt.Errorf("unexpected status code: %d, body: %s", resp.StatusCode, string(body))
+	}
+	
+	var result struct {
+		Rows []struct {
+			Doc interface{} `json:"doc"`
+		} `json:"rows"`
+	}
+	
+	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+		return nil, err
+	}
+	
+	docs := make([]interface{}, 0, len(result.Rows))
+	for _, row := range result.Rows {
+		if row.Doc != nil {
+			docs = append(docs, row.Doc)
+		}
+	}
+	
 	return docs, nil
 }
 

internal/interfaces/cli/vercurr.go

@@ -3,6 +3,6 @@ package cli
 var CurrentVersion = Version{
     Major:      0,
     Minor:      2,
-    PatchLevel: 37,
+    PatchLevel: 40,
     Suffix:     "",
 }

manifest.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.2.37",
+  "version": "0.2.40",
   "name": "Hugoverse",
   "description": "Headless CMS for Hugo",
   "author": "sunwei",