add retry

Author: yincongcyincong

Dockerfile

@@ -53,22 +53,25 @@ WORKDIR /app
 
 # 安装运行时依赖 (证书 + nodejs + opus 运行库)
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends ca-certificates curl libopus0 && \
+    apt-get install -y --no-install-recommends ca-certificates curl libopus0 supervisor && \
     curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
     apt-get install -y --no-install-recommends nodejs && \
     rm -rf /var/lib/apt/lists/*
 
 ENV PATH="/usr/share/nodejs/corepack/shims:$PATH"
 
+COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
 # Create required directories
-RUN mkdir -p ./conf/i18n ./conf/mcp
+RUN mkdir -p ./conf/i18n ./conf/mcp ./conf/img/ ./adminui
 
 # Copy compiled Go application
 COPY --from=builder /app/MuseBot .
 COPY --from=builder /app/MuseBotAdmin .
 COPY --from=builder /app/conf/i18n/ ./conf/i18n/
 COPY --from=builder /app/conf/mcp/ ./conf/mcp/
 COPY --from=builder /app/conf/img/ ./conf/img/
+COPY --from=builder /app/admin/adminui/ ./conf/adminui/
 
 # Copy FFmpeg binaries
 COPY --from=ffmpeg-builder /usr/local/bin/ffmpeg /usr/local/bin/ffmpeg

MuseAdmin

@@ -286,14 +286,6 @@ func SoftDeleteBot(w http.ResponseWriter, r *http.Request) {
 		utils.Failure(ctx, w, r, param.CodeParamError, param.MsgParamError, err)
 		return
 	}
-	
-	botInfo, err := db.GetBotByID(idStr)
-	if err != nil {
-		logger.ErrorCtx(ctx, "get bot error", "reason", "not found", "id", idStr, "err", err)
-		utils.Failure(ctx, w, r, param.CodeDBQueryFail, param.MsgDBQueryFail, err)
-		return
-	}
-	
 	err = db.SoftDeleteBot(id)
 	if err != nil {
 		logger.ErrorCtx(ctx, "soft delete bot error", "reason", "db fail", "id", id, "err", err)
@@ -302,10 +294,6 @@ func SoftDeleteBot(w http.ResponseWriter, r *http.Request) {
 	}
 
 	checkpoint.BotMap.Delete(id)
-	
-	adminUtils.GetCrtClient(botInfo).Do(GetRequest(ctx, http.MethodGet,
-		strings.TrimSuffix(botInfo.Address, "/")+"/stop", bytes.NewBuffer(nil)))
-	
 	utils.Success(ctx, w, r, "bot deleted")
 }
 

admin/controller/bot.go

@@ -1,10 +1,16 @@
 package main
 
 import (
+	"bytes"
+	"context"
+	"embed"
 	"fmt"
+	"io/fs"
 	"net/http"
 	_ "net/http/pprof"
+	"time"
 
+	"github.com/google/uuid"
 	"github.com/yincongcyincong/MuseBot/admin/checkpoint"
 	"github.com/yincongcyincong/MuseBot/admin/conf"
 	"github.com/yincongcyincong/MuseBot/admin/controller"
@@ -77,3 +83,64 @@ func main() {
 		panic(err)
 	}
 }
+
+//go:embed adminui/*
+var staticFiles embed.FS
+
+func View() http.HandlerFunc {
+	distFS, _ := fs.Sub(staticFiles, "adminui")
+	
+	staticHandler := http.FileServer(http.FS(distFS))
+	
+	return func(w http.ResponseWriter, r *http.Request) {
+		if fileExists(distFS, r.URL.Path[1:]) {
+			staticHandler.ServeHTTP(w, r)
+			return
+		}
+		
+		fileBytes, err := fs.ReadFile(distFS, "index.html")
+		if err != nil {
+			http.Error(w, "index.html not found", http.StatusInternalServerError)
+			return
+		}
+		
+		reader := bytes.NewReader(fileBytes)
+		http.ServeContent(w, r, "index.html", time.Now(), reader)
+	}
+}
+
+func fileExists(fsys fs.FS, path string) bool {
+	f, err := fsys.Open(path)
+	if err != nil {
+		return false
+	}
+	defer f.Close()
+	info, err := f.Stat()
+	if err != nil || info.IsDir() {
+		return false
+	}
+	return true
+}
+
+func WithRequestContext(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		ctx := r.Context()
+		logID := uuid.New().String()
+		
+		isSSE := r.Header.Get("Accept") == "text/event-stream"
+		
+		if !isSSE {
+			var cancel context.CancelFunc
+			ctx, cancel = context.WithTimeout(ctx, 1*time.Minute)
+			defer cancel()
+		}
+		
+		// 通用的 context 值
+		ctx = context.WithValue(ctx, "log_id", logID)
+		ctx = context.WithValue(ctx, "start_time", time.Now())
+		r = r.WithContext(ctx)
+		
+		logger.InfoCtx(ctx, "request start", "path", r.URL.Path)
+		next.ServeHTTP(w, r)
+	})
+}

admin/main.go

@@ -73,6 +73,7 @@ type BaseConf struct {
 	Powered           *string `json:"powered"`
 	SendMcpRes        *bool   `json:"send_mcp_res"`
 	DefaultModel      *string `json:"default_model"`
+	LLMRetryTimes     *int    `json:"llm_retry_times"`
 
 	CrtFile *string `json:"crt_file"`
 	KeyFile *string `json:"key_file"`
@@ -151,6 +152,7 @@ func InitConf() {
 	BaseConfInfo.CaFile = flag.String("ca_file", "", "ca file")
 	BaseConfInfo.SendMcpRes = flag.Bool("send_mcp_res", false, "send mcp res")
 	BaseConfInfo.DefaultModel = flag.String("default_model", "", "default model")
+	BaseConfInfo.LLMRetryTimes = flag.Int("llm_retry_times", 3, "llm retry times")
 
 	adminUserIds := flag.String("admin_user_ids", "", "admin user ids")
 	allowedUserIds := flag.String("allowed_user_ids", "", "allowed user ids")
@@ -415,6 +417,10 @@ func InitConf() {
 		*BaseConfInfo.DefaultModel = os.Getenv("DEFAULT_MODEL")
 	}
 
+	if os.Getenv("LLM_RETRY_TIMES") != "" {
+		*BaseConfInfo.LLMRetryTimes, _ = strconv.Atoi(os.Getenv("LLM_RETRY_TIMES"))
+	}
+	
 	for _, userIdStr := range strings.Split(*allowedUserIds, ",") {
 		if userIdStr == "" {
 			continue
@@ -492,6 +498,7 @@ func InitConf() {
 	logger.Info("CONF", "ContextExpireTime", *BaseConfInfo.ContextExpireTime)
 	logger.Info("CONF", "SendMcpRes", *BaseConfInfo.SendMcpRes)
 	logger.Info("CONF", "DefaultModel", *BaseConfInfo.DefaultModel)
+	logger.Info("CONF", "LLMRetryTimes", *BaseConfInfo.LLMRetryTimes)
 
 	EnvAudioConf()
 	EnvRagConf()

admin/view.go

@@ -122,14 +122,22 @@ func GenerateAliyunImg(ctx context.Context, prompt string, imageContent []byte)
 	start := time.Now()
 	metrics.APIRequestCount.WithLabelValues(model).Inc()
 
-	resp, err := client.Do(req)
-	metrics.APIRequestDuration.WithLabelValues(model).Observe(time.Since(start).Seconds())
-	if err != nil {
-		logger.ErrorCtx(ctx, "create image fail", "err", err)
-		return "", 0, err
+	var resp *http.Response
+	var err error
+	for i := 0; i < *conf.BaseConfInfo.LLMRetryTimes; i++ {
+		resp, err = client.Do(req)
+		if err != nil {
+			logger.ErrorCtx(ctx, "create image fail", "err", err)
+			continue
+		}
+		break
+	}
+	if err != nil || resp == nil {
+		return "", 0, fmt.Errorf("request fail %v %v", err, resp)
 	}
-	defer resp.Body.Close()
 
+	defer resp.Body.Close()
+	metrics.APIRequestDuration.WithLabelValues(model).Observe(time.Since(start).Seconds())
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		logger.ErrorCtx(ctx, "read response fail", "err", err)
@@ -220,9 +228,18 @@ func GenerateAliyunVideo(ctx context.Context, prompt string, image []byte) (stri
 		metrics.APIRequestDuration.WithLabelValues(model).Observe(time.Since(start).Seconds())
 	}()
 
-	resp, err := client.Do(req)
-	if err != nil {
-		return "", 0, err
+	var resp *http.Response
+	for i := 0; i < *conf.BaseConfInfo.LLMRetryTimes; i++ {
+		resp, err = client.Do(req)
+		if err != nil {
+			logger.ErrorCtx(ctx, "create video fail", "err", err)
+			continue
+		}
+		break
+	}
+	
+	if err != nil || resp == nil {
+		return "", 0, fmt.Errorf("request fail %v %v", err, resp)
 	}
 	defer resp.Body.Close()
 
@@ -232,34 +249,34 @@ func GenerateAliyunVideo(ctx context.Context, prompt string, image []byte) (stri
 	}
 
 	var vr TaskStatusResponse
-	if err := json.Unmarshal(body, &vr); err != nil {
+	if err = json.Unmarshal(body, &vr); err != nil {
 		return "", 0, err
 	}
 
 	for i := 0; i < 100; i++ {
 		time.Sleep(5 * time.Second)
 
-		req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://dashscope.aliyuncs.com/api/v1/tasks/%s", vr.Output.TaskID), nil)
+		req, err = http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://dashscope.aliyuncs.com/api/v1/tasks/%s", vr.Output.TaskID), nil)
 		if err != nil {
 			return "", 0, err
 		}
 
 		req.Header.Set("Authorization", "Bearer "+*conf.BaseConfInfo.AliyunToken)
 		req.Header.Set("Content-Type", "application/json")
 
-		resp, err := client.Do(req)
+		resp, err = client.Do(req)
 		if err != nil {
 			return "", 0, err
 		}
 		defer resp.Body.Close()
 
-		body, err := io.ReadAll(resp.Body)
+		body, err = io.ReadAll(resp.Body)
 		if err != nil {
 			return "", 0, err
 		}
 
 		var vr videoResponse
-		if err := json.Unmarshal(body, &vr); err != nil {
+		if err = json.Unmarshal(body, &vr); err != nil {
 			return "", 0, err
 		}
 
@@ -330,12 +347,21 @@ func GenerateAliyunText(ctx context.Context, audioContent []byte) (string, int,
 	client := utils.GetLLMProxyClient()
 	start := time.Now()
 	metrics.APIRequestCount.WithLabelValues(recModel).Inc()
-	resp, err := client.Do(req)
 
-	metrics.APIRequestDuration.WithLabelValues(recModel).Observe(time.Since(start).Seconds())
-	if err != nil {
-		return "", 0, fmt.Errorf("request failed: %w", err)
+	var resp *http.Response
+	for i := 0; i < *conf.BaseConfInfo.LLMRetryTimes; i++ {
+		resp, err = client.Do(req)
+		if err != nil {
+			logger.ErrorCtx(ctx, "create video fail", "err", err)
+			continue
+		}
+		break
 	}
+	
+	if err != nil || resp == nil {
+		return "", 0, fmt.Errorf("request fail %v %v", err, resp)
+	}
+	metrics.APIRequestDuration.WithLabelValues(recModel).Observe(time.Since(start).Seconds())
 	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
@@ -405,11 +431,22 @@ func AliyunTTS(ctx context.Context, text, encoding string) ([]byte, int, int, er
 	client := utils.GetLLMProxyClient()
 	start := time.Now()
 	metrics.APIRequestCount.WithLabelValues(model).Inc()
-	resp, err := client.Do(req)
-	metrics.APIRequestDuration.WithLabelValues(model).Observe(time.Since(start).Seconds())
-	if err != nil {
-		return nil, 0, 0, fmt.Errorf("request error: %w", err)
+	
+	var resp *http.Response
+	for i := 0; i < *conf.BaseConfInfo.LLMRetryTimes; i++ {
+		resp, err = client.Do(req)
+		if err != nil {
+			logger.ErrorCtx(ctx, "create video fail", "err", err)
+			continue
+		}
+		break
 	}
+	
+	if err != nil || resp == nil {
+		return nil, 0, 0, fmt.Errorf("request fail %v %v", err, resp)
+	}
+	
+	metrics.APIRequestDuration.WithLabelValues(model).Observe(time.Since(start).Seconds())
 	defer resp.Body.Close()
 
 	respData, err := io.ReadAll(resp.Body)