fix: memory leak issue, complete logs, incorrect gap time of recaps (#184)

* fix: memory leak issue, complete logs, incorrect gap time of recaps

Signed-off-by: Neko Ayaka <neko@ayaka.moe>

Author: nekomeowww

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/nekomeowww/fo v1.2.1
 	github.com/nekomeowww/go-pinecone v0.1.0
 	github.com/nekomeowww/timecapsule/v2 v2.2.0
-	github.com/nekomeowww/xo v1.2.0
+	github.com/nekomeowww/xo v1.3.0
 	github.com/pkoukk/tiktoken-go v0.1.6
 	github.com/redis/rueidis v1.0.23
 	github.com/samber/lo v1.39.0

go.sum

@@ -152,8 +152,8 @@ github.com/nekomeowww/go-pinecone v0.1.0 h1:byYmyHQJ4velNUeECvNuCN6KbXGuyVjSFlvO
 github.com/nekomeowww/go-pinecone v0.1.0/go.mod h1:p8I6F7G7fSrfaa9k3HpsWNxf2y+W1wqn8tw0jsV9e1s=
 github.com/nekomeowww/timecapsule/v2 v2.2.0 h1:5hdcoq7uyIdQCs9ufIg1yuDnfhzsk1bb0dyP2Mmb+cI=
 github.com/nekomeowww/timecapsule/v2 v2.2.0/go.mod h1:1rr05h26cZmOaNUw/tSByKfPdomx0Fw9tWxlu+4HQlU=
-github.com/nekomeowww/xo v1.2.0 h1:JNH2nM/nRBSTs0azX97O4NAcNhmP3EXSRh9lGlhR628=
-github.com/nekomeowww/xo v1.2.0/go.mod h1:shuoU7EL5OliCEpgzF9I7xDSJMrqmr1B3pdrBaw+pDw=
+github.com/nekomeowww/xo v1.3.0 h1:9HmI4enlu3TiBuFeT4NKAOJW0FQCjsp6R9qLk6u6iu8=
+github.com/nekomeowww/xo v1.3.0/go.mod h1:vw5BQ33eEpNO2ZwlIO2xnP25mvU2LAfT3CojIcZ+YHg=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/onsi/ginkgo/v2 v2.13.2 h1:Bi2gGVkfn6gQcjNjZJVO8Gf0FHzMPf2phUei9tejVMs=

internal/models/chathistories/chat_histories.go

@@ -335,10 +335,18 @@ func (m *Model) FindLastOneHourChatHistories(chatID int64) ([]*ent.ChatHistories
 	return m.FindChatHistoriesByTimeBefore(chatID, time.Hour)
 }
 
-func (m *Model) FindLastSixHourChatHistories(chatID int64) ([]*ent.ChatHistories, error) {
+func (m *Model) FindLast6HourChatHistories(chatID int64) ([]*ent.ChatHistories, error) {
 	return m.FindChatHistoriesByTimeBefore(chatID, 6*time.Hour)
 }
 
+func (m *Model) FindLast8HourChatHistories(chatID int64) ([]*ent.ChatHistories, error) {
+	return m.FindChatHistoriesByTimeBefore(chatID, 8*time.Hour)
+}
+
+func (m *Model) FindLast12HourChatHistories(chatID int64) ([]*ent.ChatHistories, error) {
+	return m.FindChatHistoriesByTimeBefore(chatID, 12*time.Hour)
+}
+
 func (m *Model) FindChatHistoriesByTimeBefore(chatID int64, before time.Duration) ([]*ent.ChatHistories, error) {
 	m.logger.Info("querying chat histories", zap.Int64("chat_id", chatID))
 

internal/models/smr/smr.go

@@ -26,6 +26,7 @@ import (
 	"github.com/nekomeowww/insights-bot/pkg/linkprev"
 	"github.com/nekomeowww/insights-bot/pkg/logger"
 	"github.com/nekomeowww/insights-bot/pkg/types/bot"
+	"github.com/nekomeowww/xo"
 )
 
 type NewModelParams struct {
@@ -168,14 +169,21 @@ func (m *Model) extractContentFromURL(ctx context.Context, urlString string) (*r
 	}
 
 	dumpBuffer := new(bytes.Buffer)
-	defer dumpBuffer.Reset()
+	defer func() {
+		dumpBuffer.Reset()
+		dumpBuffer = nil
+	}()
 
-	resp, err := m.req.
+	request := m.req.
 		R().
 		EnableDumpTo(dumpBuffer).
 		DisableAutoReadResponse().
-		SetContext(ctx).
-		Get(parsedURL.String())
+		SetContext(ctx)
+	defer func() {
+		request.EnableDumpTo(xo.NewNopIoWriter())
+	}()
+
+	resp, err := request.Get(parsedURL.String())
 	if err != nil {
 		return nil, fmt.Errorf("failed to get url %s, %w: %v", parsedURL.String(), ErrNetworkError, err)
 	}

internal/services/autorecap/autorecap.go

@@ -152,23 +152,61 @@ func (m *AutoRecapService) sendChatHistoriesRecapTimeCapsuleHandler(
 }
 
 func (m *AutoRecapService) summarize(chatID int64, options *ent.TelegramChatRecapsOptions, subscribers []*ent.TelegramChatAutoRecapsSubscribers) {
-	m.logger.Info("generating chat histories recap for chat", zap.Int64("chat_id", chatID))
+	m.logger.Info("generating chat histories recap for chat",
+		zap.Int64("chat_id", chatID),
+		zap.String("module", "autorecap"),
+		zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
+	)
 
 	chat, err := m.botService.GetChat(tgbotapi.ChatInfoConfig{
 		ChatConfig: tgbotapi.ChatConfig{
 			ChatID: chatID,
 		},
 	})
 	if err != nil {
-		m.logger.Error("failed to get chat", zap.Error(err), zap.Int64("chat_id", chatID))
+		m.logger.Error("failed to get chat",
+			zap.Int64("chat_id", chatID),
+			zap.String("module", "autorecap"),
+			zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
+			zap.Error(err),
+		)
+
 		return
 	}
 
 	chatType := telegram.ChatType(chat.Type)
 
-	histories, err := m.chathistories.FindLastSixHourChatHistories(chatID)
+	mAutoRecapRatesPerDayHours := map[int]int{
+		4: 6,
+		3: 8,
+		2: 12,
+	}
+
+	hours, ok := mAutoRecapRatesPerDayHours[options.AutoRecapRatesPerDay]
+	if !ok {
+		hours = 6
+	}
+
+	mFindChatHistoriesHoursBefore := map[int]func(chatID int64) ([]*ent.ChatHistories, error){
+		6:  m.chathistories.FindLast6HourChatHistories,
+		8:  m.chathistories.FindLast8HourChatHistories,
+		12: m.chathistories.FindLast12HourChatHistories,
+	}
+
+	findChatHistories, ok := mFindChatHistoriesHoursBefore[hours]
+	if !ok {
+		findChatHistories = m.chathistories.FindLast6HourChatHistories
+	}
+
+	histories, err := findChatHistories(chatID)
 	if err != nil {
-		m.logger.Error("failed to find last six hour chat histories", zap.Error(err), zap.Int64("chat_id", chatID))
+		m.logger.Error(fmt.Sprintf("failed to find last %d hour chat histories", hours),
+			zap.Int64("chat_id", chatID),
+			zap.String("module", "autorecap"),
+			zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
+			zap.Error(err),
+		)
+
 		return
 	}
 	if len(histories) <= 5 {
@@ -180,25 +218,49 @@ func (m *AutoRecapService) summarize(chatID int64, options *ent.TelegramChatReca
 
 	logID, summarizations, err := m.chathistories.SummarizeChatHistories(chatID, chatType, histories)
 	if err != nil {
-		m.logger.Error("failed to summarize last six hour chat histories", zap.Error(err), zap.Int64("chat_id", chatID))
+		m.logger.Error("failed to summarize last six hour chat histories",
+			zap.Int64("chat_id", chatID),
+			zap.String("module", "autorecap"),
+			zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
+			zap.Error(err),
+		)
+
 		return
 	}
 
 	counts, err := m.chathistories.FindFeedbackRecapsReactionCountsForChatIDAndLogID(chatID, logID)
 	if err != nil {
-		m.logger.Error("failed to find feedback recaps votes for chat", zap.Error(err), zap.Int64("chat_id", chatID))
+		m.logger.Error("failed to find feedback recaps votes for chat",
+			zap.Int64("chat_id", chatID),
+			zap.String("module", "autorecap"),
+			zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
+			zap.Error(err),
+		)
+
 		return
 	}
 
 	inlineKeyboardMarkup, err := m.chathistories.NewVoteRecapInlineKeyboardMarkup(m.botService.Bot(), chatID, logID, counts.UpVotes, counts.DownVotes, counts.Lmao)
 	if err != nil {
-		m.logger.Error("failed to create vote recap inline keyboard markup", zap.Error(err), zap.Int64("chat_id", chatID), zap.String("log_id", logID.String()))
+		m.logger.Error("failed to create vote recap inline keyboard markup",
+			zap.Int64("chat_id", chatID),
+			zap.String("log_id", logID.String()),
+			zap.String("module", "autorecap"),
+			zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
+			zap.Error(err),
+		)
+
 		return
 	}
 
 	summarizations = lo.Filter(summarizations, func(item string, _ int) bool { return item != "" })
 	if len(summarizations) == 0 {
-		m.logger.Warn("summarization is empty", zap.Int64("chat_id", chatID))
+		m.logger.Warn("summarization is empty",
+			zap.Int64("chat_id", chatID),
+			zap.String("module", "autorecap"),
+			zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
+		)
+
 		return
 	}
 
@@ -245,6 +307,8 @@ func (m *AutoRecapService) summarize(chatID int64, options *ent.TelegramChatReca
 				zap.String("status", member.Status),
 				zap.Int64("chat_id", chatID),
 				zap.Int64("user_id", subscriber.UserID),
+				zap.String("module", "autorecap"),
+				zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
 			)
 
 			_, _, err := lo.AttemptWithDelay(1000, time.Minute, func(iter int, _ time.Duration) error {
@@ -257,6 +321,8 @@ func (m *AutoRecapService) summarize(chatID int64, options *ent.TelegramChatReca
 						zap.Int64("user_id", subscriber.UserID),
 						zap.Int("iter", iter),
 						zap.Int("max_iter", 100),
+						zap.String("module", "autorecap"),
+						zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
 					)
 
 					return err
@@ -265,15 +331,24 @@ func (m *AutoRecapService) summarize(chatID int64, options *ent.TelegramChatReca
 				return nil
 			})
 			if err != nil {
-				m.logger.Error("failed to unsubscribe to auto recaps", zap.Error(err), zap.Int64("chat_id", chatID))
+				m.logger.Error("failed to unsubscribe to auto recaps",
+					zap.Int64("chat_id", chatID),
+					zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
+					zap.Error(err),
+				)
 			}
 
 			msg := tgbotapi.NewMessage(subscriber.UserID, fmt.Sprintf("由于您已不再是 <b>%s</b> 的成员，因此已自动帮您取消了您所订阅的聊天记录回顾。", tgbot.EscapeHTMLSymbols(chatTitle)))
 			msg.ParseMode = tgbotapi.ModeHTML
 
 			_, err = m.botService.Send(msg)
 			if err != nil {
-				m.logger.Error("failed to send the auto un-subscription message", zap.Error(err), zap.Int64("user_id", subscriber.UserID), zap.Int64("chat_id", chatID))
+				m.logger.Error("failed to send the auto un-subscription message",
+					zap.Int64("user_id", subscriber.UserID),
+					zap.Int64("chat_id", chatID),
+					zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
+					zap.Error(err),
+				)
 			}
 
 			continue
@@ -313,7 +388,12 @@ func (m *AutoRecapService) summarize(chatID int64, options *ent.TelegramChatReca
 
 				inlineKeyboardMarkup, err := m.chathistories.NewVoteRecapWithUnsubscribeInlineKeyboardMarkup(m.botService.Bot(), chatID, chatTitle, targetChat.chatID, logID, counts.UpVotes, counts.DownVotes, counts.Lmao)
 				if err != nil {
-					m.logger.Error("failed to assign callback query data", zap.Error(err), zap.Int64("chat_id", chatID))
+					m.logger.Error("failed to assign callback query data",
+						zap.Int64("chat_id", chatID),
+						zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
+						zap.Error(err),
+					)
+
 					continue
 				}
 
@@ -325,7 +405,11 @@ func (m *AutoRecapService) summarize(chatID int64, options *ent.TelegramChatReca
 
 			_, err = m.botService.Send(msg)
 			if err != nil {
-				m.logger.Error("failed to send chat histories recap", zap.Error(err), zap.Int64("chat_id", chatID))
+				m.logger.Error("failed to send chat histories recap",
+					zap.Int64("chat_id", chatID),
+					zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
+					zap.Error(err),
+				)
 			}
 		}
 	}

pkg/linkprev/linkprev.go

@@ -12,6 +12,7 @@ import (
 	"github.com/PuerkitoBio/goquery"
 	"github.com/imroc/req/v3"
 	"github.com/nekomeowww/insights-bot/pkg/opengraph"
+	"github.com/nekomeowww/xo"
 	"github.com/samber/lo"
 )
 
@@ -86,12 +87,19 @@ func (c *Client) alterRequestForTwitter(request *req.Request, urlStr string) *re
 
 func (c *Client) request(r *req.Request, urlStr string) (*bytes.Buffer, error) {
 	dumpBuffer := new(bytes.Buffer)
-	defer dumpBuffer.Reset()
+	defer func() {
+		dumpBuffer.Reset()
+		dumpBuffer = nil
+	}()
 
-	resp, err := r.
+	request := r.
 		EnableDumpTo(dumpBuffer).
-		DisableAutoReadResponse().
-		Get(urlStr)
+		DisableAutoReadResponse()
+	defer func() {
+		request.EnableDumpTo(xo.NewNopIoWriter())
+	}()
+
+	resp, err := request.Get(urlStr)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get a preview of url %s, %w: %v", urlStr, ErrNetworkError, err)
 	}