From ddf8b736c3eae885d9ce058b60589d718ea64b4c Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 12 Dec 2024 17:03:34 +0800 Subject: [PATCH 01/17] pb --- internal/rpc/msg/delete.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/internal/rpc/msg/delete.go b/internal/rpc/msg/delete.go index e19bba867e..371c50e2ea 100644 --- a/internal/rpc/msg/delete.go +++ b/internal/rpc/msg/delete.go @@ -138,9 +138,16 @@ func (m *msgServer) clearConversation(ctx context.Context, conversationIDs []str } isSyncSelf, isSyncOther := m.validateDeleteSyncOpt(deleteSyncOpt) if !isSyncOther { - if err := m.MsgDatabase.SetUserConversationsMinSeqs(ctx, userID, m.getMinSeqs(maxSeqs)); err != nil { + setSeqs := m.getMinSeqs(maxSeqs) + if err := m.MsgDatabase.SetUserConversationsMinSeqs(ctx, userID, setSeqs); err != nil { return err } + ownerUserIDs := []string{userID} + for conversationID, seq := range setSeqs { + if err := m.Conversation.SetConversationMinSeq(ctx, ownerUserIDs, conversationID, seq); err != nil { + return err + } + } // notification 2 self if isSyncSelf { tips := &sdkws.ClearConversationTips{UserID: userID, ConversationIDs: existConversationIDs} From 6f169ec9690f62e8ea8edb697c87bb74cd3c17c3 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 17 Dec 2024 11:58:06 +0800 Subject: [PATCH 02/17] fix: Modifying other fields while setting IsPrivateChat does not take effect --- internal/rpc/conversation/conversation.go | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index 7345c965db..8ed1323d56 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -354,7 +354,15 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver needUpdateUsersList = append(needUpdateUsersList, userID) } } + if len(m) != 0 && len(needUpdateUsersList) != 0 { + if err := c.conversationDatabase.SetUsersConversationFieldTx(ctx, needUpdateUsersList, &conversation, m); err != nil { + return nil, err + } + for _, v := range needUpdateUsersList { + c.conversationNotificationSender.ConversationChangeNotification(ctx, v, []string{req.Conversation.ConversationID}) + } + } if req.Conversation.IsPrivateChat != nil && req.Conversation.ConversationType != constant.ReadGroupChatType { var conversations []*dbModel.Conversation for _, ownerUserID := range req.UserIDs { @@ -372,16 +380,6 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver c.conversationNotificationSender.ConversationSetPrivateNotification(ctx, userID, req.Conversation.UserID, req.Conversation.IsPrivateChat.Value, req.Conversation.ConversationID) } - } else { - if len(m) != 0 && len(needUpdateUsersList) != 0 { - if err := c.conversationDatabase.SetUsersConversationFieldTx(ctx, needUpdateUsersList, &conversation, m); err != nil { - return nil, err - } - - for _, v := range needUpdateUsersList { - c.conversationNotificationSender.ConversationChangeNotification(ctx, v, []string{req.Conversation.ConversationID}) - } - } } return &pbconversation.SetConversationsResp{}, nil From d15414d8851c37b938e5d795fd40e91a836ff5f7 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 17 Dec 2024 18:28:16 +0800 Subject: [PATCH 03/17] fix: quote message error revoke --- pkg/common/storage/controller/msg.go | 52 +++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 8d82d85433..3c3cd9671b 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -269,7 +269,6 @@ func (db *commonMsgDatabase) getMsgBySeqs(ctx context.Context, userID, conversat } return totalMsgs, nil } - func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][]*model.MsgInfoModel, userID, conversationID string, msg *model.MsgInfoModel) { if msg.IsRead { msg.Msg.IsRead = true @@ -280,16 +279,53 @@ func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][ if msg.Msg.Content == "" { return } + type MsgData struct { + SendID string `json:"sendID"` + RecvID string `json:"recvID"` + GroupID string `json:"groupID"` + ClientMsgID string `json:"clientMsgID"` + ServerMsgID string `json:"serverMsgID"` + SenderPlatformID int32 `json:"senderPlatformID"` + SenderNickname string `json:"senderNickname"` + SenderFaceURL string `json:"senderFaceURL"` + SessionType int32 `json:"sessionType"` + MsgFrom int32 `json:"msgFrom"` + ContentType int32 `json:"contentType"` + Content string `json:"content"` + Seq int64 `json:"seq"` + SendTime int64 `json:"sendTime"` + CreateTime int64 `json:"createTime"` + Status int32 `json:"status"` + IsRead bool `json:"isRead"` + Options map[string]bool `json:"options,omitempty"` + OfflinePushInfo *sdkws.OfflinePushInfo `json:"offlinePushInfo"` + AtUserIDList []string `json:"atUserIDList"` + AttachedInfo string `json:"attachedInfo"` + Ex string `json:"ex"` + KeyVersion int32 `json:"keyVersion"` + DstUserIDs []string `json:"dstUserIDs"` + } var quoteMsg struct { Text string `json:"text,omitempty"` - QuoteMessage *sdkws.MsgData `json:"quoteMessage,omitempty"` + QuoteMessage *MsgData `json:"quoteMessage,omitempty"` MessageEntityList json.RawMessage `json:"messageEntityList,omitempty"` } if err := json.Unmarshal([]byte(msg.Msg.Content), "eMsg); err != nil { log.ZError(ctx, "json.Unmarshal", err) return } - if quoteMsg.QuoteMessage == nil || quoteMsg.QuoteMessage.ContentType == constant.MsgRevokeNotification { + if quoteMsg.QuoteMessage == nil || quoteMsg.QuoteMessage.Content == "" { + return + } + if quoteMsg.QuoteMessage.Content == "e30=" { + quoteMsg.QuoteMessage.Content = "{}" + data, err := json.Marshal("eMsg) + if err != nil { + return + } + msg.Msg.Content = string(data) + } + if quoteMsg.QuoteMessage.Seq <= 0 && quoteMsg.QuoteMessage.ContentType == constant.MsgRevokeNotification { return } var msgs []*model.MsgInfoModel @@ -311,9 +347,9 @@ func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][ } quoteMsg.QuoteMessage.ContentType = constant.MsgRevokeNotification if len(msgs) > 0 { - quoteMsg.QuoteMessage.Content = []byte(msgs[0].Msg.Content) + quoteMsg.QuoteMessage.Content = msgs[0].Msg.Content } else { - quoteMsg.QuoteMessage.Content = []byte("{}") + quoteMsg.QuoteMessage.Content = "{}" } data, err := json.Marshal("eMsg) if err != nil { @@ -321,9 +357,9 @@ func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][ return } msg.Msg.Content = string(data) - if _, err := db.msgDocDatabase.UpdateMsg(ctx, db.msgTable.GetDocID(conversationID, msg.Msg.Seq), db.msgTable.GetMsgIndex(msg.Msg.Seq), "msg", msg.Msg); err != nil { - log.ZError(ctx, "UpdateMsgContent", err) - } + //if _, err := db.msgDocDatabase.UpdateMsg(ctx, db.msgTable.GetDocID(conversationID, msg.Msg.Seq), db.msgTable.GetMsgIndex(msg.Msg.Seq), "msg", msg.Msg); err != nil { + // log.ZError(ctx, "UpdateMsgContent", err) + //} } func (db *commonMsgDatabase) findMsgInfoBySeq(ctx context.Context, userID, docID string, conversationID string, seqs []int64) (totalMsgs []*model.MsgInfoModel, err error) { From f03cbec6532cfadface6a55b365d36df879e8069 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 18 Dec 2024 17:41:49 +0800 Subject: [PATCH 04/17] refactoring scheduled tasks --- internal/rpc/third/s3.go | 93 +++------------- internal/rpc/third/third.go | 14 +-- internal/tools/cron_task.go | 125 ++++++++-------------- internal/tools/msg.go | 23 ++++ internal/tools/s3.go | 34 ++++++ internal/tools/user_msg.go | 31 ++++++ pkg/common/storage/controller/s3.go | 20 +--- pkg/common/storage/database/mgo/object.go | 29 +++-- pkg/common/storage/database/object.go | 6 +- 9 files changed, 175 insertions(+), 200 deletions(-) create mode 100644 internal/tools/msg.go create mode 100644 internal/tools/s3.go create mode 100644 internal/tools/user_msg.go diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index b4b064d7f4..d4146fdf48 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -23,13 +23,9 @@ import ( "strconv" "time" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "go.mongodb.org/mongo-driver/mongo" - "github.com/google/uuid" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" - "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/protocol/third" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" @@ -288,87 +284,30 @@ func (t *thirdServer) apiAddress(prefix, name string) string { } func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { - var conf config.Third + engine := t.config.RpcConfig.Object.Enable expireTime := time.UnixMilli(req.ExpireTime) - - findPagination := &sdkws.RequestPagination{ - PageNumber: 1, - ShowNumber: 500, - } - // Find all expired data in S3 database - total, models, err := t.s3dataBase.FindNeedDeleteObjectByDB(ctx, expireTime, req.ObjectGroup, findPagination) - if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { - return nil, errs.Wrap(err) - } - - if total == 0 { - log.ZDebug(ctx, "Not have OutdatedData", "delete Total", total) - return &third.DeleteOutdatedDataResp{Count: int32(total)}, nil - } - - needDelObjectKeys := make([]string, len(models)) - for _, model := range models { - needDelObjectKeys = append(needDelObjectKeys, model.Key) + models, err := t.s3dataBase.FindExpirationObject(ctx, engine, expireTime, req.ObjectGroup, int64(req.Count)) + if err != nil { + return nil, err } - - // Remove duplicate keys, have the same key use in different models - needDelObjectKeys = datautil.Distinct(needDelObjectKeys) - - for _, key := range needDelObjectKeys { - // Find all models by key - keyModels, err := t.s3dataBase.FindModelsByKey(ctx, key) - if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { + if len(models) > 0 { + names := datautil.Batch(func(o *model.Object) string { + return o.Name + }, models) + if err := t.s3dataBase.DeleteSpecifiedData(ctx, engine, names); err != nil { return nil, errs.Wrap(err) } - - // check keyModels, if all keyModels. - needDelKey := true // Default can delete - for _, keymodel := range keyModels { - // If group is empty or CreateTime is after expireTime, can't delete this key - if keymodel.Group == "" || keymodel.CreateTime.After(expireTime) { - needDelKey = false - break - } + if err := t.s3dataBase.DelS3Key(ctx, engine, names...); err != nil { + return nil, err } - - // If this object is not referenced by not expire data, delete it - if needDelKey && t.minio != nil { - // If have a thumbnail, delete it - thumbnailKey, _ := t.getMinioImageThumbnailKey(ctx, key) - if thumbnailKey != "" { - err := t.s3dataBase.DeleteObject(ctx, thumbnailKey) - if err != nil { - log.ZWarn(ctx, "Delete thumbnail object is error:", errs.Wrap(err), "thumbnailKey", thumbnailKey) - } - } - - // Delete object - err = t.s3dataBase.DeleteObject(ctx, key) - if err != nil { - log.ZWarn(ctx, "Delete object is error", errs.Wrap(err), "object key", key) - } - - // Delete cache key - err = t.s3dataBase.DelS3Key(ctx, conf.Object.Enable, key) - if err != nil { - log.ZWarn(ctx, "Delete cache key is error:", errs.Wrap(err), "cache S3 key:", key) + for _, object := range models { + if err := t.s3.DeleteObject(ctx, object.Key); err != nil { + return nil, err } } } - - // handle delete data in S3 database - for _, model := range models { - // Delete all expired data row in S3 database - err := t.s3dataBase.DeleteSpecifiedData(ctx, model.Engine, model.Name) - if err != nil { - return nil, errs.Wrap(err) - } - } - - log.ZDebug(ctx, "DeleteOutdatedData", "delete Total", total) - - return &third.DeleteOutdatedDataResp{Count: int32(total)}, nil + return &third.DeleteOutdatedDataResp{Count: int32(len(models))}, nil } type FormDataMate struct { diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index 77e6d459fa..dc964fdb15 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -44,7 +44,7 @@ type thirdServer struct { s3dataBase controller.S3Database defaultExpire time.Duration config *Config - minio *minio.Minio + s3 s3.Interface } type Config struct { @@ -79,13 +79,11 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg // Select the oss method according to the profile policy enable := config.RpcConfig.Object.Enable var ( - o s3.Interface - minioCli *minio.Minio + o s3.Interface ) switch enable { case "minio": - minioCli, err = minio.NewMinio(ctx, redis.NewMinioCache(rdb), *config.MinioConfig.Build()) - o = minioCli + o, err = minio.NewMinio(ctx, redis.NewMinioCache(rdb), *config.MinioConfig.Build()) case "cos": o, err = cos.NewCos(*config.RpcConfig.Object.Cos.Build()) case "oss": @@ -106,15 +104,11 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg s3dataBase: controller.NewS3Database(rdb, o, s3db), defaultExpire: time.Hour * 24 * 7, config: config, - minio: minioCli, + s3: o, }) return nil } -func (t *thirdServer) getMinioImageThumbnailKey(ctx context.Context, name string) (string, error) { - return t.minio.GetImageThumbnailKey(ctx, name) -} - func (t *thirdServer) FcmUpdateToken(ctx context.Context, req *third.FcmUpdateTokenReq) (resp *third.FcmUpdateTokenResp, err error) { err = t.thirdDatabase.FcmUpdateToken(ctx, req.Account, int(req.PlatformID), req.FcmToken, req.ExpireTime) if err != nil { diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index 2fe7d0e395..049a6199cf 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -2,10 +2,6 @@ package tools import ( "context" - "fmt" - "os" - "time" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" pbconversation "github.com/openimsdk/protocol/conversation" @@ -60,87 +56,58 @@ func Start(ctx context.Context, config *CronTaskConfig) error { return err } - msgClient := msg.NewMsgClient(msgConn) - conversationClient := pbconversation.NewConversationClient(conversationConn) - thirdClient := third.NewThirdClient(thirdConn) - - crontab := cron.New() - - // scheduled hard delete outdated Msgs in specific time. - destructMsgsFunc := func() { - now := time.Now() - deltime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.RetainChatRecords)) - ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deltime.UnixMilli())) - log.ZDebug(ctx, "Destruct chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) - - if _, err := msgClient.DestructMsgs(ctx, &msg.DestructMsgsReq{Timestamp: deltime.UnixMilli()}); err != nil { - log.ZError(ctx, "cron destruct chat records failed", err, "deltime", deltime, "cont", time.Since(now)) - return - } - log.ZDebug(ctx, "cron destruct chat records success", "deltime", deltime, "cont", time.Since(now)) - } - if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, destructMsgsFunc); err != nil { - return errs.Wrap(err) + srv := &cronServer{ + ctx: ctx, + config: config, + cron: cron.New(), + msgClient: msg.NewMsgClient(msgConn), + conversationClient: pbconversation.NewConversationClient(conversationConn), + thirdClient: third.NewThirdClient(thirdConn), } - // scheduled soft delete outdated Msgs in specific time when user set `is_msg_destruct` feature. - clearMsgFunc := func() { - now := time.Now() - ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), now.UnixMilli())) - log.ZDebug(ctx, "clear msg cron start", "now", now) - - conversations, err := conversationClient.GetConversationsNeedClearMsg(ctx, &pbconversation.GetConversationsNeedClearMsgReq{}) - if err != nil { - log.ZError(ctx, "Get conversation need Destruct msgs failed.", err) - return - } - - _, err = msgClient.ClearMsg(ctx, &msg.ClearMsgReq{Conversations: conversations.Conversations}) - if err != nil { - log.ZError(ctx, "Clear Msg failed.", err) - return - } - - log.ZDebug(ctx, "clear msg cron task completed", "cont", time.Since(now)) - } - if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, clearMsgFunc); err != nil { - return errs.Wrap(err) + if err := srv.registerClearS3(); err != nil { + return err } - - // scheduled delete outdated file Objects and their datas in specific time. - deleteObjectFunc := func() { - now := time.Now() - executeNum := 5 - // number of pagination. if need modify, need update value in third.DeleteOutdatedData - pageShowNumber := 500 - deleteTime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.FileExpireTime)) - ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli())) - log.ZDebug(ctx, "deleteoutDatedData", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) - - if len(config.CronTask.DeleteObjectType) == 0 { - log.ZDebug(ctx, "cron deleteoutDatedData not type need delete", "deletetime", deleteTime, "DeleteObjectType", config.CronTask.DeleteObjectType, "cont", time.Since(now)) - return - } - - for i := 0; i < executeNum; i++ { - resp, err := thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: config.CronTask.DeleteObjectType}) - if err != nil { - log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(now)) - return - } - if resp.Count == 0 || resp.Count < int32(pageShowNumber) { - break - } - } - - log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(now)) + if err := srv.registerDeleteMsg(); err != nil { + return err } - if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, deleteObjectFunc); err != nil { - return errs.Wrap(err) + if err := srv.registerClearUserMsg(); err != nil { + return err } - log.ZDebug(ctx, "start cron task", "CronExecuteTime", config.CronTask.CronExecuteTime) - crontab.Start() + srv.cron.Start() <-ctx.Done() return nil } + +type cronServer struct { + ctx context.Context + config *CronTaskConfig + cron *cron.Cron + msgClient msg.MsgClient + conversationClient pbconversation.ConversationClient + thirdClient third.ThirdClient +} + +func (c *cronServer) registerClearS3() error { + if c.config.CronTask.FileExpireTime <= 0 || len(c.config.CronTask.DeleteObjectType) == 0 { + log.ZInfo(c.ctx, "disable scheduled cleanup of s3", "fileExpireTime", c.config.CronTask.FileExpireTime, "deleteObjectType", c.config.CronTask.DeleteObjectType) + return nil + } + _, err := c.cron.AddFunc(c.config.CronTask.CronExecuteTime, c.clearS3) + return errs.WrapMsg(err, "failed to register clear s3 cron task") +} + +func (c *cronServer) registerDeleteMsg() error { + if c.config.CronTask.RetainChatRecords <= 0 { + log.ZInfo(c.ctx, "disable scheduled cleanup of chat records", "retainChatRecords", c.config.CronTask.RetainChatRecords) + return nil + } + _, err := c.cron.AddFunc(c.config.CronTask.CronExecuteTime, c.deleteMsg) + return errs.WrapMsg(err, "failed to register delete msg cron task") +} + +func (c *cronServer) registerClearUserMsg() error { + _, err := c.cron.AddFunc(c.config.CronTask.CronExecuteTime, c.clearUserMsg) + return errs.WrapMsg(err, "failed to register clear user msg cron task") +} diff --git a/internal/tools/msg.go b/internal/tools/msg.go new file mode 100644 index 0000000000..1149b1f297 --- /dev/null +++ b/internal/tools/msg.go @@ -0,0 +1,23 @@ +package tools + +import ( + "fmt" + "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "os" + "time" +) + +func (c *cronServer) deleteMsg() { + now := time.Now() + deltime := now.Add(-time.Hour * 24 * time.Duration(c.config.CronTask.RetainChatRecords)) + ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deltime.UnixMilli())) + log.ZDebug(ctx, "Destruct chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) + + if _, err := c.msgClient.DestructMsgs(ctx, &msg.DestructMsgsReq{Timestamp: deltime.UnixMilli()}); err != nil { + log.ZError(ctx, "cron destruct chat records failed", err, "deltime", deltime, "cont", time.Since(now)) + return + } + log.ZDebug(ctx, "cron destruct chat records success", "deltime", deltime, "cont", time.Since(now)) +} diff --git a/internal/tools/s3.go b/internal/tools/s3.go new file mode 100644 index 0000000000..38d8badc66 --- /dev/null +++ b/internal/tools/s3.go @@ -0,0 +1,34 @@ +package tools + +import ( + "fmt" + "github.com/openimsdk/protocol/third" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "os" + "time" +) + +func (c *cronServer) clearS3() { + start := time.Now() + executeNum := 10 + // number of pagination. if need modify, need update value in third.DeleteOutdatedData + pageShowNumber := 500 + deleteTime := start.Add(-time.Hour * 24 * time.Duration(c.config.CronTask.FileExpireTime)) + operationID := fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli()) + ctx := mcontext.SetOperationID(c.ctx, operationID) + log.ZDebug(ctx, "deleteoutDatedData", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) + for i := 1; i <= executeNum; i++ { + ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("%s_%d", operationID, i)) + resp, err := c.thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: c.config.CronTask.DeleteObjectType, Count: int32(pageShowNumber)}) + if err != nil { + log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(start)) + return + } + if resp.Count < int32(pageShowNumber) { + break + } + } + + log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(start)) +} diff --git a/internal/tools/user_msg.go b/internal/tools/user_msg.go new file mode 100644 index 0000000000..9dfaf58c36 --- /dev/null +++ b/internal/tools/user_msg.go @@ -0,0 +1,31 @@ +package tools + +import ( + "fmt" + pbconversation "github.com/openimsdk/protocol/conversation" + "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "os" + "time" +) + +func (c *cronServer) clearUserMsg() { + now := time.Now() + ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), now.UnixMilli())) + log.ZDebug(ctx, "clear msg cron start", "now", now) + + conversations, err := c.conversationClient.GetConversationsNeedClearMsg(ctx, &pbconversation.GetConversationsNeedClearMsgReq{}) + if err != nil { + log.ZError(ctx, "Get conversation need Destruct msgs failed.", err) + return + } + + _, err = c.msgClient.ClearMsg(ctx, &msg.ClearMsgReq{Conversations: conversations.Conversations}) + if err != nil { + log.ZError(ctx, "Clear Msg failed.", err) + return + } + + log.ZDebug(ctx, "clear msg cron task completed", "cont", time.Since(now)) +} diff --git a/pkg/common/storage/controller/s3.go b/pkg/common/storage/controller/s3.go index 4e5ad18b6e..6a0f11b1ce 100644 --- a/pkg/common/storage/controller/s3.go +++ b/pkg/common/storage/controller/s3.go @@ -24,7 +24,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" - "github.com/openimsdk/tools/db/pagination" "github.com/openimsdk/tools/s3" "github.com/openimsdk/tools/s3/cont" "github.com/redis/go-redis/v9" @@ -40,10 +39,8 @@ type S3Database interface { SetObject(ctx context.Context, info *model.Object) error StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) - FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) - DeleteObject(ctx context.Context, name string) error - DeleteSpecifiedData(ctx context.Context, engine string, name string) error - FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error) + FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) + DeleteSpecifiedData(ctx context.Context, engine string, name []string) error DelS3Key(ctx context.Context, engine string, keys ...string) error } @@ -120,21 +117,14 @@ func (s *s3Database) StatObject(ctx context.Context, name string) (*s3.ObjectInf func (s *s3Database) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { return s.s3.FormData(ctx, name, size, contentType, duration) } -func (s *s3Database) FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) { - return s.db.FindNeedDeleteObjectByDB(ctx, duration, needDelType, pagination) +func (s *s3Database) FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) { + return s.db.FindExpirationObject(ctx, engine, expiration, needDelType, count) } -func (s *s3Database) DeleteObject(ctx context.Context, name string) error { - return s.s3.DeleteObject(ctx, name) -} -func (s *s3Database) DeleteSpecifiedData(ctx context.Context, engine string, name string) error { +func (s *s3Database) DeleteSpecifiedData(ctx context.Context, engine string, name []string) error { return s.db.Delete(ctx, engine, name) } -func (s *s3Database) FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error) { - return s.db.FindModelsByKey(ctx, key) -} - func (s *s3Database) DelS3Key(ctx context.Context, engine string, keys ...string) error { return s.s3cache.DelS3Key(ctx, engine, keys...) } diff --git a/pkg/common/storage/database/mgo/object.go b/pkg/common/storage/database/mgo/object.go index 5bc329d33b..53f2a6ba43 100644 --- a/pkg/common/storage/database/mgo/object.go +++ b/pkg/common/storage/database/mgo/object.go @@ -22,7 +22,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/mongoutil" - "github.com/openimsdk/tools/db/pagination" "github.com/openimsdk/tools/errs" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" @@ -91,21 +90,21 @@ func (o *S3Mongo) Take(ctx context.Context, engine string, name string) (*model. return mongoutil.FindOne[*model.Object](ctx, o.coll, bson.M{"name": name, "engine": engine}) } -func (o *S3Mongo) Delete(ctx context.Context, engine string, name string) error { - return mongoutil.DeleteOne(ctx, o.coll, bson.M{"name": name, "engine": engine}) -} - -// Find Expires object -func (o *S3Mongo) FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) { - return mongoutil.FindPage[*model.Object](ctx, o.coll, bson.M{ - "create_time": bson.M{"$lt": duration}, - "group": bson.M{"$in": needDelType}, - }, pagination) +func (o *S3Mongo) Delete(ctx context.Context, engine string, name []string) error { + if len(name) == 0 { + return nil + } + return mongoutil.DeleteOne(ctx, o.coll, bson.M{"engine": engine, "name": bson.M{"$in": name}}) } -// Find object by key -func (o *S3Mongo) FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error) { +func (o *S3Mongo) FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) { + opt := options.Find() + if count > 0 { + opt.SetLimit(count) + } return mongoutil.Find[*model.Object](ctx, o.coll, bson.M{ - "key": key, - }) + "engine": engine, + "create_time": bson.M{"$lt": expiration}, + "group": bson.M{"$in": needDelType}, + }, opt) } diff --git a/pkg/common/storage/database/object.go b/pkg/common/storage/database/object.go index c741e39a60..86d47a2a6b 100644 --- a/pkg/common/storage/database/object.go +++ b/pkg/common/storage/database/object.go @@ -19,13 +19,11 @@ import ( "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/tools/db/pagination" ) type ObjectInfo interface { SetObject(ctx context.Context, obj *model.Object) error Take(ctx context.Context, engine string, name string) (*model.Object, error) - Delete(ctx context.Context, engine string, name string) error - FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) - FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error) + Delete(ctx context.Context, engine string, name []string) error + FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) } From 705bc37f9942b3059e2ec1bf82c5fb227329b271 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 19 Dec 2024 11:28:49 +0800 Subject: [PATCH 05/17] refactoring scheduled tasks --- internal/rpc/msg/clear.go | 71 ++++++------- internal/rpc/third/s3.go | 6 +- internal/tools/msg.go | 25 +++-- internal/tools/s3.go | 11 ++- internal/tools/user_msg.go | 6 +- pkg/common/storage/controller/msg.go | 104 +++++++++----------- pkg/common/storage/database/mgo/msg.go | 20 ++-- pkg/common/storage/database/mgo/msg_test.go | 78 ++++++++++----- pkg/common/storage/database/msg.go | 4 +- 9 files changed, 179 insertions(+), 146 deletions(-) diff --git a/internal/rpc/msg/clear.go b/internal/rpc/msg/clear.go index 7d62e7c8fa..8d4bd0c8ee 100644 --- a/internal/rpc/msg/clear.go +++ b/internal/rpc/msg/clear.go @@ -2,6 +2,7 @@ package msg import ( "context" + "strings" "time" "github.com/openimsdk/open-im-server/v3/pkg/authverify" @@ -26,52 +27,42 @@ func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) if req.Timestamp > time.Now().UnixMilli() { return nil, errs.ErrArgs.WrapMsg("request millisecond timestamp error") } - var ( - docNum int - msgNum int - start = time.Now() - getLimit = 5000 - ) - - destructMsg := func(ctx context.Context) (bool, error) { - docIDs, err := m.MsgDatabase.GetDocIDs(ctx) - if err != nil { - return false, err - } - - msgs, err := m.MsgDatabase.GetBeforeMsg(ctx, req.Timestamp, docIDs, getLimit) - if err != nil { - return false, err + if req.Limit <= 0 { + return nil, errs.ErrArgs.WrapMsg("request limit error") + } + docs, err := m.MsgDatabase.GetRandBeforeMsg(ctx, req.Timestamp, int(req.Limit)) + if err != nil { + return nil, err + } + for _, doc := range docs { + if err := m.MsgDatabase.DeleteDoc(ctx, doc.DocID); err != nil { + return nil, err } - if len(msgs) == 0 { - return false, nil + index := strings.LastIndex(doc.DocID, ":") + if index < 0 { + continue } - - for _, msg := range msgs { - index, err := m.MsgDatabase.DeleteDocMsgBefore(ctx, req.Timestamp, msg) - if err != nil { - return false, err + var minSeq int64 + for _, model := range doc.Msg { + if model.Msg == nil { + continue } - if len(index) == 0 { - return false, errs.ErrInternalServer.WrapMsg("delete doc msg failed") + if model.Msg.Seq > minSeq { + minSeq = model.Msg.Seq } - - docNum++ - msgNum += len(index) } - - return true, nil - } - - _, err = destructMsg(ctx) - if err != nil { - log.ZError(ctx, "clear msg failed", err, "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) - return nil, err + if minSeq <= 0 { + continue + } + conversationID := doc.DocID[:index] + if conversationID == "" { + continue + } + if err := m.MsgDatabase.SetMinSeq(ctx, conversationID, minSeq); err != nil { + return nil, err + } } - - log.ZDebug(ctx, "clearing message", "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) - - return &msg.DestructMsgsResp{}, nil + return &msg.DestructMsgsResp{Count: int32(len(docs))}, nil } // soft delete for user self diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index d4146fdf48..a541462a28 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -19,6 +19,7 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "path" "strconv" "time" @@ -284,10 +285,13 @@ func (t *thirdServer) apiAddress(prefix, name string) string { } func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { + if err := authverify.CheckAdmin(ctx, t.config.Share.IMAdminUserID); err != nil { + return nil, err + } engine := t.config.RpcConfig.Object.Enable expireTime := time.UnixMilli(req.ExpireTime) // Find all expired data in S3 database - models, err := t.s3dataBase.FindExpirationObject(ctx, engine, expireTime, req.ObjectGroup, int64(req.Count)) + models, err := t.s3dataBase.FindExpirationObject(ctx, engine, expireTime, req.ObjectGroup, int64(req.Limit)) if err != nil { return nil, err } diff --git a/internal/tools/msg.go b/internal/tools/msg.go index 1149b1f297..46f1573f28 100644 --- a/internal/tools/msg.go +++ b/internal/tools/msg.go @@ -12,12 +12,25 @@ import ( func (c *cronServer) deleteMsg() { now := time.Now() deltime := now.Add(-time.Hour * 24 * time.Duration(c.config.CronTask.RetainChatRecords)) - ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deltime.UnixMilli())) + operationID := fmt.Sprintf("cron_msg_%d_%d", os.Getpid(), deltime.UnixMilli()) + ctx := mcontext.SetOperationID(c.ctx, operationID) log.ZDebug(ctx, "Destruct chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) - - if _, err := c.msgClient.DestructMsgs(ctx, &msg.DestructMsgsReq{Timestamp: deltime.UnixMilli()}); err != nil { - log.ZError(ctx, "cron destruct chat records failed", err, "deltime", deltime, "cont", time.Since(now)) - return + const ( + deleteCount = 20 + deleteLimit = 50 + ) + var count int + for i := 1; i <= deleteCount; i++ { + ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("%s_%d", operationID, i)) + resp, err := c.msgClient.DestructMsgs(ctx, &msg.DestructMsgsReq{Timestamp: deltime.UnixMilli(), Limit: deleteLimit}) + if err != nil { + log.ZError(ctx, "cron destruct chat records failed", err) + break + } + count += int(resp.Count) + if resp.Count <= deleteLimit { + break + } } - log.ZDebug(ctx, "cron destruct chat records success", "deltime", deltime, "cont", time.Since(now)) + log.ZDebug(ctx, "cron destruct chat records end", "deltime", deltime, "cont", time.Since(now), "deleteDocs", count) } diff --git a/internal/tools/s3.go b/internal/tools/s3.go index 38d8badc66..3b0122c84e 100644 --- a/internal/tools/s3.go +++ b/internal/tools/s3.go @@ -15,20 +15,21 @@ func (c *cronServer) clearS3() { // number of pagination. if need modify, need update value in third.DeleteOutdatedData pageShowNumber := 500 deleteTime := start.Add(-time.Hour * 24 * time.Duration(c.config.CronTask.FileExpireTime)) - operationID := fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli()) + operationID := fmt.Sprintf("cron_s3_%d_%d", os.Getpid(), deleteTime.UnixMilli()) ctx := mcontext.SetOperationID(c.ctx, operationID) log.ZDebug(ctx, "deleteoutDatedData", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) + var count int for i := 1; i <= executeNum; i++ { ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("%s_%d", operationID, i)) - resp, err := c.thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: c.config.CronTask.DeleteObjectType, Count: int32(pageShowNumber)}) + resp, err := c.thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: c.config.CronTask.DeleteObjectType, Limit: int32(pageShowNumber)}) if err != nil { - log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(start)) + log.ZError(ctx, "cron deleteoutDatedData failed", err) return } + count += int(resp.Count) if resp.Count < int32(pageShowNumber) { break } } - - log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(start)) + log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(start), "count", count) } diff --git a/internal/tools/user_msg.go b/internal/tools/user_msg.go index 9dfaf58c36..52a2b6bdce 100644 --- a/internal/tools/user_msg.go +++ b/internal/tools/user_msg.go @@ -12,9 +12,9 @@ import ( func (c *cronServer) clearUserMsg() { now := time.Now() - ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), now.UnixMilli())) - log.ZDebug(ctx, "clear msg cron start", "now", now) - + operationID := fmt.Sprintf("cron_user_msg_%d_%d", os.Getpid(), now.UnixMilli()) + ctx := mcontext.SetOperationID(c.ctx, operationID) + log.ZDebug(ctx, "clear msg cron start") conversations, err := c.conversationClient.GetConversationsNeedClearMsg(ctx, &pbconversation.GetConversationsNeedClearMsgReq{}) if err != nil { log.ZError(ctx, "Get conversation need Destruct msgs failed.", err) diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 3c3cd9671b..88089f1a09 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -18,7 +18,6 @@ import ( "context" "encoding/json" "errors" - "strings" "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" @@ -69,6 +68,7 @@ type CommonMsgDatabase interface { GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) SetMinSeqs(ctx context.Context, seqs map[string]int64) error + SetMinSeq(ctx context.Context, conversationID string, seq int64) error SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) (err error) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error @@ -95,13 +95,14 @@ type CommonMsgDatabase interface { ConvertMsgsDocLen(ctx context.Context, conversationIDs []string) // get Msg when destruct msg before - GetBeforeMsg(ctx context.Context, ts int64, docIds []string, limit int) ([]*model.MsgDocModel, error) - DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) + //DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) - GetDocIDs(ctx context.Context) ([]string, error) + GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) SetUserConversationsMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error SetUserConversationsMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error + + DeleteDoc(ctx context.Context, docID string) error } func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { @@ -806,9 +807,10 @@ func (db *commonMsgDatabase) GetMaxSeq(ctx context.Context, conversationID strin return db.seqConversation.GetMaxSeq(ctx, conversationID) } -func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error { - return db.seqConversation.SetMinSeq(ctx, conversationID, minSeq) -} +// +//func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error { +// return db.seqConversation.SetMinSeq(ctx, conversationID, minSeq) +//} func (db *commonMsgDatabase) SetMinSeqs(ctx context.Context, seqs map[string]int64) error { return db.seqConversation.SetMinSeqs(ctx, seqs) @@ -947,56 +949,40 @@ func (db *commonMsgDatabase) ConvertMsgsDocLen(ctx context.Context, conversation db.msgDocDatabase.ConvertMsgsDocLen(ctx, conversationIDs) } -func (db *commonMsgDatabase) GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, limit int) ([]*model.MsgDocModel, error) { - var msgs []*model.MsgDocModel - for i := 0; i < len(docIDs); i += 1000 { - end := i + 1000 - if end > len(docIDs) { - end = len(docIDs) - } - - res, err := db.msgDocDatabase.GetBeforeMsg(ctx, ts, docIDs[i:end], limit) - if err != nil { - return nil, err - } - msgs = append(msgs, res...) - - if len(msgs) >= limit { - return msgs[:limit], nil - } - } - return msgs, nil -} - -func (db *commonMsgDatabase) DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) { - var notNull int - index := make([]int, 0, len(doc.Msg)) - for i, message := range doc.Msg { - if message.Msg != nil { - notNull++ - if message.Msg.SendTime < ts { - index = append(index, i) - } - } - } - if len(index) == 0 { - return index, nil - } - maxSeq := doc.Msg[index[len(index)-1]].Msg.Seq - conversationID := doc.DocID[:strings.LastIndex(doc.DocID, ":")] - if err := db.setMinSeq(ctx, conversationID, maxSeq+1); err != nil { - return index, err - } - if len(index) == notNull { - log.ZDebug(ctx, "Delete db in Doc", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq) - return index, db.msgDocDatabase.DeleteDoc(ctx, doc.DocID) - } else { - log.ZDebug(ctx, "delete db in index", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq) - return index, db.msgDocDatabase.DeleteMsgByIndex(ctx, doc.DocID, index) - } +func (db *commonMsgDatabase) GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) { + return db.msgDocDatabase.GetRandBeforeMsg(ctx, ts, limit) } -func (db *commonMsgDatabase) setMinSeq(ctx context.Context, conversationID string, seq int64) error { +// +//func (db *commonMsgDatabase) DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) { +// var notNull int +// index := make([]int, 0, len(doc.Msg)) +// for i, message := range doc.Msg { +// if message.Msg != nil { +// notNull++ +// if message.Msg.SendTime < ts { +// index = append(index, i) +// } +// } +// } +// if len(index) == 0 { +// return index, nil +// } +// maxSeq := doc.Msg[index[len(index)-1]].Msg.Seq +// conversationID := doc.DocID[:strings.LastIndex(doc.DocID, ":")] +// if err := db.SetMinSeq(ctx, conversationID, maxSeq+1); err != nil { +// return index, err +// } +// if len(index) == notNull { +// log.ZDebug(ctx, "Delete db in Doc", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq) +// return index, db.msgDocDatabase.DeleteDoc(ctx, doc.DocID) +// } else { +// log.ZDebug(ctx, "delete db in index", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq) +// return index, db.msgDocDatabase.DeleteMsgByIndex(ctx, doc.DocID, index) +// } +//} + +func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { dbSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) if err != nil { if errors.Is(errs.Unwrap(err), redis.Nil) { @@ -1010,8 +996,8 @@ func (db *commonMsgDatabase) setMinSeq(ctx context.Context, conversationID strin return db.seqConversation.SetMinSeq(ctx, conversationID, seq) } -func (db *commonMsgDatabase) GetDocIDs(ctx context.Context) ([]string, error) { - return db.msgDocDatabase.GetDocIDs(ctx) +func (db *commonMsgDatabase) GetRandDocIDs(ctx context.Context, limit int) ([]string, error) { + return db.msgDocDatabase.GetRandDocIDs(ctx, limit) } func (db *commonMsgDatabase) GetCacheMaxSeqWithTime(ctx context.Context, conversationIDs []string) (map[string]database.SeqTime, error) { @@ -1026,3 +1012,7 @@ func (db *commonMsgDatabase) GetMaxSeqsWithTime(ctx context.Context, conversatio // todo: only the time in the redis cache will be taken, not the message time return db.seqConversation.GetMaxSeqsWithTime(ctx, conversationIDs) } + +func (db *commonMsgDatabase) DeleteDoc(ctx context.Context, docID string) error { + return db.msgDocDatabase.DeleteDoc(ctx, docID) +} diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index fc1fe47eab..937dbae1dd 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -1227,8 +1227,7 @@ func (m *MsgMgo) ConvertMsgsDocLen(ctx context.Context, conversationIDs []string } } -func (m *MsgMgo) GetDocIDs(ctx context.Context) ([]string, error) { - limit := 5000 +func (m *MsgMgo) GetRandDocIDs(ctx context.Context, limit int) ([]string, error) { var skip int var docIDs []string var offset int @@ -1267,15 +1266,18 @@ func (m *MsgMgo) GetDocIDs(ctx context.Context) ([]string, error) { return docIDs, errs.Wrap(err) } -func (m *MsgMgo) GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, limit int) ([]*model.MsgDocModel, error) { +func (m *MsgMgo) GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) { return mongoutil.Aggregate[*model.MsgDocModel](ctx, m.coll, []bson.M{ { "$match": bson.M{ - "doc_id": bson.M{ - "$in": docIDs, - }, - "msgs.msg.send_time": bson.M{ - "$lt": ts, + "msgs": bson.M{ + "$not": bson.M{ + "$elemMatch": bson.M{ + "msg.send_time": bson.M{ + "$gt": ts, + }, + }, + }, }, }, }, @@ -1288,7 +1290,7 @@ func (m *MsgMgo) GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, li }, }, { - "$limit": limit, + "$sample": limit, }, }) } diff --git a/pkg/common/storage/database/mgo/msg_test.go b/pkg/common/storage/database/mgo/msg_test.go index 5aed4dc511..eff847d262 100644 --- a/pkg/common/storage/database/mgo/msg_test.go +++ b/pkg/common/storage/database/mgo/msg_test.go @@ -3,12 +3,11 @@ package mgo import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/protocol/msg" - "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/db/mongoutil" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" + "math" "math/rand" "strconv" "testing" @@ -18,33 +17,43 @@ import ( func TestName1(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) defer cancel() - cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) - v := &MsgMgo{ - coll: cli.Database("openim_v3").Collection("msg3"), - } + //v := &MsgMgo{ + // coll: cli.Database("openim_v3").Collection("msg3"), + //} + // + //req := &msg.SearchMessageReq{ + // //RecvID: "3187706596", + // //SendID: "7009965934", + // ContentType: 101, + // //SendTime: "2024-05-06", + // //SessionType: 3, + // Pagination: &sdkws.RequestPagination{ + // PageNumber: 1, + // ShowNumber: 10, + // }, + //} + //total, res, err := v.SearchMessage(ctx, req) + //if err != nil { + // panic(err) + //} + // + //for i, re := range res { + // t.Logf("%d => %d | %+v", i+1, re.Msg.Seq, re.Msg.Content) + //} + // + //t.Log(total) - req := &msg.SearchMessageReq{ - //RecvID: "3187706596", - //SendID: "7009965934", - ContentType: 101, - //SendTime: "2024-05-06", - //SessionType: 3, - Pagination: &sdkws.RequestPagination{ - PageNumber: 1, - ShowNumber: 10, - }, - } - total, res, err := v.SearchMessage(ctx, req) + msg, err := NewMsgMongo(cli.Database("openim_v3")) if err != nil { panic(err) } - - for i, re := range res { - t.Logf("%d => %d | %+v", i+1, re.Msg.Seq, re.Msg.Content) + res, err := msg.GetBeforeMsg(ctx, time.Now().UnixMilli(), []string{"1:0"}, 1000) + if err != nil { + panic(err) } - - t.Log(total) + t.Log(len(res)) } func TestName10(t *testing.T) { @@ -73,3 +82,26 @@ func TestName10(t *testing.T) { } } + +func TestName3(t *testing.T) { + t.Log(uint64(math.MaxUint64)) + t.Log(int64(math.MaxInt64)) + + t.Log(int64(math.MinInt64)) +} + +func TestName4(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) + defer cancel() + cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + + msg, err := NewMsgMongo(cli.Database("openim_v3")) + if err != nil { + panic(err) + } + res, err := msg.GetBeforeMsg(ctx, time.Now().UnixMilli(), []string{"1:0"}, 1000) + if err != nil { + panic(err) + } + t.Log(len(res)) +} diff --git a/pkg/common/storage/database/msg.go b/pkg/common/storage/database/msg.go index 23a99f5b96..17aaf23429 100644 --- a/pkg/common/storage/database/msg.go +++ b/pkg/common/storage/database/msg.go @@ -45,7 +45,7 @@ type Msg interface { DeleteDoc(ctx context.Context, docID string) error DeleteMsgByIndex(ctx context.Context, docID string, index []int) error - GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, limit int) ([]*model.MsgDocModel, error) + GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) - GetDocIDs(ctx context.Context) ([]string, error) + GetRandDocIDs(ctx context.Context, limit int) ([]string, error) } From 8807597ffbba18544bab40021bbc90e87a6caadd Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 19 Dec 2024 17:15:15 +0800 Subject: [PATCH 06/17] refactoring scheduled tasks --- internal/api/router.go | 16 +++ internal/rpc/conversation/conversation.go | 29 ++++++ internal/rpc/msg/clear.go | 19 ++-- internal/tools/msg.go | 2 +- internal/tools/s3.go | 58 +++++++++-- internal/tools/user_msg.go | 31 +++--- pkg/common/storage/controller/conversation.go | 6 ++ pkg/common/storage/controller/msg.go | 6 ++ pkg/common/storage/database/conversation.go | 1 + .../storage/database/mgo/conversation.go | 32 ++++++ pkg/common/storage/database/mgo/msg.go | 99 ++++++++++--------- pkg/common/storage/database/mgo/msg_test.go | 41 ++++---- pkg/common/storage/database/msg.go | 2 + 13 files changed, 247 insertions(+), 95 deletions(-) diff --git a/internal/api/router.go b/internal/api/router.go index d6bf4e130b..33f16ade32 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -1,7 +1,9 @@ package api import ( + "context" "fmt" + "github.com/openimsdk/protocol/user" "net/http" "strings" @@ -164,6 +166,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En authRouterGroup.POST("/get_user_token", a.GetUserToken) authRouterGroup.POST("/parse_token", a.ParseToken) authRouterGroup.POST("/force_logout", a.ForceLogout) + } // Third service thirdGroup := r.Group("/third") @@ -297,3 +300,16 @@ var Whitelist = []string{ "/auth/get_admin_token", "/auth/parse_token", } + +func init() { + + var uc user.UserClient + var g *gin.Engine + g.POST("/get_admin_token", New(uc, uc.AccountCheck)) + +} + +func New[A, B, C any](c C, fn func(c C, ctx context.Context, req *A, opts ...grpc.CallOption) (*B, error)) func(c *gin.Context) { + + return nil +} diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index f534ca6dea..9fcd5abb1d 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -763,3 +763,32 @@ func (c *conversationServer) GetPinnedConversationIDs(ctx context.Context, req * } return &pbconversation.GetPinnedConversationIDsResp{ConversationIDs: conversationIDs}, nil } + +func (c *conversationServer) ClearUserConversationMsg(ctx context.Context, req *pbconversation.ClearUserConversationMsgReq) (*pbconversation.ClearUserConversationMsgResp, error) { + conversations, err := c.conversationDatabase.FindRandConversation(ctx, req.Timestamp, int(req.Limit)) + if err != nil { + return nil, err + } + for _, conversation := range conversations { + if conversation.IsMsgDestruct == false || conversation.MsgDestructTime == 0 { + continue + } + rcpReq := &pbmsg.GetLastMessageSeqByTimeReq{ConversationID: conversation.ConversationID, Time: req.Timestamp - conversation.MsgDestructTime} + resp, err := pbmsg.GetLastMessageSeqByTime.Invoke(ctx, rcpReq) + if err != nil { + return nil, err + } + if resp.Seq == 0 { + continue + } + _, err = c.SetConversationMinSeq(ctx, &pbconversation.SetConversationMinSeqReq{ + ConversationID: conversation.ConversationID, + OwnerUserID: []string{conversation.OwnerUserID}, + MinSeq: resp.Seq + 1, + }) + if err != nil { + return nil, err + } + } + return &pbconversation.ClearUserConversationMsgResp{}, nil +} diff --git a/internal/rpc/msg/clear.go b/internal/rpc/msg/clear.go index 8d4bd0c8ee..eb7a4b7a60 100644 --- a/internal/rpc/msg/clear.go +++ b/internal/rpc/msg/clear.go @@ -10,7 +10,6 @@ import ( pbconv "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/wrapperspb" - "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/utils/datautil" @@ -20,16 +19,10 @@ import ( ) // hard delete in Database. -func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) (_ *msg.DestructMsgsResp, err error) { +func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) (*msg.DestructMsgsResp, error) { if err := authverify.CheckAdmin(ctx, m.config.Share.IMAdminUserID); err != nil { return nil, err } - if req.Timestamp > time.Now().UnixMilli() { - return nil, errs.ErrArgs.WrapMsg("request millisecond timestamp error") - } - if req.Limit <= 0 { - return nil, errs.ErrArgs.WrapMsg("request limit error") - } docs, err := m.MsgDatabase.GetRandBeforeMsg(ctx, req.Timestamp, int(req.Limit)) if err != nil { return nil, err @@ -66,7 +59,7 @@ func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) } // soft delete for user self -func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg.ClearMsgResp, err error) { +func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (*msg.ClearMsgResp, error) { temp := convert.ConversationsPb2DB(req.Conversations) batchNum := 100 @@ -128,3 +121,11 @@ func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg. return nil, nil } + +func (m *msgServer) GetLastMessageSeqByTime(ctx context.Context, req *msg.GetLastMessageSeqByTimeReq) (*msg.GetLastMessageSeqByTimeResp, error) { + seq, err := m.MsgDatabase.GetLastMessageSeqByTime(ctx, req.ConversationID, req.Time) + if err != nil { + return nil, err + } + return &msg.GetLastMessageSeqByTimeResp{Seq: seq}, nil +} diff --git a/internal/tools/msg.go b/internal/tools/msg.go index 46f1573f28..5226496450 100644 --- a/internal/tools/msg.go +++ b/internal/tools/msg.go @@ -16,7 +16,7 @@ func (c *cronServer) deleteMsg() { ctx := mcontext.SetOperationID(c.ctx, operationID) log.ZDebug(ctx, "Destruct chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) const ( - deleteCount = 20 + deleteCount = 200 deleteLimit = 50 ) var count int diff --git a/internal/tools/s3.go b/internal/tools/s3.go index 3b0122c84e..469b0d0cb3 100644 --- a/internal/tools/s3.go +++ b/internal/tools/s3.go @@ -11,25 +11,69 @@ import ( func (c *cronServer) clearS3() { start := time.Now() - executeNum := 10 - // number of pagination. if need modify, need update value in third.DeleteOutdatedData - pageShowNumber := 500 deleteTime := start.Add(-time.Hour * 24 * time.Duration(c.config.CronTask.FileExpireTime)) operationID := fmt.Sprintf("cron_s3_%d_%d", os.Getpid(), deleteTime.UnixMilli()) ctx := mcontext.SetOperationID(c.ctx, operationID) log.ZDebug(ctx, "deleteoutDatedData", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) + const ( + deleteCount = 200 + deleteLimit = 100 + ) + var count int - for i := 1; i <= executeNum; i++ { - ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("%s_%d", operationID, i)) - resp, err := c.thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: c.config.CronTask.DeleteObjectType, Limit: int32(pageShowNumber)}) + for i := 1; i <= deleteCount; i++ { + resp, err := c.thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: c.config.CronTask.DeleteObjectType, Limit: deleteLimit}) if err != nil { log.ZError(ctx, "cron deleteoutDatedData failed", err) return } count += int(resp.Count) - if resp.Count < int32(pageShowNumber) { + if resp.Count < deleteLimit { break } } log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(start), "count", count) } + +// var req *third.DeleteOutdatedDataReq +// count1, err := ExtractField(ctx, c.thirdClient.DeleteOutdatedData, req, (*third.DeleteOutdatedDataResp).GetCount) +// +// c.thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{}) +// msggateway.GetUsersOnlineStatusCaller.Invoke(ctx, &msggateway.GetUsersOnlineStatusReq{}) +// +// var cli ThirdClient +// +// c111, err := cli.DeleteOutdatedData(ctx, 100) +// +// cli.ThirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{}) +// +// cli.AuthSign(ctx, &third.AuthSignReq{}) +// +// cli.SetAppBadge() +// +//} +// +//func extractField[A, B, C any](ctx context.Context, fn func(ctx context.Context, req *A, opts ...grpc.CallOption) (*B, error), req *A, get func(*B) C) (C, error) { +// resp, err := fn(ctx, req) +// if err != nil { +// var c C +// return c, err +// } +// return get(resp), nil +//} +// +//func ignore(_ any, err error) error { +// return err +//} +// +//type ThirdClient struct { +// third.ThirdClient +//} +// +//func (c *ThirdClient) DeleteOutdatedData(ctx context.Context, expireTime int64) (int32, error) { +// return extractField(ctx, c.ThirdClient.DeleteOutdatedData, &third.DeleteOutdatedDataReq{ExpireTime: expireTime}, (*third.DeleteOutdatedDataResp).GetCount) +//} +// +//func (c *ThirdClient) DeleteOutdatedData1(ctx context.Context, expireTime int64) error { +// return ignore(c.ThirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: expireTime})) +//} diff --git a/internal/tools/user_msg.go b/internal/tools/user_msg.go index 52a2b6bdce..3d4ca40d3f 100644 --- a/internal/tools/user_msg.go +++ b/internal/tools/user_msg.go @@ -3,7 +3,6 @@ package tools import ( "fmt" pbconversation "github.com/openimsdk/protocol/conversation" - "github.com/openimsdk/protocol/msg" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" "os" @@ -14,18 +13,22 @@ func (c *cronServer) clearUserMsg() { now := time.Now() operationID := fmt.Sprintf("cron_user_msg_%d_%d", os.Getpid(), now.UnixMilli()) ctx := mcontext.SetOperationID(c.ctx, operationID) - log.ZDebug(ctx, "clear msg cron start") - conversations, err := c.conversationClient.GetConversationsNeedClearMsg(ctx, &pbconversation.GetConversationsNeedClearMsgReq{}) - if err != nil { - log.ZError(ctx, "Get conversation need Destruct msgs failed.", err) - return + log.ZDebug(ctx, "clear user msg cron start") + const ( + deleteCount = 200 + deleteLimit = 100 + ) + var count int + for i := 1; i <= deleteCount; i++ { + resp, err := c.conversationClient.ClearUserConversationMsg(ctx, &pbconversation.ClearUserConversationMsgReq{Timestamp: now.UnixMilli(), Limit: deleteLimit}) + if err != nil { + log.ZError(ctx, "ClearUserConversationMsg failed.", err) + return + } + count += int(resp.Count) + if resp.Count < deleteLimit { + break + } } - - _, err = c.msgClient.ClearMsg(ctx, &msg.ClearMsgReq{Conversations: conversations.Conversations}) - if err != nil { - log.ZError(ctx, "Clear Msg failed.", err) - return - } - - log.ZDebug(ctx, "clear msg cron task completed", "cont", time.Since(now)) + log.ZDebug(ctx, "clear user msg cron task completed", "cont", time.Since(now), "count", count) } diff --git a/pkg/common/storage/controller/conversation.go b/pkg/common/storage/controller/conversation.go index bf41cce957..d4088e0c0e 100644 --- a/pkg/common/storage/controller/conversation.go +++ b/pkg/common/storage/controller/conversation.go @@ -74,6 +74,8 @@ type ConversationDatabase interface { GetNotNotifyConversationIDs(ctx context.Context, userID string) ([]string, error) // GetPinnedConversationIDs gets pinned conversationIDs by userID GetPinnedConversationIDs(ctx context.Context, userID string) ([]string, error) + // FindRandConversation finds random conversations based on the specified timestamp and limit. + FindRandConversation(ctx context.Context, ts int64, limit int) ([]*relationtb.Conversation, error) } func NewConversationDatabase(conversation database.Conversation, cache cache.ConversationCache, tx tx.Tx) ConversationDatabase { @@ -401,3 +403,7 @@ func (c *conversationDatabase) GetPinnedConversationIDs(ctx context.Context, use } return conversationIDs, nil } + +func (c *conversationDatabase) FindRandConversation(ctx context.Context, ts int64, limit int) ([]*relationtb.Conversation, error) { + return c.conversationDB.FindRandConversation(ctx, ts, limit) +} diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 88089f1a09..c29544c333 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -103,6 +103,8 @@ type CommonMsgDatabase interface { SetUserConversationsMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error DeleteDoc(ctx context.Context, docID string) error + + GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) } func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { @@ -1016,3 +1018,7 @@ func (db *commonMsgDatabase) GetMaxSeqsWithTime(ctx context.Context, conversatio func (db *commonMsgDatabase) DeleteDoc(ctx context.Context, docID string) error { return db.msgDocDatabase.DeleteDoc(ctx, docID) } + +func (db *commonMsgDatabase) GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) { + return db.msgDocDatabase.GetLastMessageSeqByTime(ctx, conversationID, time) +} diff --git a/pkg/common/storage/database/conversation.go b/pkg/common/storage/database/conversation.go index 30ca01ee71..1fb53cfed2 100644 --- a/pkg/common/storage/database/conversation.go +++ b/pkg/common/storage/database/conversation.go @@ -42,4 +42,5 @@ type Conversation interface { GetConversationIDsNeedDestruct(ctx context.Context) ([]*model.Conversation, error) GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) + FindRandConversation(ctx context.Context, ts int64, limit int) ([]*model.Conversation, error) } diff --git a/pkg/common/storage/database/mgo/conversation.go b/pkg/common/storage/database/mgo/conversation.go index 10e223c893..851ec99c40 100644 --- a/pkg/common/storage/database/mgo/conversation.go +++ b/pkg/common/storage/database/mgo/conversation.go @@ -228,3 +228,35 @@ func (c *ConversationMgo) GetConversationNotReceiveMessageUserIDs(ctx context.Co func (c *ConversationMgo) FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) { return c.version.FindChangeLog(ctx, userID, version, limit) } + +func (c *ConversationMgo) FindRandConversation(ctx context.Context, ts int64, limit int) ([]*model.Conversation, error) { + pipeline := []bson.M{ + { + "$match": bson.M{ + "is_msg_destruct": true, + "msg_destruct_time": bson.M{"$ne": 0}, + }, + }, + { + "$addFields": bson.M{ + "next_msg_destruct_timestamp": bson.M{ + "$add": []any{ + bson.M{ + "$toLong": "$latest_msg_destruct_time", + }, "$msg_destruct_time"}, + }, + }, + }, + { + "$match": bson.M{ + "next_msg_destruct_timestamp": bson.M{"$lt": ts}, + }, + }, + { + "$sample": bson.M{ + "size": limit, + }, + }, + } + return mongoutil.Aggregate[*model.Conversation](ctx, c.coll, pipeline) +} diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index 937dbae1dd..e014654663 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -1290,7 +1290,9 @@ func (m *MsgMgo) GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]* }, }, { - "$sample": limit, + "$sample": bson.M{ + "size": limit, + }, }, }) } @@ -1307,53 +1309,56 @@ func (m *MsgMgo) DeleteMsgByIndex(ctx context.Context, docID string, index []int return mongoutil.UpdateOne(ctx, m.coll, bson.M{"doc_id": docID}, bson.M{"$set": set}, true) } -//func (m *MsgMgo) ClearMsg(ctx context.Context, t time.Time) (int64, error) { -// ts := t.UnixMilli() -// var count int64 -// for { -// msgs, err := m.GetBeforeMsg(ctx, ts, 100) -// if err != nil { -// return count, err -// } -// if len(msgs) == 0 { -// return count, nil -// } -// for _, msg := range msgs { -// num, err := m.deleteOneMsg(ctx, ts, msg) -// count += num -// if err != nil { -// return count, err -// } -// } -// } -//} - func (m *MsgMgo) DeleteDoc(ctx context.Context, docID string) error { return mongoutil.DeleteOne(ctx, m.coll, bson.M{"doc_id": docID}) } -//func (m *MsgMgo) DeleteDocMsg(ctx context.Context, ts int64, doc *relation.MsgDocModel) (int64, error) { -// var notNull int -// index := make([]int, 0, len(doc.Msg)) -// for i, message := range doc.Msg { -// if message.Msg != nil { -// notNull++ -// if message.Msg.SendTime < ts { -// index = append(index, i) -// } -// } -// } -// if len(index) == 0 { -// return 0, errs.New("no msg to delete").WrapMsg("deleteOneMsg", "docID", doc.DocID) -// } -// if len(index) == notNull { -// if err := m.DeleteDoc(ctx, doc.DocID); err != nil { -// return 0, err -// } -// } else { -// if err := m.setNullMsg(ctx, doc.DocID, index); err != nil { -// return 0, err -// } -// } -// return int64(len(index)), nil -//} +func (m *MsgMgo) GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) { + pipeline := []bson.M{ + { + "$match": bson.M{ + "doc_id": bson.M{ + "$regex": fmt.Sprintf("^%s:", conversationID), + }, + }, + }, + { + "$match": bson.M{ + "msgs.msg.send_time": bson.M{ + "msgs.msg.send_time": bson.M{ + "$lte": time, + }, + }, + }, + }, + { + "$sort": bson.M{ + "_id": -1, + }, + }, + { + "$limit": 1, + }, + { + "$project": bson.M{ + "_id": 0, + "msgs.msg.send_time": 1, + "msgs.msg.seq": 1, + }, + }, + } + res, err := mongoutil.Aggregate[*model.MsgDocModel](ctx, m.coll, pipeline) + if err != nil { + return 0, err + } + if len(res) == 0 { + return 0, nil + } + var seq int64 + for _, v := range res[0].Msg { + if v.Msg.SendTime <= time { + seq = v.Msg.Seq + } + } + return seq, nil +} diff --git a/pkg/common/storage/database/mgo/msg_test.go b/pkg/common/storage/database/mgo/msg_test.go index eff847d262..7218ef1ad1 100644 --- a/pkg/common/storage/database/mgo/msg_test.go +++ b/pkg/common/storage/database/mgo/msg_test.go @@ -15,10 +15,10 @@ import ( ) func TestName1(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) - defer cancel() - cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) - + //ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) + //defer cancel() + //cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + // //v := &MsgMgo{ // coll: cli.Database("openim_v3").Collection("msg3"), //} @@ -44,16 +44,16 @@ func TestName1(t *testing.T) { //} // //t.Log(total) - - msg, err := NewMsgMongo(cli.Database("openim_v3")) - if err != nil { - panic(err) - } - res, err := msg.GetBeforeMsg(ctx, time.Now().UnixMilli(), []string{"1:0"}, 1000) - if err != nil { - panic(err) - } - t.Log(len(res)) + // + //msg, err := NewMsgMongo(cli.Database("openim_v3")) + //if err != nil { + // panic(err) + //} + //res, err := msg.GetBeforeMsg(ctx, time.Now().UnixMilli(), []string{"1:0"}, 1000) + //if err != nil { + // panic(err) + //} + //t.Log(len(res)) } func TestName10(t *testing.T) { @@ -95,13 +95,20 @@ func TestName4(t *testing.T) { defer cancel() cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) - msg, err := NewMsgMongo(cli.Database("openim_v3")) + msg, err := NewConversationMongo(cli.Database("openim_v3")) if err != nil { panic(err) } - res, err := msg.GetBeforeMsg(ctx, time.Now().UnixMilli(), []string{"1:0"}, 1000) + ts := time.Now().UnixMilli() + t.Log(ts) + res, err := msg.FindRandConversation(ctx, ts, 10) if err != nil { panic(err) } - t.Log(len(res)) + t.Log(res) +} + +func TestName5(t *testing.T) { + var v time.Time + t.Log(v.UnixMilli()) } diff --git a/pkg/common/storage/database/msg.go b/pkg/common/storage/database/msg.go index 17aaf23429..abb2a44c2f 100644 --- a/pkg/common/storage/database/msg.go +++ b/pkg/common/storage/database/msg.go @@ -48,4 +48,6 @@ type Msg interface { GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) GetRandDocIDs(ctx context.Context, limit int) ([]string, error) + + GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) } From 1949d6c75684e7fd0bbd349472fac581c8111725 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 19 Dec 2024 17:34:34 +0800 Subject: [PATCH 07/17] refactoring scheduled tasks --- internal/api/router.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/internal/api/router.go b/internal/api/router.go index 33f16ade32..5e8038068c 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -1,9 +1,7 @@ package api import ( - "context" "fmt" - "github.com/openimsdk/protocol/user" "net/http" "strings" @@ -300,16 +298,3 @@ var Whitelist = []string{ "/auth/get_admin_token", "/auth/parse_token", } - -func init() { - - var uc user.UserClient - var g *gin.Engine - g.POST("/get_admin_token", New(uc, uc.AccountCheck)) - -} - -func New[A, B, C any](c C, fn func(c C, ctx context.Context, req *A, opts ...grpc.CallOption) (*B, error)) func(c *gin.Context) { - - return nil -} From 86d58253c96e0e7ac7a1285df46c816a803c366f Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 19 Dec 2024 18:53:49 +0800 Subject: [PATCH 08/17] refactoring scheduled tasks --- internal/rpc/third/s3.go | 21 +++++--- internal/tools/cron_test.go | 63 +++++++++++++++++++++++ internal/tools/msg.go | 6 +-- internal/tools/s3.go | 2 +- internal/tools/user_msg.go | 2 +- pkg/common/storage/controller/s3.go | 6 +++ pkg/common/storage/database/mgo/object.go | 4 ++ pkg/common/storage/database/object.go | 1 + 8 files changed, 92 insertions(+), 13 deletions(-) create mode 100644 internal/tools/cron_test.go diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index a541462a28..e1b42ca338 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -295,18 +295,23 @@ func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteO if err != nil { return nil, err } - if len(models) > 0 { - names := datautil.Batch(func(o *model.Object) string { - return o.Name - }, models) - if err := t.s3dataBase.DeleteSpecifiedData(ctx, engine, names); err != nil { + keyCount := make(map[string]int) + for _, obj := range models { + keyCount[obj.Key]++ + } + for _, obj := range models { + count, err := t.s3dataBase.GetKeyCount(ctx, engine, obj.Key) + if err != nil { + return nil, err + } + if err := t.s3dataBase.DeleteSpecifiedData(ctx, engine, []string{obj.Name}); err != nil { return nil, errs.Wrap(err) } - if err := t.s3dataBase.DelS3Key(ctx, engine, names...); err != nil { + if err := t.s3dataBase.DelS3Key(ctx, engine, obj.Name); err != nil { return nil, err } - for _, object := range models { - if err := t.s3.DeleteObject(ctx, object.Key); err != nil { + if int(count) <= keyCount[obj.Key] { + if err := t.s3.DeleteObject(ctx, obj.Key); err != nil { return nil, err } } diff --git a/internal/tools/cron_test.go b/internal/tools/cron_test.go new file mode 100644 index 0000000000..8903490698 --- /dev/null +++ b/internal/tools/cron_test.go @@ -0,0 +1,63 @@ +package tools + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" + pbconversation "github.com/openimsdk/protocol/conversation" + "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/third" + "github.com/openimsdk/tools/mcontext" + "github.com/openimsdk/tools/mw" + "github.com/robfig/cron/v3" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "testing" +) + +func TestName(t *testing.T) { + conf := &config.Discovery{ + Enable: config.ETCD, + Etcd: config.Etcd{ + RootDirectory: "openim", + Address: []string{"localhost:12379"}, + }, + } + client, err := kdisc.NewDiscoveryRegister(conf, "source") + if err != nil { + panic(err) + } + client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials())) + ctx := mcontext.SetOpUserID(context.Background(), "imAdmin") + msgConn, err := client.GetConn(ctx, "msg-rpc-service") + if err != nil { + panic(err) + } + thirdConn, err := client.GetConn(ctx, "third-rpc-service") + if err != nil { + panic(err) + } + + conversationConn, err := client.GetConn(ctx, "conversation-rpc-service") + if err != nil { + panic(err) + } + + srv := &cronServer{ + ctx: ctx, + config: &CronTaskConfig{ + CronTask: config.CronTask{ + RetainChatRecords: 1, + FileExpireTime: 1, + DeleteObjectType: []string{"msg-picture", "msg-file", "msg-voice", "msg-video", "msg-video-snapshot", "sdklog", ""}, + }, + }, + cron: cron.New(), + msgClient: msg.NewMsgClient(msgConn), + conversationClient: pbconversation.NewConversationClient(conversationConn), + thirdClient: third.NewThirdClient(thirdConn), + } + srv.deleteMsg() + //srv.clearS3() + //srv.clearUserMsg() +} diff --git a/internal/tools/msg.go b/internal/tools/msg.go index 5226496450..cc00cc5b83 100644 --- a/internal/tools/msg.go +++ b/internal/tools/msg.go @@ -16,7 +16,7 @@ func (c *cronServer) deleteMsg() { ctx := mcontext.SetOperationID(c.ctx, operationID) log.ZDebug(ctx, "Destruct chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) const ( - deleteCount = 200 + deleteCount = 10000 deleteLimit = 50 ) var count int @@ -28,9 +28,9 @@ func (c *cronServer) deleteMsg() { break } count += int(resp.Count) - if resp.Count <= deleteLimit { + if resp.Count < deleteLimit { break } } - log.ZDebug(ctx, "cron destruct chat records end", "deltime", deltime, "cont", time.Since(now), "deleteDocs", count) + log.ZDebug(ctx, "cron destruct chat records end", "deltime", deltime, "cont", time.Since(now), "count", count) } diff --git a/internal/tools/s3.go b/internal/tools/s3.go index 469b0d0cb3..9b6b9c4089 100644 --- a/internal/tools/s3.go +++ b/internal/tools/s3.go @@ -16,7 +16,7 @@ func (c *cronServer) clearS3() { ctx := mcontext.SetOperationID(c.ctx, operationID) log.ZDebug(ctx, "deleteoutDatedData", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) const ( - deleteCount = 200 + deleteCount = 10000 deleteLimit = 100 ) diff --git a/internal/tools/user_msg.go b/internal/tools/user_msg.go index 3d4ca40d3f..a4afa769ed 100644 --- a/internal/tools/user_msg.go +++ b/internal/tools/user_msg.go @@ -15,7 +15,7 @@ func (c *cronServer) clearUserMsg() { ctx := mcontext.SetOperationID(c.ctx, operationID) log.ZDebug(ctx, "clear user msg cron start") const ( - deleteCount = 200 + deleteCount = 10000 deleteLimit = 100 ) var count int diff --git a/pkg/common/storage/controller/s3.go b/pkg/common/storage/controller/s3.go index 6a0f11b1ce..6693d2ddea 100644 --- a/pkg/common/storage/controller/s3.go +++ b/pkg/common/storage/controller/s3.go @@ -42,6 +42,7 @@ type S3Database interface { FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) DeleteSpecifiedData(ctx context.Context, engine string, name []string) error DelS3Key(ctx context.Context, engine string, keys ...string) error + GetKeyCount(ctx context.Context, engine string, key string) (int64, error) } func NewS3Database(rdb redis.UniversalClient, s3 s3.Interface, obj database.ObjectInfo) S3Database { @@ -117,10 +118,15 @@ func (s *s3Database) StatObject(ctx context.Context, name string) (*s3.ObjectInf func (s *s3Database) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { return s.s3.FormData(ctx, name, size, contentType, duration) } + func (s *s3Database) FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) { return s.db.FindExpirationObject(ctx, engine, expiration, needDelType, count) } +func (s *s3Database) GetKeyCount(ctx context.Context, engine string, key string) (int64, error) { + return s.db.GetKeyCount(ctx, engine, key) +} + func (s *s3Database) DeleteSpecifiedData(ctx context.Context, engine string, name []string) error { return s.db.Delete(ctx, engine, name) } diff --git a/pkg/common/storage/database/mgo/object.go b/pkg/common/storage/database/mgo/object.go index 53f2a6ba43..624eb84a0d 100644 --- a/pkg/common/storage/database/mgo/object.go +++ b/pkg/common/storage/database/mgo/object.go @@ -108,3 +108,7 @@ func (o *S3Mongo) FindExpirationObject(ctx context.Context, engine string, expir "group": bson.M{"$in": needDelType}, }, opt) } + +func (o *S3Mongo) GetKeyCount(ctx context.Context, engine string, key string) (int64, error) { + return mongoutil.Count(ctx, o.coll, bson.M{"engine": engine, "key": key}) +} diff --git a/pkg/common/storage/database/object.go b/pkg/common/storage/database/object.go index 86d47a2a6b..5541a159b4 100644 --- a/pkg/common/storage/database/object.go +++ b/pkg/common/storage/database/object.go @@ -26,4 +26,5 @@ type ObjectInfo interface { Take(ctx context.Context, engine string, name string) (*model.Object, error) Delete(ctx context.Context, engine string, name []string) error FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) + GetKeyCount(ctx context.Context, engine string, key string) (int64, error) } From 46a8d171049faefba46f0c3cc19a9c4099b35cc7 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 20 Dec 2024 15:26:32 +0800 Subject: [PATCH 09/17] refactoring scheduled tasks --- go.mod | 4 +-- go.sum | 8 ++--- internal/rpc/conversation/conversation.go | 39 ++++++++++++++++----- internal/rpc/msg/clear.go | 5 ++- internal/rpc/third/s3.go | 17 ++++----- pkg/common/storage/database/mgo/msg.go | 10 +++--- pkg/common/storage/database/mgo/msg_test.go | 6 ++-- 7 files changed, 56 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index 4eaa18ccc8..6c1d421c84 100644 --- a/go.mod +++ b/go.mod @@ -14,8 +14,8 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.66 - github.com/openimsdk/tools v0.0.50-alpha.57 + github.com/openimsdk/protocol v0.0.72-alpha.67 + github.com/openimsdk/tools v0.0.50-alpha.58 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 6e283eb668..86ed9ed6e3 100644 --- a/go.sum +++ b/go.sum @@ -347,10 +347,10 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.66 h1:5KoDY6M4T+pXg449ScF6hqeQ+WenBwNyUJn/t8W0oBQ= -github.com/openimsdk/protocol v0.0.72-alpha.66/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.57 h1:oIKV6vYhqp7TRmZ6Pe+r9RNl1D5s7aB/kE9yQVEWcSY= -github.com/openimsdk/tools v0.0.50-alpha.57/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= +github.com/openimsdk/protocol v0.0.72-alpha.67 h1:zlLbVkoT0OYsjO2RCutQuDFllcfNvZfdYchvlR6UIe0= +github.com/openimsdk/protocol v0.0.72-alpha.67/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= +github.com/openimsdk/tools v0.0.50-alpha.58 h1:hkFL02Bzzp/l5x+tb7kJ9zes7hilh65EQ4qEIthsQX4= +github.com/openimsdk/tools v0.0.50-alpha.58/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index 9fcd5abb1d..696ada1521 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -769,7 +769,8 @@ func (c *conversationServer) ClearUserConversationMsg(ctx context.Context, req * if err != nil { return nil, err } - for _, conversation := range conversations { + latestMsgDestructTime := time.UnixMilli(req.Timestamp) + for i, conversation := range conversations { if conversation.IsMsgDestruct == false || conversation.MsgDestructTime == 0 { continue } @@ -778,17 +779,37 @@ func (c *conversationServer) ClearUserConversationMsg(ctx context.Context, req * if err != nil { return nil, err } - if resp.Seq == 0 { + if resp.Seq <= 0 { + log.ZDebug(ctx, "ClearUserConversationMsg GetLastMessageSeqByTime seq <= 0", "index", i, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID, "msgDestructTime", conversation.MsgDestructTime, "seq", resp.Seq) + if err := c.setConversationMinSeqAndLatestMsgDestructTime(ctx, conversation.ConversationID, conversation.OwnerUserID, -1, latestMsgDestructTime); err != nil { + return nil, err + } continue } - _, err = c.SetConversationMinSeq(ctx, &pbconversation.SetConversationMinSeqReq{ - ConversationID: conversation.ConversationID, - OwnerUserID: []string{conversation.OwnerUserID}, - MinSeq: resp.Seq + 1, - }) - if err != nil { + resp.Seq++ + if err := c.setConversationMinSeqAndLatestMsgDestructTime(ctx, conversation.ConversationID, conversation.OwnerUserID, resp.Seq, latestMsgDestructTime); err != nil { return nil, err } + log.ZDebug(ctx, "ClearUserConversationMsg set min seq", "index", i, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID, "seq", resp.Seq, "msgDestructTime", conversation.MsgDestructTime) + } + return &pbconversation.ClearUserConversationMsgResp{Count: int32(len(conversations))}, nil +} + +func (c *conversationServer) setConversationMinSeqAndLatestMsgDestructTime(ctx context.Context, conversationID string, ownerUserID string, minSeq int64, latestMsgDestructTime time.Time) error { + update := map[string]any{ + "latest_msg_destruct_time": latestMsgDestructTime, } - return &pbconversation.ClearUserConversationMsgResp{}, nil + if minSeq >= 0 { + req := &pbmsg.SetUserConversationMinSeqReq{ConversationID: conversationID, OwnerUserID: []string{ownerUserID}, MinSeq: minSeq} + if _, err := pbmsg.SetUserConversationMinSeqCaller.Invoke(ctx, req); err != nil { + return err + } + update["min_seq"] = minSeq + } + + if err := c.conversationDatabase.UpdateUsersConversationField(ctx, []string{ownerUserID}, conversationID, update); err != nil { + return err + } + c.conversationNotificationSender.ConversationChangeNotification(ctx, ownerUserID, []string{conversationID}) + return nil } diff --git a/internal/rpc/msg/clear.go b/internal/rpc/msg/clear.go index eb7a4b7a60..7a2d363009 100644 --- a/internal/rpc/msg/clear.go +++ b/internal/rpc/msg/clear.go @@ -27,10 +27,11 @@ func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) if err != nil { return nil, err } - for _, doc := range docs { + for i, doc := range docs { if err := m.MsgDatabase.DeleteDoc(ctx, doc.DocID); err != nil { return nil, err } + log.ZDebug(ctx, "DestructMsgs delete doc", "index", i, "docID", doc.DocID) index := strings.LastIndex(doc.DocID, ":") if index < 0 { continue @@ -51,9 +52,11 @@ func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) if conversationID == "" { continue } + minSeq++ if err := m.MsgDatabase.SetMinSeq(ctx, conversationID, minSeq); err != nil { return nil, err } + log.ZDebug(ctx, "DestructMsgs delete doc set min seq", "index", i, "docID", doc.DocID, "conversationID", conversationID, "setMinSeq", minSeq) } return &msg.DestructMsgsResp{Count: int32(len(docs))}, nil } diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index e1b42ca338..8796fe824e 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -295,22 +295,19 @@ func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteO if err != nil { return nil, err } - keyCount := make(map[string]int) - for _, obj := range models { - keyCount[obj.Key]++ - } - for _, obj := range models { - count, err := t.s3dataBase.GetKeyCount(ctx, engine, obj.Key) - if err != nil { - return nil, err - } + for i, obj := range models { if err := t.s3dataBase.DeleteSpecifiedData(ctx, engine, []string{obj.Name}); err != nil { return nil, errs.Wrap(err) } if err := t.s3dataBase.DelS3Key(ctx, engine, obj.Name); err != nil { return nil, err } - if int(count) <= keyCount[obj.Key] { + count, err := t.s3dataBase.GetKeyCount(ctx, engine, obj.Key) + if err != nil { + return nil, err + } + log.ZDebug(ctx, "delete s3 object record", "index", i, "s3", obj, "count", count) + if count == 0 { if err := t.s3.DeleteObject(ctx, obj.Key); err != nil { return nil, err } diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index e014654663..f371766958 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -1318,16 +1318,14 @@ func (m *MsgMgo) GetLastMessageSeqByTime(ctx context.Context, conversationID str { "$match": bson.M{ "doc_id": bson.M{ - "$regex": fmt.Sprintf("^%s:", conversationID), + "$regex": fmt.Sprintf("^%s", conversationID), }, }, }, { "$match": bson.M{ "msgs.msg.send_time": bson.M{ - "msgs.msg.send_time": bson.M{ - "$lte": time, - }, + "$lte": time, }, }, }, @@ -1342,6 +1340,7 @@ func (m *MsgMgo) GetLastMessageSeqByTime(ctx context.Context, conversationID str { "$project": bson.M{ "_id": 0, + "doc_id": 1, "msgs.msg.send_time": 1, "msgs.msg.seq": 1, }, @@ -1356,6 +1355,9 @@ func (m *MsgMgo) GetLastMessageSeqByTime(ctx context.Context, conversationID str } var seq int64 for _, v := range res[0].Msg { + if v.Msg == nil { + continue + } if v.Msg.SendTime <= time { seq = v.Msg.Seq } diff --git a/pkg/common/storage/database/mgo/msg_test.go b/pkg/common/storage/database/mgo/msg_test.go index 7218ef1ad1..992090552e 100644 --- a/pkg/common/storage/database/mgo/msg_test.go +++ b/pkg/common/storage/database/mgo/msg_test.go @@ -95,13 +95,13 @@ func TestName4(t *testing.T) { defer cancel() cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) - msg, err := NewConversationMongo(cli.Database("openim_v3")) + msg, err := NewMsgMongo(cli.Database("openim_v3")) if err != nil { panic(err) } - ts := time.Now().UnixMilli() + ts := time.Now().Add(-time.Hour * 24 * 5).UnixMilli() t.Log(ts) - res, err := msg.FindRandConversation(ctx, ts, 10) + res, err := msg.GetLastMessageSeqByTime(ctx, "sg_1523453548", ts) if err != nil { panic(err) } From 2e8ca5f5b938fa929b0123c2a66d246aa3537ed2 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 24 Dec 2024 17:50:05 +0800 Subject: [PATCH 10/17] upgrading pkg tools --- go.mod | 2 +- go.sum | 4 ++-- internal/msggateway/client.go | 4 ++-- internal/msgtransfer/init.go | 2 +- internal/msgtransfer/online_history_msg_handler.go | 2 +- internal/rpc/msg/send.go | 2 +- pkg/rpccache/subscriber.go | 3 ++- 7 files changed, 10 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 26f167a19e..05fe6db15e 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.68 - github.com/openimsdk/tools v0.0.50-alpha.60 + github.com/openimsdk/tools v0.0.50-alpha.61 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index c0385481ca..1fc3c33db3 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrk github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.68 h1:Ekn6S9Ftt12Xs/p9kJ39RDr2gSwIczz+MmSHQE4lAek= github.com/openimsdk/protocol v0.0.72-alpha.68/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.60 h1:dYqYpSdSN5o6CxlEjua2USfwfUiG0tUWFBpqghTjbWE= -github.com/openimsdk/tools v0.0.50-alpha.60/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= +github.com/openimsdk/tools v0.0.50-alpha.61 h1:zKEZwrj+fUVuyC6KR3kZp9zFaCCIFgoSbHO0r0mZ6h4= +github.com/openimsdk/tools v0.0.50-alpha.61/go.mod h1:JowL2jYr8tu4vcQe+5hJh4v3BtSx1T0CIS3pgU/Mw+U= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 161300d6d0..1040f2be23 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -131,7 +131,7 @@ func (c *Client) readMessage() { defer func() { if r := recover(); r != nil { c.closedErr = ErrPanic - log.ZPanic(c.ctx, "socket have panic err:", r) + log.ZPanic(c.ctx, "socket have panic err:", errs.ErrPanic(r)) } c.close() }() @@ -376,7 +376,7 @@ func (c *Client) activeHeartbeat(ctx context.Context) { go func() { defer func() { if r := recover(); r != nil { - log.ZPanic(ctx, "activeHeartbeat Panic", r) + log.ZPanic(ctx, "activeHeartbeat Panic", errs.ErrPanic(r)) } }() log.ZDebug(ctx, "server initiative send heartbeat start.") diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 5cb6131238..ee9f066448 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -195,7 +195,7 @@ func (m *MsgTransfer) Start(index int, config *Config, client discovery.SvcDisco go func() { defer func() { if r := recover(); r != nil { - log.ZPanic(m.ctx, "MsgTransfer Start Panic", r) + log.ZPanic(m.ctx, "MsgTransfer Start Panic", errs.ErrPanic(r)) } }() if err := prommetrics.TransferInit(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index 83b0750613..7b989070b0 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -361,7 +361,7 @@ func (och *OnlineHistoryRedisConsumerHandler) handleNotification(ctx context.Con func (och *OnlineHistoryRedisConsumerHandler) HandleUserHasReadSeqMessages(ctx context.Context) { defer func() { if r := recover(); r != nil { - log.ZPanic(ctx, "HandleUserHasReadSeqMessages Panic", r) + log.ZPanic(ctx, "HandleUserHasReadSeqMessages Panic", errs.ErrPanic(r)) } }() diff --git a/internal/rpc/msg/send.go b/internal/rpc/msg/send.go index b9bbf615f3..f01134f8f0 100644 --- a/internal/rpc/msg/send.go +++ b/internal/rpc/msg/send.go @@ -88,7 +88,7 @@ func (m *msgServer) setConversationAtInfo(nctx context.Context, msg *sdkws.MsgDa defer func() { if r := recover(); r != nil { - log.ZPanic(nctx, "setConversationAtInfo Panic", r) + log.ZPanic(nctx, "setConversationAtInfo Panic", errs.ErrPanic(r)) } }() diff --git a/pkg/rpccache/subscriber.go b/pkg/rpccache/subscriber.go index d28d1aa295..0cb35bebfb 100644 --- a/pkg/rpccache/subscriber.go +++ b/pkg/rpccache/subscriber.go @@ -17,6 +17,7 @@ package rpccache import ( "context" "encoding/json" + "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" ) @@ -24,7 +25,7 @@ import ( func subscriberRedisDeleteCache(ctx context.Context, client redis.UniversalClient, channel string, del func(ctx context.Context, key ...string)) { defer func() { if r := recover(); r != nil { - log.ZPanic(ctx, "subscriberRedisDeleteCache Panic", r) + log.ZPanic(ctx, "subscriberRedisDeleteCache Panic", errs.ErrPanic(r)) } }() for message := range client.Subscribe(ctx, channel).Channel() { From 4d1655e32ca39b8ac6c7b38716abec828bf9d053 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 25 Dec 2024 18:30:58 +0800 Subject: [PATCH 11/17] redis msg cache --- internal/rpc/msg/delete.go | 6 +- pkg/common/storage/cache/cachekey/msg.go | 39 +-- pkg/common/storage/cache/msg.go | 14 +- pkg/common/storage/cache/redis/msg.go | 105 +++---- pkg/common/storage/controller/msg.go | 365 ++++++++-------------- pkg/common/storage/database/mgo/msg.go | 371 ++++------------------- pkg/common/storage/database/msg.go | 5 +- pkg/common/storage/model/msg.go | 4 + 8 files changed, 233 insertions(+), 676 deletions(-) diff --git a/internal/rpc/msg/delete.go b/internal/rpc/msg/delete.go index f45713ff12..d3485faaa8 100644 --- a/internal/rpc/msg/delete.go +++ b/internal/rpc/msg/delete.go @@ -106,10 +106,8 @@ func (m *msgServer) DeleteMsgPhysical(ctx context.Context, req *msg.DeleteMsgPhy return nil, err } remainTime := timeutil.GetCurrentTimestampBySecond() - req.Timestamp - for _, conversationID := range req.ConversationIDs { - if err := m.MsgDatabase.DeleteConversationMsgsAndSetMinSeq(ctx, conversationID, remainTime); err != nil { - log.ZWarn(ctx, "DeleteConversationMsgsAndSetMinSeq error", err, "conversationID", conversationID, "err", err) - } + if _, err := m.DestructMsgs(ctx, &msg.DestructMsgsReq{Timestamp: remainTime, Limit: 9999}); err != nil { + return nil, err } return &msg.DeleteMsgPhysicalResp{}, nil } diff --git a/pkg/common/storage/cache/cachekey/msg.go b/pkg/common/storage/cache/cachekey/msg.go index 8e05b64f1f..4c8b32cd05 100644 --- a/pkg/common/storage/cache/cachekey/msg.go +++ b/pkg/common/storage/cache/cachekey/msg.go @@ -15,50 +15,21 @@ package cachekey import ( - "github.com/openimsdk/protocol/constant" "strconv" ) const ( - messageCache = "MESSAGE_CACHE:" - messageDelUserList = "MESSAGE_DEL_USER_LIST:" - userDelMessagesList = "USER_DEL_MESSAGES_LIST:" - sendMsgFailedFlag = "SEND_MSG_FAILED_FLAG:" - exTypeKeyLocker = "EX_LOCK:" - reactionExSingle = "EX_SINGLE_" - reactionWriteGroup = "EX_GROUP_" - reactionReadGroup = "EX_SUPER_GROUP_" - reactionNotification = "EX_NOTIFICATION_" + messageCache = "MESSAGE_CACHE:" + sendMsgFailedFlag = "SEND_MSG_FAILED_FLAG:" + messageCacheV2 = "MESSAGE_CACHE_V2:" ) func GetMessageCacheKey(conversationID string, seq int64) string { return messageCache + conversationID + "_" + strconv.Itoa(int(seq)) } -func GetMessageDelUserListKey(conversationID string, seq int64) string { - return messageDelUserList + conversationID + ":" + strconv.Itoa(int(seq)) -} - -func GetUserDelListKey(conversationID, userID string) string { - return userDelMessagesList + conversationID + ":" + userID -} - -func GetMessageReactionExKey(clientMsgID string, sessionType int32) string { - switch sessionType { - case constant.SingleChatType: - return reactionExSingle + clientMsgID - case constant.WriteGroupChatType: - return reactionWriteGroup + clientMsgID - case constant.ReadGroupChatType: - return reactionReadGroup + clientMsgID - case constant.NotificationChatType: - return reactionNotification + clientMsgID - } - - return "" -} -func GetLockMessageTypeKey(clientMsgID string, TypeKey string) string { - return exTypeKeyLocker + clientMsgID + "_" + TypeKey +func GetMessageCacheKeyV2(conversationID string, seq int64) string { + return messageCacheV2 + conversationID + "_" + strconv.Itoa(int(seq)) } func GetSendMsgKey(id string) string { diff --git a/pkg/common/storage/cache/msg.go b/pkg/common/storage/cache/msg.go index 00eb28c02e..084fc63b56 100644 --- a/pkg/common/storage/cache/msg.go +++ b/pkg/common/storage/cache/msg.go @@ -16,8 +16,7 @@ package cache import ( "context" - "time" - + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/protocol/sdkws" ) @@ -27,12 +26,7 @@ type MsgCache interface { DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error SetSendMsgStatus(ctx context.Context, id string, status int32) error GetSendMsgStatus(ctx context.Context, id string) (int32, error) - JudgeMessageReactionExist(ctx context.Context, clientMsgID string, sessionType int32) (bool, error) - GetOneMessageAllReactionList(ctx context.Context, clientMsgID string, sessionType int32) (map[string]string, error) - DeleteOneMessageKey(ctx context.Context, clientMsgID string, sessionType int32, subKey string) error - SetMessageReactionExpire(ctx context.Context, clientMsgID string, sessionType int32, expiration time.Duration) (bool, error) - GetMessageTypeKeyValue(ctx context.Context, clientMsgID string, sessionType int32, typeKey string) (string, error) - SetMessageTypeKeyValue(ctx context.Context, clientMsgID string, sessionType int32, typeKey, value string) error - LockMessageTypeKey(ctx context.Context, clientMsgID string, TypeKey string) error - UnLockMessageTypeKey(ctx context.Context, clientMsgID string, TypeKey string) error + + GetMessageBySeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) + DelMessageBySeqs(ctx context.Context, conversationID string, seqs []int64) error } diff --git a/pkg/common/storage/cache/redis/msg.go b/pkg/common/storage/cache/redis/msg.go index b04bc5c357..5fa43d97a4 100644 --- a/pkg/common/storage/cache/redis/msg.go +++ b/pkg/common/storage/cache/redis/msg.go @@ -2,8 +2,11 @@ package redis import ( "context" + "github.com/dtm-labs/rockscache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/errs" @@ -13,39 +16,27 @@ import ( ) // // msgCacheTimeout is expiration time of message cache, 86400 seconds -const msgCacheTimeout = 86400 +const msgCacheTimeout = time.Hour * 24 func NewMsgCache(client redis.UniversalClient) cache.MsgCache { return &msgCache{rdb: client} } type msgCache struct { - rdb redis.UniversalClient + rdb redis.UniversalClient + rcClient *rockscache.Client + msgDocDatabase database.Msg + msgTable model.MsgDocModel } func (c *msgCache) getMessageCacheKey(conversationID string, seq int64) string { return cachekey.GetMessageCacheKey(conversationID, seq) } -func (c *msgCache) getMessageDelUserListKey(conversationID string, seq int64) string { - return cachekey.GetMessageDelUserListKey(conversationID, seq) -} - -func (c *msgCache) getUserDelList(conversationID, userID string) string { - return cachekey.GetUserDelListKey(conversationID, userID) -} func (c *msgCache) getSendMsgKey(id string) string { return cachekey.GetSendMsgKey(id) } -func (c *msgCache) getLockMessageTypeKey(clientMsgID string, TypeKey string) string { - return cachekey.GetLockMessageTypeKey(clientMsgID, TypeKey) -} - -func (c *msgCache) getMessageReactionExPrefix(clientMsgID string, sessionType int32) string { - return cachekey.GetMessageReactionExKey(clientMsgID, sessionType) -} - func (c *msgCache) SetMessagesToCache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (int, error) { msgMap := datautil.SliceToMap(msgs, func(msg *sdkws.MsgData) string { return c.getMessageCacheKey(conversationID, msg.Seq) @@ -64,7 +55,7 @@ func (c *msgCache) SetMessagesToCache(ctx context.Context, conversationID string values = append(values, s) } } - return LuaSetBatchWithCommonExpire(ctx, c.rdb, keys, values, msgCacheTimeout) + return LuaSetBatchWithCommonExpire(ctx, c.rdb, keys, values, int(msgCacheTimeout/time.Second)) }) if err != nil { return 0, err @@ -77,7 +68,6 @@ func (c *msgCache) DeleteMessagesFromCache(ctx context.Context, conversationID s for _, seq := range seqs { keys = append(keys, c.getMessageCacheKey(conversationID, seq)) } - return ProcessKeysBySlot(ctx, c.rdb, keys, func(ctx context.Context, slot int64, keys []string) error { return LuaDeleteBatch(ctx, c.rdb, keys) }) @@ -92,48 +82,6 @@ func (c *msgCache) GetSendMsgStatus(ctx context.Context, id string) (int32, erro return int32(result), errs.Wrap(err) } -func (c *msgCache) LockMessageTypeKey(ctx context.Context, clientMsgID string, TypeKey string) error { - key := c.getLockMessageTypeKey(clientMsgID, TypeKey) - return errs.Wrap(c.rdb.SetNX(ctx, key, 1, time.Minute).Err()) -} - -func (c *msgCache) UnLockMessageTypeKey(ctx context.Context, clientMsgID string, TypeKey string) error { - key := c.getLockMessageTypeKey(clientMsgID, TypeKey) - return errs.Wrap(c.rdb.Del(ctx, key).Err()) -} - -func (c *msgCache) JudgeMessageReactionExist(ctx context.Context, clientMsgID string, sessionType int32) (bool, error) { - n, err := c.rdb.Exists(ctx, c.getMessageReactionExPrefix(clientMsgID, sessionType)).Result() - if err != nil { - return false, errs.Wrap(err) - } - - return n > 0, nil -} - -func (c *msgCache) SetMessageTypeKeyValue(ctx context.Context, clientMsgID string, sessionType int32, typeKey, value string) error { - return errs.Wrap(c.rdb.HSet(ctx, c.getMessageReactionExPrefix(clientMsgID, sessionType), typeKey, value).Err()) -} - -func (c *msgCache) SetMessageReactionExpire(ctx context.Context, clientMsgID string, sessionType int32, expiration time.Duration) (bool, error) { - val, err := c.rdb.Expire(ctx, c.getMessageReactionExPrefix(clientMsgID, sessionType), expiration).Result() - return val, errs.Wrap(err) -} - -func (c *msgCache) GetMessageTypeKeyValue(ctx context.Context, clientMsgID string, sessionType int32, typeKey string) (string, error) { - val, err := c.rdb.HGet(ctx, c.getMessageReactionExPrefix(clientMsgID, sessionType), typeKey).Result() - return val, errs.Wrap(err) -} - -func (c *msgCache) GetOneMessageAllReactionList(ctx context.Context, clientMsgID string, sessionType int32) (map[string]string, error) { - val, err := c.rdb.HGetAll(ctx, c.getMessageReactionExPrefix(clientMsgID, sessionType)).Result() - return val, errs.Wrap(err) -} - -func (c *msgCache) DeleteOneMessageKey(ctx context.Context, clientMsgID string, sessionType int32, subKey string) error { - return errs.Wrap(c.rdb.HDel(ctx, c.getMessageReactionExPrefix(clientMsgID, sessionType), subKey).Err()) -} - func (c *msgCache) GetMessagesBySeq(ctx context.Context, conversationID string, seqs []int64) (seqMsgs []*sdkws.MsgData, failedSeqs []int64, err error) { var keys []string keySeqMap := make(map[string]int64, 10) @@ -170,3 +118,38 @@ func (c *msgCache) GetMessagesBySeq(ctx context.Context, conversationID string, } return seqMsgs, failedSeqs, nil } + +func (c *msgCache) GetMessageBySeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) { + if len(seqs) == 0 { + return nil, nil + } + getKey := func(seq int64) string { + return cachekey.GetMessageCacheKeyV2(conversationID, seq) + } + getMsgID := func(msg *model.MsgInfoModel) int64 { + return msg.Msg.Seq + } + find := func(ctx context.Context, seqs []int64) ([]*model.MsgInfoModel, error) { + return c.msgDocDatabase.FindSeqs(ctx, conversationID, seqs) + } + return batchGetCache2(ctx, c.rcClient, msgCacheTimeout, seqs, getKey, getMsgID, find) +} + +func (c *msgCache) DelMessageBySeqs(ctx context.Context, conversationID string, seqs []int64) error { + if len(seqs) == 0 { + return nil + } + keys := datautil.Slice(seqs, func(seq int64) string { + return cachekey.GetMessageCacheKeyV2(conversationID, seq) + }) + slotKeys, err := groupKeysBySlot(ctx, getRocksCacheRedisClient(c.rcClient), keys) + if err != nil { + return err + } + for _, keys := range slotKeys { + if err := c.rcClient.TagAsDeletedBatch2(ctx, keys); err != nil { + return err + } + } + return nil +} diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index c29544c333..e09cd14cf7 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -18,6 +18,9 @@ import ( "context" "encoding/json" "errors" + "github.com/openimsdk/tools/utils/jsonutil" + "strconv" + "strings" "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" @@ -36,7 +39,6 @@ import ( "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/utils/datautil" - "github.com/openimsdk/tools/utils/timeutil" ) const ( @@ -54,10 +56,8 @@ type CommonMsgDatabase interface { GetMsgBySeqsRange(ctx context.Context, userID string, conversationID string, begin, end, num, userMaxSeq int64) (minSeq int64, maxSeq int64, seqMsg []*sdkws.MsgData, err error) // GetMsgBySeqs retrieves messages for large groups from MongoDB by sequence numbers. GetMsgBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) (minSeq int64, maxSeq int64, seqMsg []*sdkws.MsgData, err error) - // DeleteConversationMsgsAndSetMinSeq deletes conversation messages and resets the minimum sequence number. If `remainTime` is 0, all messages are deleted (this method does not delete Redis - // cache). + GetMessagesBySeqWithBounds(ctx context.Context, userID string, conversationID string, seqs []int64, pullOrder sdkws.PullOrder) (bool, int64, []*sdkws.MsgData, error) - DeleteConversationMsgsAndSetMinSeq(ctx context.Context, conversationID string, remainTime int64) error // ClearUserMsgs marks messages for deletion based on clear time and returns a list of sequence numbers for marked messages. ClearUserMsgs(ctx context.Context, userID string, conversationID string, clearTime int64, lastMsgClearTime time.Time) (seqs []int64, err error) // DeleteUserMsgsBySeqs allows a user to delete messages based on sequence numbers. @@ -80,8 +80,6 @@ type CommonMsgDatabase interface { GetMaxSeqWithTime(ctx context.Context, conversationID string) (database.SeqTime, error) GetCacheMaxSeqWithTime(ctx context.Context, conversationIDs []string) (map[string]database.SeqTime, error) - //GetMongoMaxAndMinSeq(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo int64, err error) - //GetConversationMinMaxSeqInMongoAndCache(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error) SetSendMsgStatus(ctx context.Context, id string, status int32) error GetSendMsgStatus(ctx context.Context, id string) (int32, error) SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int64, msgData []*pbmsg.SearchedMsgData, err error) @@ -92,10 +90,6 @@ type CommonMsgDatabase interface { RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*model.UserCount, dateCount map[string]int64, err error) RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*model.GroupCount, dateCount map[string]int64, err error) - ConvertMsgsDocLen(ctx context.Context, conversationIDs []string) - - // get Msg when destruct msg before - //DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) @@ -139,7 +133,7 @@ func (db *commonMsgDatabase) MsgToMQ(ctx context.Context, key string, msg2mq *sd return err } -func (db *commonMsgDatabase) BatchInsertBlock(ctx context.Context, conversationID string, fields []any, key int8, firstSeq int64) error { +func (db *commonMsgDatabase) batchInsertBlock(ctx context.Context, conversationID string, fields []any, key int8, firstSeq int64) error { if len(fields) == 0 { return nil } @@ -237,11 +231,15 @@ func (db *commonMsgDatabase) BatchInsertBlock(ctx context.Context, conversationI tryUpdate = false // The current block is inserted successfully, and the next block is inserted preferentially i += insert - 1 // Skip the inserted data } + return nil } func (db *commonMsgDatabase) RevokeMsg(ctx context.Context, conversationID string, seq int64, revoke *model.RevokeModel) error { - return db.BatchInsertBlock(ctx, conversationID, []any{revoke}, updateKeyRevoke, seq) + if err := db.batchInsertBlock(ctx, conversationID, []any{revoke}, updateKeyRevoke, seq); err != nil { + return err + } + return db.msg.DelMessageBySeqs(ctx, conversationID, []int64{seq}) } func (db *commonMsgDatabase) MarkSingleChatMsgsAsRead(ctx context.Context, userID string, conversationID string, totalSeqs []int64) error { @@ -256,22 +254,13 @@ func (db *commonMsgDatabase) MarkSingleChatMsgsAsRead(ctx context.Context, userI return err } } - return nil + return db.msg.DelMessageBySeqs(ctx, conversationID, totalSeqs) } func (db *commonMsgDatabase) getMsgBySeqs(ctx context.Context, userID, conversationID string, seqs []int64) (totalMsgs []*sdkws.MsgData, err error) { - for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, seqs) { - // log.ZDebug(ctx, "getMsgBySeqs", "docID", docID, "seqs", seqs) - msgs, err := db.findMsgInfoBySeq(ctx, userID, docID, conversationID, seqs) - if err != nil { - return nil, err - } - for _, msg := range msgs { - totalMsgs = append(totalMsgs, convert.MsgDB2Pb(msg.Msg)) - } - } - return totalMsgs, nil + return db.GetMessageBySeqs(ctx, conversationID, userID, seqs) } + func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][]*model.MsgInfoModel, userID, conversationID string, msg *model.MsgInfoModel) { if msg.IsRead { msg.Msg.IsRead = true @@ -360,9 +349,6 @@ func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][ return } msg.Msg.Content = string(data) - //if _, err := db.msgDocDatabase.UpdateMsg(ctx, db.msgTable.GetDocID(conversationID, msg.Msg.Seq), db.msgTable.GetMsgIndex(msg.Msg.Seq), "msg", msg.Msg); err != nil { - // log.ZError(ctx, "UpdateMsgContent", err) - //} } func (db *commonMsgDatabase) findMsgInfoBySeq(ctx context.Context, userID, docID string, conversationID string, seqs []int64) (totalMsgs []*model.MsgInfoModel, err error) { @@ -377,24 +363,6 @@ func (db *commonMsgDatabase) findMsgInfoBySeq(ctx context.Context, userID, docID return msgs, err } -func (db *commonMsgDatabase) getMsgBySeqsRange(ctx context.Context, userID string, conversationID string, allSeqs []int64, begin, end int64) (seqMsgs []*sdkws.MsgData, err error) { - log.ZDebug(ctx, "getMsgBySeqsRange", "conversationID", conversationID, "allSeqs", allSeqs, "begin", begin, "end", end) - for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, allSeqs) { - log.ZDebug(ctx, "getMsgBySeqsRange", "docID", docID, "seqs", seqs) - msgs, err := db.findMsgInfoBySeq(ctx, userID, docID, conversationID, seqs) - if err != nil { - return nil, err - } - for _, msg := range msgs { - if msg.IsRead { - msg.Msg.IsRead = true - } - seqMsgs = append(seqMsgs, convert.MsgDB2Pb(msg.Msg)) - } - } - return seqMsgs, nil -} - // GetMsgBySeqsRange In the context of group chat, we have the following parameters: // // "maxSeq" of a conversation: It represents the maximum value of messages in the group conversation. @@ -463,37 +431,10 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin seqs = append(seqs, i) } } - - if len(seqs) == 0 { - return 0, 0, nil, nil - } - newBegin := seqs[0] - newEnd := seqs[len(seqs)-1] - var successMsgs []*sdkws.MsgData - log.ZDebug(ctx, "GetMsgBySeqsRange", "first seqs", seqs, "newBegin", newBegin, "newEnd", newEnd) - cachedMsgs, failedSeqs, err := db.msg.GetMessagesBySeq(ctx, conversationID, seqs) - if err != nil && !errors.Is(err, redis.Nil) { - log.ZError(ctx, "get message from redis exception", err, "conversationID", conversationID, "seqs", seqs) - } - successMsgs = append(successMsgs, cachedMsgs...) - log.ZDebug(ctx, "get msgs from cache", "cachedMsgs", cachedMsgs) - // get from cache or db - - if len(failedSeqs) > 0 { - log.ZDebug(ctx, "msgs not exist in redis", "seqs", failedSeqs) - mongoMsgs, err := db.getMsgBySeqsRange(ctx, userID, conversationID, failedSeqs, begin, end) - if err != nil { - - return 0, 0, nil, err - } - successMsgs = append(mongoMsgs, successMsgs...) - - //_, err = db.msg.SetMessagesToCache(ctx, conversationID, mongoMsgs) - //if err != nil { - // return 0, 0, nil, err - //} + successMsgs, err := db.GetMessageBySeqs(ctx, conversationID, userID, seqs) + if err != nil { + return 0, 0, nil, err } - return minSeq, maxSeq, successMsgs, nil } @@ -529,31 +470,9 @@ func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, co newSeqs = append(newSeqs, seq) } } - if len(newSeqs) == 0 { - return minSeq, maxSeq, nil, nil - } - successMsgs, failedSeqs, err := db.msg.GetMessagesBySeq(ctx, conversationID, newSeqs) + successMsgs, err := db.GetMessageBySeqs(ctx, conversationID, userID, newSeqs) if err != nil { - if !errors.Is(err, redis.Nil) { - log.ZWarn(ctx, "get message from redis exception", err, "failedSeqs", failedSeqs, "conversationID", conversationID) - } - } - log.ZDebug(ctx, "db.seq.GetMessagesBySeq", "userID", userID, "conversationID", conversationID, "seqs", - seqs, "len(successMsgs)", len(successMsgs), "failedSeqs", failedSeqs) - - if len(failedSeqs) > 0 { - mongoMsgs, err := db.getMsgBySeqs(ctx, userID, conversationID, failedSeqs) - if err != nil { - - return 0, 0, nil, err - } - - successMsgs = append(successMsgs, mongoMsgs...) - - //_, err = db.msg.SetMessagesToCache(ctx, conversationID, mongoMsgs) - //if err != nil { - // return 0, 0, nil, err - //} + return 0, 0, nil, err } return minSeq, maxSeq, successMsgs, nil } @@ -607,46 +526,13 @@ func (db *commonMsgDatabase) GetMessagesBySeqWithBounds(ctx context.Context, use if len(newSeqs) == 0 { return isEnd, endSeq, nil, nil } - successMsgs, failedSeqs, err := db.msg.GetMessagesBySeq(ctx, conversationID, newSeqs) + successMsgs, err := db.GetMessageBySeqs(ctx, conversationID, userID, newSeqs) if err != nil { - if !errors.Is(err, redis.Nil) { - log.ZWarn(ctx, "get message from redis exception", err, "failedSeqs", failedSeqs, "conversationID", conversationID) - } - } - log.ZDebug(ctx, "db.seq.GetMessagesBySeq", "userID", userID, "conversationID", conversationID, "seqs", - seqs, "len(successMsgs)", len(successMsgs), "failedSeqs", failedSeqs) - - if len(failedSeqs) > 0 { - mongoMsgs, err := db.getMsgBySeqs(ctx, userID, conversationID, failedSeqs) - if err != nil { - - return false, 0, nil, err - } - - successMsgs = append(successMsgs, mongoMsgs...) - - //_, err = db.msg.SetMessagesToCache(ctx, conversationID, mongoMsgs) - //if err != nil { - // return 0, 0, nil, err - //} + return false, 0, nil, err } return isEnd, endSeq, successMsgs, nil } -func (db *commonMsgDatabase) DeleteConversationMsgsAndSetMinSeq(ctx context.Context, conversationID string, remainTime int64) error { - var delStruct delMsgRecursionStruct - var skip int64 - minSeq, err := db.deleteMsgRecursion(ctx, conversationID, skip, &delStruct, remainTime) - if err != nil { - return err - } - log.ZDebug(ctx, "DeleteConversationMsgsAndSetMinSeq", "conversationID", conversationID, "minSeq", minSeq) - if minSeq == 0 { - return nil - } - return db.seqConversation.SetMinSeq(ctx, conversationID, minSeq) -} - func (db *commonMsgDatabase) ClearUserMsgs(ctx context.Context, userID string, conversationID string, clearTime int64, lastMsgClearTime time.Time) (seqs []int64, err error) { var index int64 for { @@ -721,56 +607,6 @@ func (d *delMsgRecursionStruct) getSetMinSeq() int64 { return d.minSeq } -// index 0....19(del) 20...69 -// seq 70 -// set minSeq 21 -// recursion deletes the list and returns the set minimum seq. -func (db *commonMsgDatabase) deleteMsgRecursion(ctx context.Context, conversationID string, index int64, delStruct *delMsgRecursionStruct, remainTime int64) (int64, error) { - // find from oldest list - msgDocModel, err := db.msgDocDatabase.GetMsgDocModelByIndex(ctx, conversationID, index, 1) - if err != nil || msgDocModel.DocID == "" { - if err != nil { - if err == model.ErrMsgListNotExist { - log.ZDebug(ctx, "deleteMsgRecursion ErrMsgListNotExist", "conversationID", conversationID, "index:", index) - } else { - log.ZError(ctx, "deleteMsgRecursion GetUserMsgListByIndex failed", err, "conversationID", conversationID, "index", index) - } - } - // If an error is reported, or the error cannot be obtained, it is physically deleted and seq delMongoMsgsPhysical(delStruct.delDocIDList) is returned to end the recursion - err = db.msgDocDatabase.DeleteDocs(ctx, delStruct.delDocIDs) - if err != nil { - return 0, err - } - return delStruct.getSetMinSeq() + 1, nil - } - log.ZDebug(ctx, "doc info", "conversationID", conversationID, "index", index, "docID", msgDocModel.DocID, "len", len(msgDocModel.Msg)) - if int64(len(msgDocModel.Msg)) > db.msgTable.GetSingleGocMsgNum() { - log.ZWarn(ctx, "msgs too large", nil, "length", len(msgDocModel.Msg), "docID:", msgDocModel.DocID) - } - if msgDocModel.IsFull() && msgDocModel.Msg[len(msgDocModel.Msg)-1].Msg.SendTime+(remainTime*1000) < timeutil.GetCurrentTimestampByMill() { - log.ZDebug(ctx, "doc is full and all msg is expired", "docID", msgDocModel.DocID) - delStruct.delDocIDs = append(delStruct.delDocIDs, msgDocModel.DocID) - delStruct.minSeq = msgDocModel.Msg[len(msgDocModel.Msg)-1].Msg.Seq - } else { - var delMsgIndexs []int - for i, MsgInfoModel := range msgDocModel.Msg { - if MsgInfoModel != nil && MsgInfoModel.Msg != nil { - if timeutil.GetCurrentTimestampByMill() > MsgInfoModel.Msg.SendTime+(remainTime*1000) { - delMsgIndexs = append(delMsgIndexs, i) - } - } - } - if len(delMsgIndexs) > 0 { - if err = db.msgDocDatabase.DeleteMsgsInOneDocByIndex(ctx, msgDocModel.DocID, delMsgIndexs); err != nil { - log.ZError(ctx, "deleteMsgRecursion DeleteMsgsInOneDocByIndex failed", err, "conversationID", conversationID, "index", index) - } - delStruct.minSeq = int64(msgDocModel.Msg[delMsgIndexs[len(delMsgIndexs)-1]].Msg.Seq) - } - } - seq, err := db.deleteMsgRecursion(ctx, conversationID, index+1, delStruct, remainTime) - return seq, err -} - func (db *commonMsgDatabase) DeleteMsgsPhysicalBySeqs(ctx context.Context, conversationID string, allSeqs []int64) error { if err := db.msg.DeleteMessagesFromCache(ctx, conversationID, allSeqs); err != nil { return err @@ -784,7 +620,7 @@ func (db *commonMsgDatabase) DeleteMsgsPhysicalBySeqs(ctx context.Context, conve return err } } - return nil + return db.msg.DelMessageBySeqs(ctx, conversationID, allSeqs) } func (db *commonMsgDatabase) DeleteUserMsgsBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) error { @@ -798,7 +634,7 @@ func (db *commonMsgDatabase) DeleteUserMsgsBySeqs(ctx context.Context, userID st } } } - return nil + return db.msg.DelMessageBySeqs(ctx, conversationID, seqs) } func (db *commonMsgDatabase) GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { @@ -809,11 +645,6 @@ func (db *commonMsgDatabase) GetMaxSeq(ctx context.Context, conversationID strin return db.seqConversation.GetMaxSeq(ctx, conversationID) } -// -//func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error { -// return db.seqConversation.SetMinSeq(ctx, conversationID, minSeq) -//} - func (db *commonMsgDatabase) SetMinSeqs(ctx context.Context, seqs map[string]int64) error { return db.seqConversation.SetMinSeqs(ctx, seqs) } @@ -888,26 +719,11 @@ func (db *commonMsgDatabase) GetMinMaxSeqMongo(ctx context.Context, conversation return } -func (db *commonMsgDatabase) RangeUserSendCount( - ctx context.Context, - start time.Time, - end time.Time, - group bool, - ase bool, - pageNumber int32, - showNumber int32, -) (msgCount int64, userCount int64, users []*model.UserCount, dateCount map[string]int64, err error) { +func (db *commonMsgDatabase) RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*model.UserCount, dateCount map[string]int64, err error) { return db.msgDocDatabase.RangeUserSendCount(ctx, start, end, group, ase, pageNumber, showNumber) } -func (db *commonMsgDatabase) RangeGroupSendCount( - ctx context.Context, - start time.Time, - end time.Time, - ase bool, - pageNumber int32, - showNumber int32, -) (msgCount int64, userCount int64, groups []*model.GroupCount, dateCount map[string]int64, err error) { +func (db *commonMsgDatabase) RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*model.GroupCount, dateCount map[string]int64, err error) { return db.msgDocDatabase.RangeGroupSendCount(ctx, start, end, ase, pageNumber, showNumber) } @@ -947,43 +763,10 @@ func (db *commonMsgDatabase) FindOneByDocIDs(ctx context.Context, conversationID return totalMsgs, nil } -func (db *commonMsgDatabase) ConvertMsgsDocLen(ctx context.Context, conversationIDs []string) { - db.msgDocDatabase.ConvertMsgsDocLen(ctx, conversationIDs) -} - func (db *commonMsgDatabase) GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) { return db.msgDocDatabase.GetRandBeforeMsg(ctx, ts, limit) } -// -//func (db *commonMsgDatabase) DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) { -// var notNull int -// index := make([]int, 0, len(doc.Msg)) -// for i, message := range doc.Msg { -// if message.Msg != nil { -// notNull++ -// if message.Msg.SendTime < ts { -// index = append(index, i) -// } -// } -// } -// if len(index) == 0 { -// return index, nil -// } -// maxSeq := doc.Msg[index[len(index)-1]].Msg.Seq -// conversationID := doc.DocID[:strings.LastIndex(doc.DocID, ":")] -// if err := db.SetMinSeq(ctx, conversationID, maxSeq+1); err != nil { -// return index, err -// } -// if len(index) == notNull { -// log.ZDebug(ctx, "Delete db in Doc", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq) -// return index, db.msgDocDatabase.DeleteDoc(ctx, doc.DocID) -// } else { -// log.ZDebug(ctx, "delete db in index", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq) -// return index, db.msgDocDatabase.DeleteMsgByIndex(ctx, doc.DocID, index) -// } -//} - func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { dbSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) if err != nil { @@ -998,10 +781,6 @@ func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID strin return db.seqConversation.SetMinSeq(ctx, conversationID, seq) } -func (db *commonMsgDatabase) GetRandDocIDs(ctx context.Context, limit int) ([]string, error) { - return db.msgDocDatabase.GetRandDocIDs(ctx, limit) -} - func (db *commonMsgDatabase) GetCacheMaxSeqWithTime(ctx context.Context, conversationIDs []string) (map[string]database.SeqTime, error) { return db.seqConversation.GetCacheMaxSeqWithTime(ctx, conversationIDs) } @@ -1016,9 +795,103 @@ func (db *commonMsgDatabase) GetMaxSeqsWithTime(ctx context.Context, conversatio } func (db *commonMsgDatabase) DeleteDoc(ctx context.Context, docID string) error { - return db.msgDocDatabase.DeleteDoc(ctx, docID) + index := strings.LastIndex(docID, ":") + if index <= 0 { + return errs.ErrInternalServer.WrapMsg("docID is invalid", "docID", docID) + } + index, err := strconv.Atoi(docID[index+1:]) + if err != nil { + return errs.WrapMsg(err, "strconv.Atoi", "docID", docID) + } + conversationID := docID[:index] + seqs := make([]int64, db.msgTable.GetSingleGocMsgNum()) + minSeq := db.msgTable.GetMinSeq(index) + for i := range seqs { + seqs[i] = minSeq + int64(i) + } + if err := db.msgDocDatabase.DeleteDoc(ctx, docID); err != nil { + return err + } + return db.msg.DelMessageBySeqs(ctx, conversationID, seqs) } func (db *commonMsgDatabase) GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) { return db.msgDocDatabase.GetLastMessageSeqByTime(ctx, conversationID, time) } + +func (db *commonMsgDatabase) handlerDeleteAndRevoked(ctx context.Context, userID string, msgs []*model.MsgInfoModel) { + for i := range msgs { + msg := msgs[i] + if msg == nil || msg.Msg == nil { + continue + } + msg.Msg.IsRead = msg.IsRead + if datautil.Contain(userID, msg.DelList...) { + msg.Msg.Content = "" + msg.Msg.Status = constant.MsgDeleted + } + if msg.Revoke == nil { + continue + } + msg.Msg.ContentType = constant.MsgRevokeNotification + revokeContent := sdkws.MessageRevokedContent{ + RevokerID: msg.Revoke.UserID, + RevokerRole: msg.Revoke.Role, + ClientMsgID: msg.Msg.ClientMsgID, + RevokerNickname: msg.Revoke.Nickname, + RevokeTime: msg.Revoke.Time, + SourceMessageSendTime: msg.Msg.SendTime, + SourceMessageSendID: msg.Msg.SendID, + SourceMessageSenderNickname: msg.Msg.SenderNickname, + SessionType: msg.Msg.SessionType, + Seq: msg.Msg.Seq, + Ex: msg.Msg.Ex, + } + data, err := jsonutil.JsonMarshal(&revokeContent) + if err != nil { + log.ZWarn(ctx, "handlerDeleteAndRevoked JsonMarshal MessageRevokedContent", err, "msg", msg) + continue + } + elem := sdkws.NotificationElem{ + Detail: string(data), + } + content, err := jsonutil.JsonMarshal(&elem) + if err != nil { + log.ZWarn(ctx, "handlerDeleteAndRevoked JsonMarshal NotificationElem", err, "msg", msg) + continue + } + msg.Msg.Content = string(content) + } +} + +func (db *commonMsgDatabase) handlerQuote(ctx context.Context, userID, conversationID string, msgs []*model.MsgInfoModel) { + temp := make(map[int64][]*model.MsgInfoModel) + for i := range msgs { + db.handlerDBMsg(ctx, temp, userID, conversationID, msgs[i]) + } +} + +func (db *commonMsgDatabase) GetMessageBySeqs(ctx context.Context, conversationID string, userID string, seqs []int64) ([]*sdkws.MsgData, error) { + msgs, err := db.msg.GetMessageBySeqs(ctx, conversationID, seqs) + if err != nil { + return nil, err + } + db.handlerDeleteAndRevoked(ctx, userID, msgs) + db.handlerQuote(ctx, userID, conversationID, msgs) + seqMsgs := make(map[int64]*model.MsgInfoModel) + for i, msg := range msgs { + if msg.Msg == nil { + continue + } + seqMsgs[msg.Msg.Seq] = msgs[i] + } + res := make([]*sdkws.MsgData, 0, len(seqs)) + for _, seq := range seqs { + if v, ok := seqMsgs[seq]; ok { + res = append(res, convert.MsgDB2Pb(v.Msg)) + } else { + res = append(res, &sdkws.MsgData{Seq: seq}) + } + } + return res, nil +} diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index f371766958..13c58a31de 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -7,15 +7,12 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/tools/utils/datautil" - "golang.org/x/exp/rand" - "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/jsonutil" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" @@ -279,95 +276,6 @@ func (m *MsgMgo) MarkSingleChatMsgsAsRead(ctx context.Context, userID string, do return nil } -//func (m *MsgMgo) searchCount(ctx context.Context, filter any) (int64, error) { -// -// return nil, nil -//} - -//func (m *MsgMgo) searchMessage(ctx context.Context, filter any, nextID primitive.ObjectID, content bool, limit int) (int64, []*model.MsgInfoModel, primitive.ObjectID, error) { -// var pipeline bson.A -// if !nextID.IsZero() { -// pipeline = append(pipeline, bson.M{"$match": bson.M{"_id": bson.M{"$gt": nextID}}}) -// } -// pipeline = append(pipeline, -// bson.M{"$match": filter}, -// bson.M{"$limit": limit}, -// bson.M{"$unwind": "$msgs"}, -// bson.M{"$match": filter}, -// bson.M{ -// "$group": bson.M{ -// "_id": "$_id", -// "doc_id": bson.M{ -// "$first": "$doc_id", -// }, -// "msgs": bson.M{"$push": "$msgs"}, -// }, -// }, -// ) -// if !content { -// pipeline = append(pipeline, -// bson.M{ -// "$project": bson.M{ -// "_id": 1, -// "count": bson.M{"$size": "$msgs"}, -// }, -// }, -// ) -// type result struct { -// ID primitive.ObjectID `bson:"_id"` -// Count int64 `bson:"count"` -// } -// res, err := mongoutil.Aggregate[result](ctx, m.coll, pipeline) -// if err != nil { -// return 0, nil, primitive.ObjectID{}, err -// } -// if len(res) == 0 { -// return 0, nil, primitive.ObjectID{}, nil -// } -// var count int64 -// for _, r := range res { -// count += r.Count -// } -// return count, nil, res[len(res)-1].ID, nil -// } -// type result struct { -// ID primitive.ObjectID `bson:"_id"` -// Msg []*model.MsgInfoModel `bson:"msgs"` -// } -// res, err := mongoutil.Aggregate[result](ctx, m.coll, pipeline) -// if err != nil { -// return 0, nil, primitive.ObjectID{}, err -// } -// if len(res) == 0 { -// return 0, nil, primitive.ObjectID{}, err -// } -// var count int -// for _, r := range res { -// count += len(r.Msg) -// } -// msgs := make([]*model.MsgInfoModel, 0, count) -// for _, r := range res { -// msgs = append(msgs, r.Msg...) -// } -// return int64(count), msgs, res[len(res)-1].ID, nil -//} - -/* - -db.msg3.aggregate( - [ - { - "$match": { - "doc_id": "si_7009965934_8710838466:0" - }, - - } - ] -) - - -*/ - type searchMessageIndex struct { ID primitive.ObjectID `bson:"_id"` Index []int64 `bson:"index"` @@ -556,143 +464,6 @@ func (m *MsgMgo) SearchMessage(ctx context.Context, req *msg.SearchMessageReq) ( return count, msgs, nil } -//func (m *MsgMgo) SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (int32, []*model.MsgInfoModel, error) { -// where := make(bson.A, 0, 6) -// if req.RecvID != "" { -// if req.SessionType == constant.ReadGroupChatType { -// where = append(where, bson.M{ -// "$or": bson.A{ -// bson.M{"doc_id": "^n_" + req.RecvID + ":"}, -// bson.M{"doc_id": "^sg_" + req.RecvID + ":"}, -// }, -// }) -// } else { -// where = append(where, bson.M{"msgs.msg.recv_id": req.RecvID}) -// } -// } -// if req.SendID != "" { -// where = append(where, bson.M{"msgs.msg.send_id": req.SendID}) -// } -// if req.ContentType != 0 { -// where = append(where, bson.M{"msgs.msg.content_type": req.ContentType}) -// } -// if req.SessionType != 0 { -// where = append(where, bson.M{"msgs.msg.session_type": req.SessionType}) -// } -// if req.SendTime != "" { -// sendTime, err := time.Parse(time.DateOnly, req.SendTime) -// if err != nil { -// return 0, nil, errs.ErrArgs.WrapMsg("invalid sendTime", "req", req.SendTime, "format", time.DateOnly, "cause", err.Error()) -// } -// where = append(where, -// bson.M{ -// "msgs.msg.send_time": bson.M{ -// "$gte": sendTime.UnixMilli(), -// }, -// }, -// bson.M{ -// "msgs.msg.send_time": bson.M{ -// "$lt": sendTime.Add(time.Hour * 24).UnixMilli(), -// }, -// }, -// ) -// } -// opt := options.Find().SetLimit(100) -// res, err := mongoutil.Find[model.MsgDocModel](ctx, m.coll, bson.M{"$and": where}, opt) -// if err != nil { -// return 0, nil, err -// } -// _ = res -// fmt.Println() -// -// return 0, nil, nil -// pipeline := bson.A{ -// bson.M{ -// "$unwind": "$msgs", -// }, -// } -// if len(where) > 0 { -// pipeline = append(pipeline, bson.M{ -// "$match": bson.M{"$and": where}, -// }) -// } -// pipeline = append(pipeline, -// bson.M{ -// "$project": bson.M{ -// "_id": 0, -// "msg": "$msgs.msg", -// }, -// }, -// bson.M{ -// "$count": "count", -// }, -// ) -// //count, err := mongoutil.Aggregate[int32](ctx, m.coll, pipeline) -// //if err != nil { -// // return 0, nil, err -// //} -// //if len(count) == 0 || count[0] == 0 { -// // return 0, nil, nil -// //} -// count := []int32{0} -// pipeline = pipeline[:len(pipeline)-1] -// pipeline = append(pipeline, -// bson.M{ -// "$skip": (req.Pagination.GetPageNumber() - 1) * req.Pagination.GetShowNumber(), -// }, -// bson.M{ -// "$limit": req.Pagination.GetShowNumber(), -// }, -// ) -// msgs, err := mongoutil.Aggregate[*model.MsgInfoModel](ctx, m.coll, pipeline) -// if err != nil { -// return 0, nil, err -// } -// for i := range msgs { -// msgInfo := msgs[i] -// if msgInfo == nil || msgInfo.Msg == nil { -// continue -// } -// if msgInfo.Revoke != nil { -// revokeContent := sdkws.MessageRevokedContent{ -// RevokerID: msgInfo.Revoke.UserID, -// RevokerRole: msgInfo.Revoke.Role, -// ClientMsgID: msgInfo.Msg.ClientMsgID, -// RevokerNickname: msgInfo.Revoke.Nickname, -// RevokeTime: msgInfo.Revoke.Time, -// SourceMessageSendTime: msgInfo.Msg.SendTime, -// SourceMessageSendID: msgInfo.Msg.SendID, -// SourceMessageSenderNickname: msgInfo.Msg.SenderNickname, -// SessionType: msgInfo.Msg.SessionType, -// Seq: msgInfo.Msg.Seq, -// Ex: msgInfo.Msg.Ex, -// } -// data, err := jsonutil.JsonMarshal(&revokeContent) -// if err != nil { -// return 0, nil, errs.WrapMsg(err, "json.Marshal revokeContent") -// } -// elem := sdkws.NotificationElem{Detail: string(data)} -// content, err := jsonutil.JsonMarshal(&elem) -// if err != nil { -// return 0, nil, errs.WrapMsg(err, "json.Marshal elem") -// } -// msgInfo.Msg.ContentType = constant.MsgRevokeNotification -// msgInfo.Msg.Content = string(content) -// } -// } -// //start := (req.Pagination.PageNumber - 1) * req.Pagination.ShowNumber -// //n := int32(len(msgs)) -// //if start >= n { -// // return n, []*relation.MsgInfoModel{}, nil -// //} -// //if start+req.Pagination.ShowNumber < n { -// // msgs = msgs[start : start+req.Pagination.ShowNumber] -// //} else { -// // msgs = msgs[start:] -// //} -// return count[0], msgs, nil -//} - func (m *MsgMgo) RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*model.UserCount, dateCount map[string]int64, err error) { var sort int if ase { @@ -1178,94 +949,6 @@ func (m *MsgMgo) RangeGroupSendCount(ctx context.Context, start time.Time, end t return result[0].MsgCount, result[0].UserCount, groups, dateCount, nil } -func (m *MsgMgo) ConvertMsgsDocLen(ctx context.Context, conversationIDs []string) { - for _, conversationID := range conversationIDs { - regex := primitive.Regex{Pattern: fmt.Sprintf("^%s:", conversationID)} - msgDocs, err := mongoutil.Find[*model.MsgDocModel](ctx, m.coll, bson.M{"doc_id": regex}) - if err != nil { - log.ZError(ctx, "convertAll find msg doc failed", err, "conversationID", conversationID) - continue - } - if len(msgDocs) < 1 { - continue - } - log.ZDebug(ctx, "msg doc convert", "conversationID", conversationID, "len(msgDocs)", len(msgDocs)) - if len(msgDocs[0].Msg) == int(m.model.GetSingleGocMsgNum5000()) { - if err := mongoutil.DeleteMany(ctx, m.coll, bson.M{"doc_id": regex}); err != nil { - log.ZError(ctx, "convertAll delete many failed", err, "conversationID", conversationID) - continue - } - var newMsgDocs []any - for _, msgDoc := range msgDocs { - if int64(len(msgDoc.Msg)) == m.model.GetSingleGocMsgNum() { - continue - } - var index int64 - for index < int64(len(msgDoc.Msg)) { - msg := msgDoc.Msg[index] - if msg != nil && msg.Msg != nil { - msgDocModel := model.MsgDocModel{DocID: m.model.GetDocID(conversationID, msg.Msg.Seq)} - end := index + m.model.GetSingleGocMsgNum() - if int(end) >= len(msgDoc.Msg) { - msgDocModel.Msg = msgDoc.Msg[index:] - } else { - msgDocModel.Msg = msgDoc.Msg[index:end] - } - newMsgDocs = append(newMsgDocs, msgDocModel) - index = end - } else { - break - } - } - } - if err = mongoutil.InsertMany(ctx, m.coll, newMsgDocs); err != nil { - log.ZError(ctx, "convertAll insert many failed", err, "conversationID", conversationID, "len(newMsgDocs)", len(newMsgDocs)) - } else { - log.ZDebug(ctx, "msg doc convert", "conversationID", conversationID, "len(newMsgDocs)", len(newMsgDocs)) - } - } - } -} - -func (m *MsgMgo) GetRandDocIDs(ctx context.Context, limit int) ([]string, error) { - var skip int - var docIDs []string - var offset int - - count, err := m.coll.CountDocuments(ctx, bson.M{}) - if err != nil { - return nil, err - } - - if count < int64(limit) { - skip = 0 - } else { - rand.Seed(uint64(time.Now().UnixMilli())) - skip = rand.Intn(int(count / int64(limit))) - offset = skip * limit - } - log.ZDebug(ctx, "offset", "skip", skip, "offset", offset) - res, err := mongoutil.Aggregate[*model.MsgDocModel](ctx, m.coll, []bson.M{ - { - "$project": bson.M{ - "doc_id": 1, - }, - }, - { - "$skip": offset, - }, - { - "$limit": limit, - }, - }) - - for _, doc := range res { - docIDs = append(docIDs, doc.DocID) - } - - return docIDs, errs.Wrap(err) -} - func (m *MsgMgo) GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) { return mongoutil.Aggregate[*model.MsgDocModel](ctx, m.coll, []bson.M{ { @@ -1364,3 +1047,55 @@ func (m *MsgMgo) GetLastMessageSeqByTime(ctx context.Context, conversationID str } return seq, nil } + +func (m *MsgMgo) onlyFindDocIndex(ctx context.Context, docID string, indexes []int64) ([]*model.MsgInfoModel, error) { + if len(indexes) == 0 { + return nil, nil + } + pipeline := mongo.Pipeline{ + bson.D{{Key: "$match", Value: bson.D{ + {Key: "doc_id", Value: docID}, + }}}, + bson.D{{Key: "$project", Value: bson.D{ + {Key: "_id", Value: 0}, + {Key: "doc_id", Value: 1}, + {Key: "msgs", Value: bson.D{ + {Key: "$map", Value: bson.D{ + {Key: "input", Value: indexes}, + {Key: "as", Value: "index"}, + {Key: "in", Value: bson.D{ + {Key: "$arrayElemAt", Value: bson.A{"$msgs", "$$index"}}, + }}, + }}, + }}, + }}}, + } + msgDocModel, err := mongoutil.Aggregate[*model.MsgDocModel](ctx, m.coll, pipeline) + if err != nil { + return nil, err + } + if len(msgDocModel) == 0 { + return nil, nil + } + return msgDocModel[0].Msg, nil +} + +func (m *MsgMgo) FindSeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) { + if len(seqs) == 0 { + return nil, nil + } + result := make([]*model.MsgInfoModel, 0, len(seqs)) + for docID, seqs := range m.model.GetDocIDSeqsMap(conversationID, seqs) { + res, err := m.onlyFindDocIndex(ctx, docID, datautil.Slice(seqs, m.model.GetMsgIndex)) + if err != nil { + return nil, err + } + for i, re := range res { + if re == nil || re.Msg == nil { + continue + } + result = append(result, res[i]) + } + } + return result, nil +} diff --git a/pkg/common/storage/database/msg.go b/pkg/common/storage/database/msg.go index abb2a44c2f..d964b18269 100644 --- a/pkg/common/storage/database/msg.go +++ b/pkg/common/storage/database/msg.go @@ -41,13 +41,12 @@ type Msg interface { SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (int64, []*model.MsgInfoModel, error) RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*model.UserCount, dateCount map[string]int64, err error) RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*model.GroupCount, dateCount map[string]int64, err error) - ConvertMsgsDocLen(ctx context.Context, conversationIDs []string) DeleteDoc(ctx context.Context, docID string) error DeleteMsgByIndex(ctx context.Context, docID string, index []int) error GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) - GetRandDocIDs(ctx context.Context, limit int) ([]string, error) - GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) + + FindSeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) } diff --git a/pkg/common/storage/model/msg.go b/pkg/common/storage/model/msg.go index e16233973b..69113032da 100644 --- a/pkg/common/storage/model/msg.go +++ b/pkg/common/storage/model/msg.go @@ -143,3 +143,7 @@ func (*MsgDocModel) GenExceptionMessageBySeqs(seqs []int64) (exceptionMsg []*sdk } return exceptionMsg } + +func (*MsgDocModel) GetMinSeq(index int) int64 { + return int64(index*singleGocMsgNum) + 1 +} From 9e64d8a2ca1d78a80220b99cc3edc70b7c564f4c Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 26 Dec 2024 15:52:55 +0800 Subject: [PATCH 12/17] redis msg cache --- internal/msgtransfer/init.go | 2 +- internal/rpc/msg/server.go | 2 +- pkg/common/storage/cache/cachekey/msg.go | 11 +- pkg/common/storage/cache/msg.go | 4 - pkg/common/storage/cache/redis/msg.go | 92 ++---------- pkg/common/storage/cache/redis/msg_test.go | 133 ------------------ pkg/common/storage/controller/msg.go | 100 ++----------- pkg/common/storage/controller/msg_transfer.go | 27 ++-- 8 files changed, 39 insertions(+), 332 deletions(-) delete mode 100644 pkg/common/storage/cache/redis/msg_test.go diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index ee9f066448..13721d61b0 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -92,11 +92,11 @@ func Start(ctx context.Context, index int, config *Config) error { } client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) - msgModel := redis.NewMsgCache(rdb) msgDocModel, err := mgo.NewMsgMongo(mgocli.GetDB()) if err != nil { return err } + msgModel := redis.NewMsgCache(rdb, msgDocModel) seqConversation, err := mgo.NewSeqConversationMongo(mgocli.GetDB()) if err != nil { return err diff --git a/internal/rpc/msg/server.go b/internal/rpc/msg/server.go index 991fb87044..7ccda6bd57 100644 --- a/internal/rpc/msg/server.go +++ b/internal/rpc/msg/server.go @@ -89,7 +89,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } - msgModel := redis.NewMsgCache(rdb) + msgModel := redis.NewMsgCache(rdb, msgDocModel) seqConversation, err := mgo.NewSeqConversationMongo(mgocli.GetDB()) if err != nil { return err diff --git a/pkg/common/storage/cache/cachekey/msg.go b/pkg/common/storage/cache/cachekey/msg.go index 4c8b32cd05..ac449df38e 100644 --- a/pkg/common/storage/cache/cachekey/msg.go +++ b/pkg/common/storage/cache/cachekey/msg.go @@ -19,17 +19,12 @@ import ( ) const ( - messageCache = "MESSAGE_CACHE:" sendMsgFailedFlag = "SEND_MSG_FAILED_FLAG:" - messageCacheV2 = "MESSAGE_CACHE_V2:" + messageCache = "MSG_CACHE:" ) -func GetMessageCacheKey(conversationID string, seq int64) string { - return messageCache + conversationID + "_" + strconv.Itoa(int(seq)) -} - -func GetMessageCacheKeyV2(conversationID string, seq int64) string { - return messageCacheV2 + conversationID + "_" + strconv.Itoa(int(seq)) +func GetMsgCacheKey(conversationID string, seq int64) string { + return messageCache + conversationID + ":" + strconv.Itoa(int(seq)) } func GetSendMsgKey(id string) string { diff --git a/pkg/common/storage/cache/msg.go b/pkg/common/storage/cache/msg.go index 084fc63b56..9a04f3cf82 100644 --- a/pkg/common/storage/cache/msg.go +++ b/pkg/common/storage/cache/msg.go @@ -17,13 +17,9 @@ package cache import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/protocol/sdkws" ) type MsgCache interface { - GetMessagesBySeq(ctx context.Context, conversationID string, seqs []int64) (seqMsg []*sdkws.MsgData, failedSeqList []int64, err error) - SetMessagesToCache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (int, error) - DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error SetSendMsgStatus(ctx context.Context, id string, status int32) error GetSendMsgStatus(ctx context.Context, id string) (int32, error) diff --git a/pkg/common/storage/cache/redis/msg.go b/pkg/common/storage/cache/redis/msg.go index 5fa43d97a4..16eacc126b 100644 --- a/pkg/common/storage/cache/redis/msg.go +++ b/pkg/common/storage/cache/redis/msg.go @@ -7,8 +7,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" - "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" @@ -18,61 +16,24 @@ import ( // msgCacheTimeout is expiration time of message cache, 86400 seconds const msgCacheTimeout = time.Hour * 24 -func NewMsgCache(client redis.UniversalClient) cache.MsgCache { - return &msgCache{rdb: client} +func NewMsgCache(client redis.UniversalClient, db database.Msg) cache.MsgCache { + return &msgCache{ + rdb: client, + rcClient: rockscache.NewClient(client, *GetRocksCacheOptions()), + msgDocDatabase: db, + } } type msgCache struct { rdb redis.UniversalClient rcClient *rockscache.Client msgDocDatabase database.Msg - msgTable model.MsgDocModel -} - -func (c *msgCache) getMessageCacheKey(conversationID string, seq int64) string { - return cachekey.GetMessageCacheKey(conversationID, seq) } func (c *msgCache) getSendMsgKey(id string) string { return cachekey.GetSendMsgKey(id) } -func (c *msgCache) SetMessagesToCache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (int, error) { - msgMap := datautil.SliceToMap(msgs, func(msg *sdkws.MsgData) string { - return c.getMessageCacheKey(conversationID, msg.Seq) - }) - keys := datautil.Slice(msgs, func(msg *sdkws.MsgData) string { - return c.getMessageCacheKey(conversationID, msg.Seq) - }) - err := ProcessKeysBySlot(ctx, c.rdb, keys, func(ctx context.Context, slot int64, keys []string) error { - var values []string - for _, key := range keys { - if msg, ok := msgMap[key]; ok { - s, err := msgprocessor.Pb2String(msg) - if err != nil { - return err - } - values = append(values, s) - } - } - return LuaSetBatchWithCommonExpire(ctx, c.rdb, keys, values, int(msgCacheTimeout/time.Second)) - }) - if err != nil { - return 0, err - } - return len(msgs), nil -} - -func (c *msgCache) DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error { - var keys []string - for _, seq := range seqs { - keys = append(keys, c.getMessageCacheKey(conversationID, seq)) - } - return ProcessKeysBySlot(ctx, c.rdb, keys, func(ctx context.Context, slot int64, keys []string) error { - return LuaDeleteBatch(ctx, c.rdb, keys) - }) -} - func (c *msgCache) SetSendMsgStatus(ctx context.Context, id string, status int32) error { return errs.Wrap(c.rdb.Set(ctx, c.getSendMsgKey(id), status, time.Hour*24).Err()) } @@ -82,49 +43,12 @@ func (c *msgCache) GetSendMsgStatus(ctx context.Context, id string) (int32, erro return int32(result), errs.Wrap(err) } -func (c *msgCache) GetMessagesBySeq(ctx context.Context, conversationID string, seqs []int64) (seqMsgs []*sdkws.MsgData, failedSeqs []int64, err error) { - var keys []string - keySeqMap := make(map[string]int64, 10) - for _, seq := range seqs { - key := c.getMessageCacheKey(conversationID, seq) - keys = append(keys, key) - keySeqMap[key] = seq - } - err = ProcessKeysBySlot(ctx, c.rdb, keys, func(ctx context.Context, slot int64, keys []string) error { - result, err := LuaGetBatch(ctx, c.rdb, keys) - if err != nil { - return err - } - for i, value := range result { - seq := keySeqMap[keys[i]] - if value == nil { - failedSeqs = append(failedSeqs, seq) - continue - } - - msg := &sdkws.MsgData{} - msgString, ok := value.(string) - if !ok || msgprocessor.String2Pb(msgString, msg) != nil { - failedSeqs = append(failedSeqs, seq) - continue - } - seqMsgs = append(seqMsgs, msg) - - } - return nil - }) - if err != nil { - return nil, nil, err - } - return seqMsgs, failedSeqs, nil -} - func (c *msgCache) GetMessageBySeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) { if len(seqs) == 0 { return nil, nil } getKey := func(seq int64) string { - return cachekey.GetMessageCacheKeyV2(conversationID, seq) + return cachekey.GetMsgCacheKey(conversationID, seq) } getMsgID := func(msg *model.MsgInfoModel) int64 { return msg.Msg.Seq @@ -140,7 +64,7 @@ func (c *msgCache) DelMessageBySeqs(ctx context.Context, conversationID string, return nil } keys := datautil.Slice(seqs, func(seq int64) string { - return cachekey.GetMessageCacheKeyV2(conversationID, seq) + return cachekey.GetMsgCacheKey(conversationID, seq) }) slotKeys, err := groupKeysBySlot(ctx, getRocksCacheRedisClient(c.rcClient), keys) if err != nil { diff --git a/pkg/common/storage/cache/redis/msg_test.go b/pkg/common/storage/cache/redis/msg_test.go deleted file mode 100644 index 10b9ce18b0..0000000000 --- a/pkg/common/storage/cache/redis/msg_test.go +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package redis - -import ( - "context" - "fmt" - "github.com/openimsdk/protocol/sdkws" - "github.com/redis/go-redis/v9" - "github.com/stretchr/testify/assert" - "google.golang.org/protobuf/proto" - "testing" -) - -func Test_msgCache_SetMessagesToCache(t *testing.T) { - type fields struct { - rdb redis.UniversalClient - } - type args struct { - ctx context.Context - conversationID string - msgs []*sdkws.MsgData - } - tests := []struct { - name string - fields fields - args args - want int - wantErr assert.ErrorAssertionFunc - }{ - {"test1", fields{rdb: redis.NewClient(&redis.Options{Addr: "localhost:16379", Username: "", Password: "openIM123", DB: 0})}, args{context.Background(), - "cid", []*sdkws.MsgData{{Seq: 1}, {Seq: 2}, {Seq: 3}}}, 3, assert.NoError}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := &msgCache{ - rdb: tt.fields.rdb, - } - got, err := c.SetMessagesToCache(tt.args.ctx, tt.args.conversationID, tt.args.msgs) - if !tt.wantErr(t, err, fmt.Sprintf("SetMessagesToCache(%v, %v, %v)", tt.args.ctx, tt.args.conversationID, tt.args.msgs)) { - return - } - assert.Equalf(t, tt.want, got, "SetMessagesToCache(%v, %v, %v)", tt.args.ctx, tt.args.conversationID, tt.args.msgs) - }) - } -} - -func Test_msgCache_GetMessagesBySeq(t *testing.T) { - type fields struct { - rdb redis.UniversalClient - } - type args struct { - ctx context.Context - conversationID string - seqs []int64 - } - var failedSeq []int64 - tests := []struct { - name string - fields fields - args args - wantSeqMsgs []*sdkws.MsgData - wantFailedSeqs []int64 - wantErr assert.ErrorAssertionFunc - }{ - {"test1", fields{rdb: redis.NewClient(&redis.Options{Addr: "localhost:16379", Password: "openIM123", DB: 0})}, - args{context.Background(), "cid", []int64{1, 2, 3}}, - []*sdkws.MsgData{{Seq: 1}, {Seq: 2}, {Seq: 3}}, failedSeq, assert.NoError}, - {"test2", fields{rdb: redis.NewClient(&redis.Options{Addr: "localhost:16379", Password: "openIM123", DB: 0})}, - args{context.Background(), "cid", []int64{4, 5, 6}}, - nil, []int64{4, 5, 6}, assert.NoError}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := &msgCache{ - rdb: tt.fields.rdb, - } - gotSeqMsgs, gotFailedSeqs, err := c.GetMessagesBySeq(tt.args.ctx, tt.args.conversationID, tt.args.seqs) - if !tt.wantErr(t, err, fmt.Sprintf("GetMessagesBySeq(%v, %v, %v)", tt.args.ctx, tt.args.conversationID, tt.args.seqs)) { - return - } - equalMsgDataSlices(t, tt.wantSeqMsgs, gotSeqMsgs) - assert.Equalf(t, tt.wantFailedSeqs, gotFailedSeqs, "GetMessagesBySeq(%v, %v, %v)", tt.args.ctx, tt.args.conversationID, tt.args.seqs) - }) - } -} - -func equalMsgDataSlices(t *testing.T, expected, actual []*sdkws.MsgData) { - assert.Equal(t, len(expected), len(actual), "Slices have different lengths") - for i := range expected { - assert.True(t, proto.Equal(expected[i], actual[i]), "Element %d not equal: expected %v, got %v", i, expected[i], actual[i]) - } -} - -func Test_msgCache_DeleteMessagesFromCache(t *testing.T) { - type fields struct { - rdb redis.UniversalClient - } - type args struct { - ctx context.Context - conversationID string - seqs []int64 - } - tests := []struct { - name string - fields fields - args args - wantErr assert.ErrorAssertionFunc - }{ - {"test1", fields{rdb: redis.NewClient(&redis.Options{Addr: "localhost:16379", Password: "openIM123"})}, - args{context.Background(), "cid", []int64{1, 2, 3}}, assert.NoError}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := &msgCache{ - rdb: tt.fields.rdb, - } - tt.wantErr(t, c.DeleteMessagesFromCache(tt.args.ctx, tt.args.conversationID, tt.args.seqs), - fmt.Sprintf("DeleteMessagesFromCache(%v, %v, %v)", tt.args.ctx, tt.args.conversationID, tt.args.seqs)) - }) - } -} diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index e09cd14cf7..62587c8731 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -58,8 +58,6 @@ type CommonMsgDatabase interface { GetMsgBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) (minSeq int64, maxSeq int64, seqMsg []*sdkws.MsgData, err error) GetMessagesBySeqWithBounds(ctx context.Context, userID string, conversationID string, seqs []int64, pullOrder sdkws.PullOrder) (bool, int64, []*sdkws.MsgData, error) - // ClearUserMsgs marks messages for deletion based on clear time and returns a list of sequence numbers for marked messages. - ClearUserMsgs(ctx context.Context, userID string, conversationID string, clearTime int64, lastMsgClearTime time.Time) (seqs []int64, err error) // DeleteUserMsgsBySeqs allows a user to delete messages based on sequence numbers. DeleteUserMsgsBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) error // DeleteMsgsPhysicalBySeqs physically deletes messages by emptying them based on sequence numbers. @@ -112,7 +110,7 @@ func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser } return &commonMsgDatabase{ msgDocDatabase: msgDocModel, - msg: msg, + msgCache: msg, seqUser: seqUser, seqConversation: seqConversation, producer: producerToRedis, @@ -122,7 +120,7 @@ func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser type commonMsgDatabase struct { msgDocDatabase database.Msg msgTable model.MsgDocModel - msg cache.MsgCache + msgCache cache.MsgCache seqConversation cache.SeqConversationCache seqUser cache.SeqUser producer *kafka.Producer @@ -239,7 +237,7 @@ func (db *commonMsgDatabase) RevokeMsg(ctx context.Context, conversationID strin if err := db.batchInsertBlock(ctx, conversationID, []any{revoke}, updateKeyRevoke, seq); err != nil { return err } - return db.msg.DelMessageBySeqs(ctx, conversationID, []int64{seq}) + return db.msgCache.DelMessageBySeqs(ctx, conversationID, []int64{seq}) } func (db *commonMsgDatabase) MarkSingleChatMsgsAsRead(ctx context.Context, userID string, conversationID string, totalSeqs []int64) error { @@ -254,7 +252,7 @@ func (db *commonMsgDatabase) MarkSingleChatMsgsAsRead(ctx context.Context, userI return err } } - return db.msg.DelMessageBySeqs(ctx, conversationID, totalSeqs) + return db.msgCache.DelMessageBySeqs(ctx, conversationID, totalSeqs) } func (db *commonMsgDatabase) getMsgBySeqs(ctx context.Context, userID, conversationID string, seqs []int64) (totalMsgs []*sdkws.MsgData, err error) { @@ -533,82 +531,8 @@ func (db *commonMsgDatabase) GetMessagesBySeqWithBounds(ctx context.Context, use return isEnd, endSeq, successMsgs, nil } -func (db *commonMsgDatabase) ClearUserMsgs(ctx context.Context, userID string, conversationID string, clearTime int64, lastMsgClearTime time.Time) (seqs []int64, err error) { - var index int64 - for { - // from oldest 2 newest, ASC - msgDocModel, err := db.msgDocDatabase.GetMsgDocModelByIndex(ctx, conversationID, index, 1) - if err != nil || msgDocModel.DocID == "" { - if err != nil { - if err == model.ErrMsgListNotExist { - log.ZDebug(ctx, "not doc find", "conversationID", conversationID, "userID", userID, "index", index) - } else { - log.ZError(ctx, "deleteMsgRecursion GetUserMsgListByIndex failed", err, "conversationID", conversationID, "index", index) - } - } - // If an error is reported, or the error cannot be obtained, it is physically deleted and seq delMongoMsgsPhysical(delStruct.delDocIDList) is returned to end the recursion - break - } - - index++ - - // && msgDocModel.Msg[0].Msg.SendTime > lastMsgClearTime.UnixMilli() - if len(msgDocModel.Msg) > 0 { - i := 0 - var over bool - for _, msg := range msgDocModel.Msg { - i++ - // over clear time, need to clear - if msg != nil && msg.Msg != nil && msg.Msg.SendTime+clearTime*1000 <= time.Now().UnixMilli() { - // if msg is not in del list, add to del list - if msg.Msg.SendTime+clearTime*1000 > lastMsgClearTime.UnixMilli() && !datautil.Contain(userID, msg.DelList...) { - seqs = append(seqs, msg.Msg.Seq) - } - } else { - log.ZDebug(ctx, "all msg need destruct is found", "conversationID", conversationID, "userID", userID, "index", index, "stop index", i) - over = true - break - } - } - if over { - break - } - } - } - - log.ZDebug(ctx, "ClearUserMsgs", "conversationID", conversationID, "userID", userID, "seqs", seqs) - - // have msg need to destruct - if len(seqs) > 0 { - // update min seq to clear after - userMinSeq := seqs[len(seqs)-1] + 1 // user min seq when clear after - currentUserMinSeq, err := db.seqUser.GetUserMinSeq(ctx, conversationID, userID) // user min seq when clear before - if err != nil { - return nil, err - } - - // if before < after, update min seq - if currentUserMinSeq < userMinSeq { - if err := db.seqUser.SetUserMinSeq(ctx, conversationID, userID, userMinSeq); err != nil { - return nil, err - } - } - } - return seqs, nil -} - -// this is struct for recursion. -type delMsgRecursionStruct struct { - minSeq int64 - delDocIDs []string -} - -func (d *delMsgRecursionStruct) getSetMinSeq() int64 { - return d.minSeq -} - func (db *commonMsgDatabase) DeleteMsgsPhysicalBySeqs(ctx context.Context, conversationID string, allSeqs []int64) error { - if err := db.msg.DeleteMessagesFromCache(ctx, conversationID, allSeqs); err != nil { + if err := db.msgCache.DelMessageBySeqs(ctx, conversationID, allSeqs); err != nil { return err } for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, allSeqs) { @@ -620,11 +544,11 @@ func (db *commonMsgDatabase) DeleteMsgsPhysicalBySeqs(ctx context.Context, conve return err } } - return db.msg.DelMessageBySeqs(ctx, conversationID, allSeqs) + return db.msgCache.DelMessageBySeqs(ctx, conversationID, allSeqs) } func (db *commonMsgDatabase) DeleteUserMsgsBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) error { - if err := db.msg.DeleteMessagesFromCache(ctx, conversationID, seqs); err != nil { + if err := db.msgCache.DelMessageBySeqs(ctx, conversationID, seqs); err != nil { return err } for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, seqs) { @@ -634,7 +558,7 @@ func (db *commonMsgDatabase) DeleteUserMsgsBySeqs(ctx context.Context, userID st } } } - return db.msg.DelMessageBySeqs(ctx, conversationID, seqs) + return db.msgCache.DelMessageBySeqs(ctx, conversationID, seqs) } func (db *commonMsgDatabase) GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { @@ -678,11 +602,11 @@ func (db *commonMsgDatabase) GetHasReadSeq(ctx context.Context, userID string, c } func (db *commonMsgDatabase) SetSendMsgStatus(ctx context.Context, id string, status int32) error { - return db.msg.SetSendMsgStatus(ctx, id, status) + return db.msgCache.SetSendMsgStatus(ctx, id, status) } func (db *commonMsgDatabase) GetSendMsgStatus(ctx context.Context, id string) (int32, error) { - return db.msg.GetSendMsgStatus(ctx, id) + return db.msgCache.GetSendMsgStatus(ctx, id) } func (db *commonMsgDatabase) GetConversationMinMaxSeqInMongoAndCache(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error) { @@ -812,7 +736,7 @@ func (db *commonMsgDatabase) DeleteDoc(ctx context.Context, docID string) error if err := db.msgDocDatabase.DeleteDoc(ctx, docID); err != nil { return err } - return db.msg.DelMessageBySeqs(ctx, conversationID, seqs) + return db.msgCache.DelMessageBySeqs(ctx, conversationID, seqs) } func (db *commonMsgDatabase) GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) { @@ -872,7 +796,7 @@ func (db *commonMsgDatabase) handlerQuote(ctx context.Context, userID, conversat } func (db *commonMsgDatabase) GetMessageBySeqs(ctx context.Context, conversationID string, userID string, seqs []int64) ([]*sdkws.MsgData, error) { - msgs, err := db.msg.GetMessageBySeqs(ctx, conversationID, seqs) + msgs, err := db.msgCache.GetMessageBySeqs(ctx, conversationID, seqs) if err != nil { return nil, err } diff --git a/pkg/common/storage/controller/msg_transfer.go b/pkg/common/storage/controller/msg_transfer.go index 1ecd786aa3..58b3a24374 100644 --- a/pkg/common/storage/controller/msg_transfer.go +++ b/pkg/common/storage/controller/msg_transfer.go @@ -5,7 +5,6 @@ import ( "github.com/openimsdk/protocol/constant" "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" @@ -50,7 +49,7 @@ func NewMsgTransferDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUse } return &msgTransferDatabase{ msgDocDatabase: msgDocModel, - msg: msg, + msgCache: msg, seqUser: seqUser, seqConversation: seqConversation, producerToMongo: producerToMongo, @@ -61,7 +60,7 @@ func NewMsgTransferDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUse type msgTransferDatabase struct { msgDocDatabase database.Msg msgTable model.MsgDocModel - msg cache.MsgCache + msgCache cache.MsgCache seqConversation cache.SeqConversationCache seqUser cache.SeqUser producerToMongo *kafka.Producer @@ -73,10 +72,12 @@ func (db *msgTransferDatabase) BatchInsertChat2DB(ctx context.Context, conversat return errs.ErrArgs.WrapMsg("msgList is empty") } msgs := make([]any, len(msgList)) + seqs := make([]int64, len(msgList)) for i, msg := range msgList { if msg == nil { continue } + seqs[i] = msg.Seq var offlinePushModel *model.OfflinePushModel if msg.OfflinePushInfo != nil { offlinePushModel = &model.OfflinePushModel{ @@ -114,7 +115,10 @@ func (db *msgTransferDatabase) BatchInsertChat2DB(ctx context.Context, conversat Ex: msg.Ex, } } - return db.BatchInsertBlock(ctx, conversationID, msgs, updateKeyMsg, msgList[0].Seq) + if err := db.BatchInsertBlock(ctx, conversationID, msgs, updateKeyMsg, msgList[0].Seq); err != nil { + return err + } + return db.msgCache.DelMessageBySeqs(ctx, conversationID, seqs) } func (db *msgTransferDatabase) BatchInsertBlock(ctx context.Context, conversationID string, fields []any, key int8, firstSeq int64) error { @@ -219,7 +223,7 @@ func (db *msgTransferDatabase) BatchInsertBlock(ctx context.Context, conversatio } func (db *msgTransferDatabase) DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error { - return db.msg.DeleteMessagesFromCache(ctx, conversationID, seqs) + return db.msgCache.DelMessageBySeqs(ctx, conversationID, seqs) } func (db *msgTransferDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, userHasReadMap map[string]int64, err error) { @@ -238,20 +242,17 @@ func (db *msgTransferDatabase) BatchInsertChat2Cache(ctx context.Context, conver isNew = currentMaxSeq == 0 lastMaxSeq := currentMaxSeq userSeqMap := make(map[string]int64) + seqs := make([]int64, 0, lenList) for _, m := range msgs { currentMaxSeq++ m.Seq = currentMaxSeq userSeqMap[m.SendID] = m.Seq + seqs = append(seqs, m.Seq) } - - failedNum, err := db.msg.SetMessagesToCache(ctx, conversationID, msgs) - if err != nil { - prommetrics.MsgInsertRedisFailedCounter.Add(float64(failedNum)) - log.ZError(ctx, "setMessageToCache error", err, "len", len(msgs), "conversationID", conversationID) - } else { - prommetrics.MsgInsertRedisSuccessCounter.Inc() + if err := db.msgCache.DelMessageBySeqs(ctx, conversationID, seqs); err != nil { + return 0, false, nil, err } - return lastMaxSeq, isNew, userSeqMap, errs.Wrap(err) + return lastMaxSeq, isNew, userSeqMap, nil } func (db *msgTransferDatabase) SetHasReadSeqs(ctx context.Context, conversationID string, userSeqMap map[string]int64) error { From ca8d55138bde8aacf8f475f658f4578533bc875e Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 26 Dec 2024 16:48:19 +0800 Subject: [PATCH 13/17] redis msg cache --- .../online_msg_to_mongo_handler.go | 20 +++---------------- pkg/common/storage/cache/msg.go | 1 + pkg/common/storage/cache/redis/msg.go | 17 ++++++++++++++++ pkg/common/storage/controller/msg_transfer.go | 5 +++++ 4 files changed, 26 insertions(+), 17 deletions(-) diff --git a/internal/msgtransfer/online_msg_to_mongo_handler.go b/internal/msgtransfer/online_msg_to_mongo_handler.go index 82002c26b9..d8836d54eb 100644 --- a/internal/msgtransfer/online_msg_to_mongo_handler.go +++ b/internal/msgtransfer/online_msg_to_mongo_handler.go @@ -77,27 +77,13 @@ func (mc *OnlineHistoryMongoConsumerHandler) handleChatWs2Mongo(ctx context.Cont for _, msg := range msgFromMQ.MsgData { seqs = append(seqs, msg.Seq) } - err = mc.msgTransferDatabase.DeleteMessagesFromCache(ctx, msgFromMQ.ConversationID, seqs) - if err != nil { - log.ZError( - ctx, - "remove cache msg from redis err", - err, - "msg", - msgFromMQ.MsgData, - "conversationID", - msgFromMQ.ConversationID, - ) - } } -func (*OnlineHistoryMongoConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil } +func (*OnlineHistoryMongoConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil } + func (*OnlineHistoryMongoConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error { return nil } -func (mc *OnlineHistoryMongoConsumerHandler) ConsumeClaim( - sess sarama.ConsumerGroupSession, - claim sarama.ConsumerGroupClaim, -) error { // an instance in the consumer group +func (mc *OnlineHistoryMongoConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { // an instance in the consumer group log.ZDebug(context.Background(), "online new session msg come", "highWaterMarkOffset", claim.HighWaterMarkOffset(), "topic", claim.Topic(), "partition", claim.Partition()) for msg := range claim.Messages() { diff --git a/pkg/common/storage/cache/msg.go b/pkg/common/storage/cache/msg.go index 9a04f3cf82..f780cd5377 100644 --- a/pkg/common/storage/cache/msg.go +++ b/pkg/common/storage/cache/msg.go @@ -25,4 +25,5 @@ type MsgCache interface { GetMessageBySeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) DelMessageBySeqs(ctx context.Context, conversationID string, seqs []int64) error + SetMessageBySeqs(ctx context.Context, conversationID string, msgs []*model.MsgDataModel) error } diff --git a/pkg/common/storage/cache/redis/msg.go b/pkg/common/storage/cache/redis/msg.go index 16eacc126b..b6799d5dcc 100644 --- a/pkg/common/storage/cache/redis/msg.go +++ b/pkg/common/storage/cache/redis/msg.go @@ -2,6 +2,7 @@ package redis import ( "context" + "encoding/json" "github.com/dtm-labs/rockscache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" @@ -77,3 +78,19 @@ func (c *msgCache) DelMessageBySeqs(ctx context.Context, conversationID string, } return nil } + +func (c *msgCache) SetMessageBySeqs(ctx context.Context, conversationID string, msgs []*model.MsgDataModel) error { + for _, msg := range msgs { + if msg == nil || msg.Seq <= 0 { + continue + } + data, err := json.Marshal(msg) + if err != nil { + return err + } + if err := c.rcClient.RawSet(ctx, cachekey.GetMsgCacheKey(conversationID, msg.Seq), string(data), msgCacheTimeout); err != nil { + return err + } + } + return nil +} diff --git a/pkg/common/storage/controller/msg_transfer.go b/pkg/common/storage/controller/msg_transfer.go index 58b3a24374..edb98c7f98 100644 --- a/pkg/common/storage/controller/msg_transfer.go +++ b/pkg/common/storage/controller/msg_transfer.go @@ -2,7 +2,9 @@ package controller import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/convert" "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" @@ -252,6 +254,9 @@ func (db *msgTransferDatabase) BatchInsertChat2Cache(ctx context.Context, conver if err := db.msgCache.DelMessageBySeqs(ctx, conversationID, seqs); err != nil { return 0, false, nil, err } + if err := db.msgCache.SetMessageBySeqs(ctx, conversationID, datautil.Slice(msgs, convert.MsgPb2DB)); err != nil { + return 0, false, nil, err + } return lastMaxSeq, isNew, userSeqMap, nil } From 9a009a69c92fb07b3785c300600bb922b840e1fd Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 26 Dec 2024 16:50:14 +0800 Subject: [PATCH 14/17] redis msg cache --- pkg/common/storage/controller/msg_transfer.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/common/storage/controller/msg_transfer.go b/pkg/common/storage/controller/msg_transfer.go index edb98c7f98..f5d6be2b71 100644 --- a/pkg/common/storage/controller/msg_transfer.go +++ b/pkg/common/storage/controller/msg_transfer.go @@ -251,9 +251,6 @@ func (db *msgTransferDatabase) BatchInsertChat2Cache(ctx context.Context, conver userSeqMap[m.SendID] = m.Seq seqs = append(seqs, m.Seq) } - if err := db.msgCache.DelMessageBySeqs(ctx, conversationID, seqs); err != nil { - return 0, false, nil, err - } if err := db.msgCache.SetMessageBySeqs(ctx, conversationID, datautil.Slice(msgs, convert.MsgPb2DB)); err != nil { return 0, false, nil, err } From 9d64781fc74b369dd1fc0838f418371240fd6309 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 26 Dec 2024 16:59:20 +0800 Subject: [PATCH 15/17] redis msg cache --- pkg/common/storage/controller/msg.go | 6 ------ pkg/common/storage/controller/msg_transfer.go | 3 ++- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 62587c8731..8c97aa6cee 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -532,9 +532,6 @@ func (db *commonMsgDatabase) GetMessagesBySeqWithBounds(ctx context.Context, use } func (db *commonMsgDatabase) DeleteMsgsPhysicalBySeqs(ctx context.Context, conversationID string, allSeqs []int64) error { - if err := db.msgCache.DelMessageBySeqs(ctx, conversationID, allSeqs); err != nil { - return err - } for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, allSeqs) { var indexes []int for _, seq := range seqs { @@ -548,9 +545,6 @@ func (db *commonMsgDatabase) DeleteMsgsPhysicalBySeqs(ctx context.Context, conve } func (db *commonMsgDatabase) DeleteUserMsgsBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) error { - if err := db.msgCache.DelMessageBySeqs(ctx, conversationID, seqs); err != nil { - return err - } for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, seqs) { for _, seq := range seqs { if _, err := db.msgDocDatabase.PushUnique(ctx, docID, db.msgTable.GetMsgIndex(seq), "del_list", []string{userID}); err != nil { diff --git a/pkg/common/storage/controller/msg_transfer.go b/pkg/common/storage/controller/msg_transfer.go index f5d6be2b71..f0e4f43328 100644 --- a/pkg/common/storage/controller/msg_transfer.go +++ b/pkg/common/storage/controller/msg_transfer.go @@ -120,7 +120,8 @@ func (db *msgTransferDatabase) BatchInsertChat2DB(ctx context.Context, conversat if err := db.BatchInsertBlock(ctx, conversationID, msgs, updateKeyMsg, msgList[0].Seq); err != nil { return err } - return db.msgCache.DelMessageBySeqs(ctx, conversationID, seqs) + //return db.msgCache.DelMessageBySeqs(ctx, conversationID, seqs) + return nil } func (db *msgTransferDatabase) BatchInsertBlock(ctx context.Context, conversationID string, fields []any, key int8, firstSeq int64) error { From 1219b47c79246b79553580c243f758d7b5db7cb6 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 26 Dec 2024 17:24:36 +0800 Subject: [PATCH 16/17] redis msg cache --- internal/rpc/msg/revoke.go | 5 +++-- pkg/common/storage/cache/msg.go | 2 +- pkg/common/storage/cache/redis/msg.go | 6 +++--- pkg/common/storage/controller/msg.go | 3 +++ pkg/common/storage/controller/msg_transfer.go | 7 ++++++- pkg/common/storage/database/msg.go | 2 +- 6 files changed, 17 insertions(+), 8 deletions(-) diff --git a/internal/rpc/msg/revoke.go b/internal/rpc/msg/revoke.go index b7cc7df62e..97de0f48a7 100644 --- a/internal/rpc/msg/revoke.go +++ b/internal/rpc/msg/revoke.go @@ -63,7 +63,8 @@ func (m *msgServer) RevokeMsg(ctx context.Context, req *msg.RevokeMsgReq) (*msg. log.ZDebug(ctx, "GetMsgBySeqs", "conversationID", req.ConversationID, "seq", req.Seq, "msg", string(data)) var role int32 if !authverify.IsAppManagerUid(ctx, m.config.Share.IMAdminUserID) { - switch msgs[0].SessionType { + sessionType := msgs[0].SessionType + switch sessionType { case constant.SingleChatType: if err := authverify.CheckAccessV3(ctx, msgs[0].SendID, m.config.Share.IMAdminUserID); err != nil { return nil, err @@ -89,7 +90,7 @@ func (m *msgServer) RevokeMsg(ctx context.Context, req *msg.RevokeMsgReq) (*msg. role = member.RoleLevel } default: - return nil, errs.ErrInternalServer.WrapMsg("msg sessionType not supported") + return nil, errs.ErrInternalServer.WrapMsg("msg sessionType not supported", "sessionType", sessionType) } } now := time.Now().UnixMilli() diff --git a/pkg/common/storage/cache/msg.go b/pkg/common/storage/cache/msg.go index f780cd5377..271ed19fe3 100644 --- a/pkg/common/storage/cache/msg.go +++ b/pkg/common/storage/cache/msg.go @@ -25,5 +25,5 @@ type MsgCache interface { GetMessageBySeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) DelMessageBySeqs(ctx context.Context, conversationID string, seqs []int64) error - SetMessageBySeqs(ctx context.Context, conversationID string, msgs []*model.MsgDataModel) error + SetMessageBySeqs(ctx context.Context, conversationID string, msgs []*model.MsgInfoModel) error } diff --git a/pkg/common/storage/cache/redis/msg.go b/pkg/common/storage/cache/redis/msg.go index b6799d5dcc..0651f02830 100644 --- a/pkg/common/storage/cache/redis/msg.go +++ b/pkg/common/storage/cache/redis/msg.go @@ -79,16 +79,16 @@ func (c *msgCache) DelMessageBySeqs(ctx context.Context, conversationID string, return nil } -func (c *msgCache) SetMessageBySeqs(ctx context.Context, conversationID string, msgs []*model.MsgDataModel) error { +func (c *msgCache) SetMessageBySeqs(ctx context.Context, conversationID string, msgs []*model.MsgInfoModel) error { for _, msg := range msgs { - if msg == nil || msg.Seq <= 0 { + if msg == nil || msg.Msg == nil || msg.Msg.Seq <= 0 { continue } data, err := json.Marshal(msg) if err != nil { return err } - if err := c.rcClient.RawSet(ctx, cachekey.GetMsgCacheKey(conversationID, msg.Seq), string(data), msgCacheTimeout); err != nil { + if err := c.rcClient.RawSet(ctx, cachekey.GetMsgCacheKey(conversationID, msg.Msg.Seq), string(data), msgCacheTimeout); err != nil { return err } } diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 8c97aa6cee..d5ad12584c 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -260,6 +260,9 @@ func (db *commonMsgDatabase) getMsgBySeqs(ctx context.Context, userID, conversat } func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][]*model.MsgInfoModel, userID, conversationID string, msg *model.MsgInfoModel) { + if msg == nil || msg.Msg == nil { + return + } if msg.IsRead { msg.Msg.IsRead = true } diff --git a/pkg/common/storage/controller/msg_transfer.go b/pkg/common/storage/controller/msg_transfer.go index f0e4f43328..f4c0c6270f 100644 --- a/pkg/common/storage/controller/msg_transfer.go +++ b/pkg/common/storage/controller/msg_transfer.go @@ -252,7 +252,12 @@ func (db *msgTransferDatabase) BatchInsertChat2Cache(ctx context.Context, conver userSeqMap[m.SendID] = m.Seq seqs = append(seqs, m.Seq) } - if err := db.msgCache.SetMessageBySeqs(ctx, conversationID, datautil.Slice(msgs, convert.MsgPb2DB)); err != nil { + msgToDB := func(msg *sdkws.MsgData) *model.MsgInfoModel { + return &model.MsgInfoModel{ + Msg: convert.MsgPb2DB(msg), + } + } + if err := db.msgCache.SetMessageBySeqs(ctx, conversationID, datautil.Slice(msgs, msgToDB)); err != nil { return 0, false, nil, err } return lastMaxSeq, isNew, userSeqMap, nil diff --git a/pkg/common/storage/database/msg.go b/pkg/common/storage/database/msg.go index d964b18269..c92f182a37 100644 --- a/pkg/common/storage/database/msg.go +++ b/pkg/common/storage/database/msg.go @@ -24,7 +24,7 @@ import ( ) type Msg interface { - PushMsgsToDoc(ctx context.Context, docID string, msgsToMongo []model.MsgInfoModel) error + //PushMsgsToDoc(ctx context.Context, docID string, msgsToMongo []model.MsgInfoModel) error Create(ctx context.Context, model *model.MsgDocModel) error UpdateMsg(ctx context.Context, docID string, index int64, key string, value any) (*mongo.UpdateResult, error) PushUnique(ctx context.Context, docID string, index int64, key string, value any) (*mongo.UpdateResult, error) From d44fab851805e23b2b388f14a32c152b9a2180b7 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 26 Dec 2024 17:28:10 +0800 Subject: [PATCH 17/17] redis msg cache --- pkg/common/storage/database/mgo/msg.go | 51 -------------------------- pkg/common/storage/database/msg.go | 9 ----- 2 files changed, 60 deletions(-) diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index 13c58a31de..03ebff6110 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -39,12 +39,6 @@ type MsgMgo struct { model model.MsgDocModel } -func (m *MsgMgo) PushMsgsToDoc(ctx context.Context, docID string, msgsToMongo []model.MsgInfoModel) error { - filter := bson.M{"doc_id": docID} - update := bson.M{"$push": bson.M{"msgs": bson.M{"$each": msgsToMongo}}} - return mongoutil.UpdateOne(ctx, m.coll, filter, update, false) -} - func (m *MsgMgo) Create(ctx context.Context, msg *model.MsgDocModel) error { return mongoutil.InsertMany(ctx, m.coll, []*model.MsgDocModel{msg}) } @@ -77,16 +71,6 @@ func (m *MsgMgo) PushUnique(ctx context.Context, docID string, index int64, key return mongoutil.UpdateOneResult(ctx, m.coll, filter, update) } -func (m *MsgMgo) UpdateMsgContent(ctx context.Context, docID string, index int64, msg []byte) error { - filter := bson.M{"doc_id": docID} - update := bson.M{"$set": bson.M{fmt.Sprintf("msgs.%d.msg", index): msg}} - return mongoutil.UpdateOne(ctx, m.coll, filter, update, false) -} - -func (m *MsgMgo) IsExistDocID(ctx context.Context, docID string) (bool, error) { - return mongoutil.Exist(ctx, m.coll, bson.M{"doc_id": docID}) -} - func (m *MsgMgo) FindOneByDocID(ctx context.Context, docID string) (*model.MsgDocModel, error) { return mongoutil.FindOne[*model.MsgDocModel](ctx, m.coll, bson.M{"doc_id": docID}) } @@ -215,13 +199,6 @@ func (m *MsgMgo) GetOldestMsg(ctx context.Context, conversationID string) (*mode } } -func (m *MsgMgo) DeleteDocs(ctx context.Context, docIDs []string) error { - if len(docIDs) == 0 { - return nil - } - return mongoutil.DeleteMany(ctx, m.coll, bson.M{"doc_id": bson.M{"$in": docIDs}}) -} - func (m *MsgMgo) GetMsgDocModelByIndex(ctx context.Context, conversationID string, index, sort int64) (*model.MsgDocModel, error) { if sort != 1 && sort != -1 { return nil, errs.ErrArgs.WrapMsg("mongo sort must be 1 or -1") @@ -420,22 +397,6 @@ func (m *MsgMgo) searchMessage(ctx context.Context, req *msg.SearchMessageReq) ( } } -func (m *MsgMgo) getDocRange(ctx context.Context, id primitive.ObjectID, index []int64) ([]*model.MsgInfoModel, error) { - if len(index) == 0 { - return nil, nil - } - - pipeline := bson.A{ - bson.M{"$match": bson.M{"_id": id}}, - bson.M{"$project": "$msgs"}, - } - msgs, err := mongoutil.Aggregate[*model.MsgInfoModel](ctx, m.coll, pipeline) - if err != nil { - return nil, err - } - return msgs, nil -} - func (m *MsgMgo) SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (int64, []*model.MsgInfoModel, error) { count, data, err := m.searchMessage(ctx, req) if err != nil { @@ -980,18 +941,6 @@ func (m *MsgMgo) GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]* }) } -func (m *MsgMgo) DeleteMsgByIndex(ctx context.Context, docID string, index []int) error { - if len(index) == 0 { - return nil - } - model := &model.MsgInfoModel{DelList: []string{}} - set := make(map[string]any) - for i := range index { - set[fmt.Sprintf("msgs.%d", i)] = model - } - return mongoutil.UpdateOne(ctx, m.coll, bson.M{"doc_id": docID}, bson.M{"$set": set}, true) -} - func (m *MsgMgo) DeleteDoc(ctx context.Context, docID string) error { return mongoutil.DeleteOne(ctx, m.coll, bson.M{"doc_id": docID}) } diff --git a/pkg/common/storage/database/msg.go b/pkg/common/storage/database/msg.go index c92f182a37..b44e702964 100644 --- a/pkg/common/storage/database/msg.go +++ b/pkg/common/storage/database/msg.go @@ -24,29 +24,20 @@ import ( ) type Msg interface { - //PushMsgsToDoc(ctx context.Context, docID string, msgsToMongo []model.MsgInfoModel) error Create(ctx context.Context, model *model.MsgDocModel) error UpdateMsg(ctx context.Context, docID string, index int64, key string, value any) (*mongo.UpdateResult, error) PushUnique(ctx context.Context, docID string, index int64, key string, value any) (*mongo.UpdateResult, error) - UpdateMsgContent(ctx context.Context, docID string, index int64, msg []byte) error - IsExistDocID(ctx context.Context, docID string) (bool, error) FindOneByDocID(ctx context.Context, docID string) (*model.MsgDocModel, error) GetMsgBySeqIndexIn1Doc(ctx context.Context, userID, docID string, seqs []int64) ([]*model.MsgInfoModel, error) GetNewestMsg(ctx context.Context, conversationID string) (*model.MsgInfoModel, error) GetOldestMsg(ctx context.Context, conversationID string) (*model.MsgInfoModel, error) - DeleteDocs(ctx context.Context, docIDs []string) error - GetMsgDocModelByIndex(ctx context.Context, conversationID string, index, sort int64) (*model.MsgDocModel, error) DeleteMsgsInOneDocByIndex(ctx context.Context, docID string, indexes []int) error MarkSingleChatMsgsAsRead(ctx context.Context, userID string, docID string, indexes []int64) error SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (int64, []*model.MsgInfoModel, error) RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*model.UserCount, dateCount map[string]int64, err error) RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*model.GroupCount, dateCount map[string]int64, err error) - DeleteDoc(ctx context.Context, docID string) error - DeleteMsgByIndex(ctx context.Context, docID string, index []int) error GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) - GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) - FindSeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) }