Skip to content

Commit 8f2ad17

Browse files
fanyufanyu
committed
Filter messages with rowid
1 parent d4228d9 commit 8f2ad17

File tree

7 files changed

+106
-80
lines changed

7 files changed

+106
-80
lines changed

‎Mixin/Service/DeviceTransfer/DeviceTransferServerDataSource.swift‎

Lines changed: 48 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -290,14 +290,32 @@ extension DeviceTransferServerDataSource {
290290
}
291291
}
292292
case .pinMessage:
293-
let pinMessages = PinMessageDAO.shared.pinMessages(limit: limit,
294-
after: location.primaryID,
295-
matching: conversationIDs,
296-
sinceDate: fromDate)
293+
let rowID: Int
294+
if let primaryID = location.primaryID?.intValue {
295+
rowID = primaryID
296+
} else {
297+
if let fromDate {
298+
if let startRowID = PinMessageDAO.shared.messageRowID(createdAt: fromDate) {
299+
rowID = startRowID
300+
} else {
301+
return (0, [], nil, nil)
302+
}
303+
} else {
304+
rowID = -1
305+
}
306+
}
307+
let pinMessages = PinMessageDAO.shared.pinMessages(limit: limit, after: rowID, matching: conversationIDs)
297308
databaseItemCount = pinMessages.count
298-
nextPrimaryID = pinMessages.last?.messageId
309+
if let messageID = pinMessages.last?.messageId, let rowID = PinMessageDAO.shared.messageRowID(messageID: messageID) {
310+
nextPrimaryID = "\(rowID)"
311+
} else {
312+
nextPrimaryID = nil
313+
}
299314
nextSecondaryID = nil
300315
transferItems = pinMessages.compactMap { pinMessage in
316+
if let fromDate, pinMessage.createdAt < fromDate {
317+
return nil
318+
}
301319
let deviceTransferPinMessage = DeviceTransferPinMessage(pinMessage: pinMessage)
302320
do {
303321
let outputData = try DeviceTransferProtocol.output(type: location.type, data: deviceTransferPinMessage, key: key)
@@ -309,10 +327,7 @@ extension DeviceTransferServerDataSource {
309327
}
310328
case .transcriptMessage:
311329
if needsFilterData {
312-
databaseItemCount = 0
313-
nextPrimaryID = nil
314-
nextSecondaryID = nil
315-
transferItems = []
330+
return (0, [], nil, nil)
316331
} else {
317332
let transcriptMessages = TranscriptMessageDAO.shared.transcriptMessages(limit: limit, after: location.primaryID, with: location.secondaryID)
318333
databaseItemCount = transcriptMessages.count
@@ -321,16 +336,34 @@ extension DeviceTransferServerDataSource {
321336
transferItems = transcriptTransferItems(for: transcriptMessages)
322337
}
323338
case .message:
324-
let messages = MessageDAO.shared.messages(limit: limit,
325-
after: location.primaryID,
326-
matching: conversationIDs,
327-
sinceDate: fromDate)
339+
let rowID: Int
340+
if let primaryID = location.primaryID?.intValue {
341+
rowID = primaryID
342+
} else {
343+
if let fromDate {
344+
if let startRowID = MessageDAO.shared.messageRowID(createdAt: fromDate) {
345+
rowID = startRowID
346+
} else {
347+
return (0, [], nil, nil)
348+
}
349+
} else {
350+
rowID = -1
351+
}
352+
}
353+
let messages = MessageDAO.shared.messages(limit: limit, after: rowID, matching: conversationIDs)
328354
databaseItemCount = messages.count
329-
nextPrimaryID = messages.last?.messageId
355+
if let messageID = messages.last?.messageId, let rowID = MessageDAO.shared.messageRowID(messageID: messageID) {
356+
nextPrimaryID = "\(rowID)"
357+
} else {
358+
nextPrimaryID = nil
359+
}
330360
nextSecondaryID = nil
331361
var messageItems = [TransferItem]()
332362
var transcriptMessageItems = [TransferItem]()
333363
for message in messages {
364+
if let fromDate, message.createdAt < fromDate {
365+
continue
366+
}
334367
let deviceTransferMessage = DeviceTransferMessage(message: message, to: remotePlatform)
335368
do {
336369
let outputData = try DeviceTransferProtocol.output(type: location.type, data: deviceTransferMessage, key: key)
@@ -412,7 +445,7 @@ extension DeviceTransferServerDataSource {
412445
return false
413446
}
414447
#if DEBUG
415-
Logger.general.debug(category: "DeviceTransferServerDataSource", message: "Send File: \(url)")
448+
//Logger.general.debug(category: "DeviceTransferServerDataSource", message: "Send File: \(url)")
416449
#endif
417450

418451
let encryptor = try AESCryptor(operation: .encrypt, iv: iv, key: key.aes)

‎MixinServices/MixinServices/Database/User/DAO/ConversationDAO.swift‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -662,8 +662,8 @@ public final class ConversationDAO: UserDatabaseDAO {
662662
public func conversations(limit: Int, after conversationId: String?, matching conversationIDs: [String]?) -> [Conversation] {
663663
if let conversationIDs {
664664
var totalConversations = [Conversation]()
665-
for i in stride(from: 0, to: conversationIDs.count, by: Self.maxCountOfHostParameter) {
666-
let endIndex = min(i + Self.maxCountOfHostParameter, conversationIDs.count)
665+
for i in stride(from: 0, to: conversationIDs.count, by: Self.strideForDeviceTransfer) {
666+
let endIndex = min(i + Self.strideForDeviceTransfer, conversationIDs.count)
667667
let ids = Array(conversationIDs[i..<endIndex]).joined(separator: "', '")
668668
var sql = "SELECT * FROM conversations WHERE conversation_id in ('\(ids)')"
669669
if let conversationId {
@@ -687,8 +687,8 @@ public final class ConversationDAO: UserDatabaseDAO {
687687
public func conversationsCount(matching conversationIDs: [String]?) -> Int {
688688
if let conversationIDs {
689689
var totalCount = 0
690-
for i in stride(from: 0, to: conversationIDs.count, by: Self.maxCountOfHostParameter) {
691-
let endIndex = min(i + Self.maxCountOfHostParameter, conversationIDs.count)
690+
for i in stride(from: 0, to: conversationIDs.count, by: Self.strideForDeviceTransfer) {
691+
let endIndex = min(i + Self.strideForDeviceTransfer, conversationIDs.count)
692692
let ids = Array(conversationIDs[i..<endIndex]).joined(separator: "', '")
693693
let sql = "SELECT COUNT(*) FROM conversations WHERE conversation_id in ('\(ids)')"
694694
let count: Int? = db.select(with: sql)

‎MixinServices/MixinServices/Database/User/DAO/MessageDAO.swift‎

Lines changed: 21 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -980,43 +980,36 @@ extension MessageDAO {
980980
silentNotification: silentNotification)
981981
}
982982

983-
public func messages(limit: Int, after messageId: String?, matching conversationIDs: [String]?, sinceDate date: String?) -> [Message] {
983+
public func messages(limit: Int, after rowID: Int, matching conversationIDs: [String]?) -> [Message] {
984984
if let conversationIDs {
985985
var totalMessages = [Message]()
986-
for i in stride(from: 0, to: conversationIDs.count, by: Self.maxCountOfHostParameter) {
987-
let endIndex = min(i + Self.maxCountOfHostParameter, conversationIDs.count)
986+
for i in stride(from: 0, to: conversationIDs.count, by: Self.strideForDeviceTransfer) {
987+
let endIndex = min(i + Self.strideForDeviceTransfer, conversationIDs.count)
988988
let ids = Array(conversationIDs[i..<endIndex]).joined(separator: "', '")
989-
var sql = "SELECT * FROM messages WHERE conversation_id in ('\(ids)')"
990-
if let date {
991-
sql += " AND created_at >= '\(date)'"
992-
}
993-
if let messageId {
994-
sql += " AND ROWID > IFNULL((SELECT ROWID FROM messages WHERE id = '\(messageId)'), 0)"
995-
}
996-
sql += " ORDER BY ROWID LIMIT ?"
997-
let messages: [Message] = db.select(with: sql, arguments: [limit])
989+
let sql = "SELECT * FROM messages WHERE ROWID > ? AND conversation_id in ('\(ids)') ORDER BY ROWID LIMIT ?"
990+
let messages: [Message] = db.select(with: sql, arguments: [rowID, limit])
998991
totalMessages += messages
999992
}
1000993
return totalMessages
1001994
} else {
1002-
var sql = "SELECT * FROM messages"
1003-
if let date {
1004-
sql += " WHERE created_at >= '\(date)'"
1005-
}
1006-
if let messageId {
1007-
sql += date == nil ? " WHERE " : " AND "
1008-
sql += "ROWID > IFNULL((SELECT ROWID FROM messages WHERE id = '\(messageId)'), 0)"
1009-
}
1010-
sql += " ORDER BY ROWID LIMIT ?"
1011-
return db.select(with: sql, arguments: [limit])
995+
let sql = "SELECT * FROM messages WHERE ROWID > ? ORDER BY ROWID LIMIT ?"
996+
return db.select(with: sql, arguments: [rowID, limit])
1012997
}
1013998
}
1014999

1000+
public func messageRowID(createdAt: String) -> Int? {
1001+
db.select(with: "SELECT ROWID FROM messages WHERE created_at >= ? LIMIT 1", arguments: [createdAt])
1002+
}
1003+
1004+
public func messageRowID(messageID: String) -> Int? {
1005+
db.select(with: "SELECT ROWID FROM messages WHERE id = ?", arguments: [messageID])
1006+
}
1007+
10151008
public func messagesCount(matching conversationIDs: [String]?, sinceDate date: String?) -> Int {
10161009
if let conversationIDs {
10171010
var totalCount = 0
1018-
for i in stride(from: 0, to: conversationIDs.count, by: Self.maxCountOfHostParameter) {
1019-
let endIndex = min(i + Self.maxCountOfHostParameter, conversationIDs.count)
1011+
for i in stride(from: 0, to: conversationIDs.count, by: Self.strideForDeviceTransfer) {
1012+
let endIndex = min(i + Self.strideForDeviceTransfer, conversationIDs.count)
10201013
let ids = Array(conversationIDs[i..<endIndex]).joined(separator: "', '")
10211014
var sql = "SELECT COUNT(*) FROM messages WHERE conversation_id in ('\(ids)')"
10221015
if let date {
@@ -1040,8 +1033,8 @@ extension MessageDAO {
10401033
let categories = MessageCategory.allMediaCategories.map(\.rawValue).joined(separator: "', '")
10411034
if let conversationIDs {
10421035
var totalCount = 0
1043-
for i in stride(from: 0, to: conversationIDs.count, by: Self.maxCountOfHostParameter) {
1044-
let endIndex = min(i + Self.maxCountOfHostParameter, conversationIDs.count)
1036+
for i in stride(from: 0, to: conversationIDs.count, by: Self.strideForDeviceTransfer) {
1037+
let endIndex = min(i + Self.strideForDeviceTransfer, conversationIDs.count)
10451038
let ids = Array(conversationIDs[i..<endIndex]).joined(separator: "', '")
10461039
var sql = "SELECT COUNT(*) FROM messages WHERE conversation_id in ('\(ids)') AND category in ('\(categories)')"
10471040
let count: Int? = db.select(with: sql)
@@ -1059,8 +1052,8 @@ extension MessageDAO {
10591052
let categories = MessageCategory.transcriptCategories.map(\.rawValue).joined(separator: "', '")
10601053
if let conversationIDs {
10611054
var totalCount = 0
1062-
for i in stride(from: 0, to: conversationIDs.count, by: Self.maxCountOfHostParameter) {
1063-
let endIndex = min(i + Self.maxCountOfHostParameter, conversationIDs.count)
1055+
for i in stride(from: 0, to: conversationIDs.count, by: Self.strideForDeviceTransfer) {
1056+
let endIndex = min(i + Self.strideForDeviceTransfer, conversationIDs.count)
10641057
let ids = Array(conversationIDs[i..<endIndex]).joined(separator: "', '")
10651058
var sql = "SELECT COUNT(*) FROM messages WHERE conversation_id in ('\(ids)') AND category in ('\(categories)')"
10661059
if let date {

‎MixinServices/MixinServices/Database/User/DAO/MessageMentionDAO.swift‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ public final class MessageMentionDAO: UserDatabaseDAO {
1818
public func messageMentions(limit: Int, after messageId: String?, matching conversationIDs: [String]?) -> [MessageMention] {
1919
if let conversationIDs {
2020
var totalMessageMentions = [MessageMention]()
21-
for i in stride(from: 0, to: conversationIDs.count, by: Self.maxCountOfHostParameter) {
22-
let endIndex = min(i + Self.maxCountOfHostParameter, conversationIDs.count)
21+
for i in stride(from: 0, to: conversationIDs.count, by: Self.strideForDeviceTransfer) {
22+
let endIndex = min(i + Self.strideForDeviceTransfer, conversationIDs.count)
2323
let ids = Array(conversationIDs[i..<endIndex]).joined(separator: "', '")
2424
var sql = "SELECT * FROM message_mentions WHERE conversation_id in ('\(ids)')"
2525
if let messageId {
@@ -43,8 +43,8 @@ public final class MessageMentionDAO: UserDatabaseDAO {
4343
public func messageMentionsCount(matching conversationIDs: [String]?) -> Int {
4444
if let conversationIDs {
4545
var totalCount = 0
46-
for i in stride(from: 0, to: conversationIDs.count, by: Self.maxCountOfHostParameter) {
47-
let endIndex = min(i + Self.maxCountOfHostParameter, conversationIDs.count)
46+
for i in stride(from: 0, to: conversationIDs.count, by: Self.strideForDeviceTransfer) {
47+
let endIndex = min(i + Self.strideForDeviceTransfer, conversationIDs.count)
4848
let ids = Array(conversationIDs[i..<endIndex]).joined(separator: "', '")
4949
let sql = "SELECT COUNT(*) FROM message_mentions WHERE conversation_id in ('\(ids)')"
5050
let count: Int? = db.select(with: sql)

‎MixinServices/MixinServices/Database/User/DAO/ParticipantDAO.swift‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -178,8 +178,8 @@ public final class ParticipantDAO: UserDatabaseDAO {
178178
public func participants(limit: Int, after conversationId: String?, with userId: String?, matching conversationIDs: [String]?) -> [Participant] {
179179
if let conversationIDs {
180180
var totalParticipants = [Participant]()
181-
for i in stride(from: 0, to: conversationIDs.count, by: Self.maxCountOfHostParameter) {
182-
let endIndex = min(i + Self.maxCountOfHostParameter, conversationIDs.count)
181+
for i in stride(from: 0, to: conversationIDs.count, by: Self.strideForDeviceTransfer) {
182+
let endIndex = min(i + Self.strideForDeviceTransfer, conversationIDs.count)
183183
let ids = Array(conversationIDs[i..<endIndex]).joined(separator: "', '")
184184
var sql = "SELECT * FROM participants WHERE conversation_id in ('\(ids)')"
185185
if let conversationId, let userId {
@@ -203,8 +203,8 @@ public final class ParticipantDAO: UserDatabaseDAO {
203203
public func participantsCount(matching conversationIDs: [String]?) -> Int {
204204
if let conversationIDs {
205205
var totalCount = 0
206-
for i in stride(from: 0, to: conversationIDs.count, by: Self.maxCountOfHostParameter) {
207-
let endIndex = min(i + Self.maxCountOfHostParameter, conversationIDs.count)
206+
for i in stride(from: 0, to: conversationIDs.count, by: Self.strideForDeviceTransfer) {
207+
let endIndex = min(i + Self.strideForDeviceTransfer, conversationIDs.count)
208208
let ids = Array(conversationIDs[i..<endIndex]).joined(separator: "', '")
209209
let sql = "SELECT COUNT(*) FROM participants WHERE conversation_id in ('\(conversationIDs)')"
210210
let count: Int? = db.select(with: sql)

‎MixinServices/MixinServices/Database/User/DAO/PinMessageDAO.swift‎

Lines changed: 17 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -113,43 +113,28 @@ public final class PinMessageDAO: UserDatabaseDAO {
113113
where: PinMessage.column(of: .conversationId) == conversationId)
114114
}
115115

116-
public func pinMessages(limit: Int, after messageId: String?, matching conversationIDs: [String]?, sinceDate date: String?) -> [PinMessage] {
116+
public func pinMessages(limit: Int, after rowID: Int, matching conversationIDs: [String]?) -> [PinMessage] {
117117
if let conversationIDs {
118118
var totalPinMessages = [PinMessage]()
119-
for i in stride(from: 0, to: conversationIDs.count, by: Self.maxCountOfHostParameter) {
120-
let endIndex = min(i + Self.maxCountOfHostParameter, conversationIDs.count)
119+
for i in stride(from: 0, to: conversationIDs.count, by: Self.strideForDeviceTransfer) {
120+
let endIndex = min(i + Self.strideForDeviceTransfer, conversationIDs.count)
121121
let ids = Array(conversationIDs[i..<endIndex]).joined(separator: "', '")
122-
var sql = "SELECT * FROM pin_messages WHERE conversation_id in ('\(ids)')"
123-
if let date {
124-
sql += " AND created_at >= '\(date)'"
125-
}
126-
if let messageId {
127-
sql += " AND ROWID > IFNULL((SELECT ROWID FROM pin_messages WHERE message_id = '\(messageId)'), 0)"
128-
}
129-
sql += " ORDER BY ROWID LIMIT ?"
130-
let pinMessages: [PinMessage] = db.select(with: sql, arguments: [limit])
122+
let sql = "SELECT * FROM pin_messages WHERE ROWID > ? AND conversation_id in ('\(ids)') ORDER BY ROWID LIMIT ?"
123+
let pinMessages: [PinMessage] = db.select(with: sql, arguments: [rowID, limit])
131124
totalPinMessages += pinMessages
132125
}
133126
return totalPinMessages
134127
} else {
135-
var sql = "SELECT * FROM pin_messages"
136-
if let date {
137-
sql += " WHERE created_at >= '\(date)'"
138-
}
139-
if let messageId {
140-
sql += date == nil ? " WHERE " : " AND "
141-
sql += "ROWID > IFNULL((SELECT ROWID FROM pin_messages WHERE message_id = '\(messageId)'), 0)"
142-
}
143-
sql += " ORDER BY ROWID LIMIT ?"
144-
return db.select(with: sql, arguments: [limit])
128+
let sql = "SELECT * FROM pin_messages WHERE ROWID > ? ORDER BY ROWID LIMIT ?"
129+
return db.select(with: sql, arguments: [rowID, limit])
145130
}
146131
}
147132

148133
public func pinMessagesCount(matching conversationIDs: [String]?, sinceDate date: String?) -> Int {
149134
if let conversationIDs {
150135
var totalCount = 0
151-
for i in stride(from: 0, to: conversationIDs.count, by: Self.maxCountOfHostParameter) {
152-
let endIndex = min(i + Self.maxCountOfHostParameter, conversationIDs.count)
136+
for i in stride(from: 0, to: conversationIDs.count, by: Self.strideForDeviceTransfer) {
137+
let endIndex = min(i + Self.strideForDeviceTransfer, conversationIDs.count)
153138
let ids = Array(conversationIDs[i..<endIndex]).joined(separator: "', '")
154139
var sql = "SELECT COUNT(*) FROM pin_messages WHERE conversation_id in ('\(ids)')"
155140
if let date {
@@ -169,6 +154,14 @@ public final class PinMessageDAO: UserDatabaseDAO {
169154
}
170155
}
171156

157+
public func messageRowID(createdAt: String) -> Int? {
158+
db.select(with: "SELECT ROWID FROM pin_messages WHERE created_at >= ? LIMIT 1", arguments: [createdAt])
159+
}
160+
161+
public func messageRowID(messageID: String) -> Int? {
162+
db.select(with: "SELECT ROWID FROM pin_messages WHERE message_id = ?", arguments: [messageID])
163+
}
164+
172165
public func save(pinMessage: PinMessage) {
173166
db.save(pinMessage)
174167
}

‎MixinServices/MixinServices/Database/User/UserDatabaseDAO.swift‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,13 @@ public class UserDatabaseDAO {
66
UserDatabase.current
77
}
88

9-
public static let maxCountOfHostParameter = 999
9+
}
10+
11+
extension UserDatabaseDAO {
12+
13+
// To prevent excessive memory allocations,
14+
// the maximum value of a host parameter number is SQLITE_MAX_VARIABLE_NUMBER,
15+
// which defaults to 999 for SQLite versions prior to 3.32.0 (2020-05-22) or 32766 for SQLite versions after 3.32.0.
16+
public static let strideForDeviceTransfer = 900
1017

1118
}

0 commit comments

Comments
 (0)