Skip to content

Commit f024896

Browse files
committed
Implement prewarm for MLXLanguageModel
1 parent e28bf43 commit f024896

File tree

1 file changed

+45
-0
lines changed

1 file changed

+45
-0
lines changed

Sources/AnyLanguageModel/Models/MLXLanguageModel.swift

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -357,6 +357,51 @@ import Foundation
357357

358358
return LanguageModelSession.ResponseStream(stream: stream)
359359
}
360+
361+
/// Prewarms the model for the given session and optional prompt prefix.
362+
public func prewarm(
363+
for session: LanguageModelSession,
364+
promptPrefix: Prompt?
365+
) {
366+
let modelId = self.modelId
367+
let hub = self.hub
368+
let directory = self.directory
369+
370+
let instructions = session.instructions?.description
371+
let tools = session.tools
372+
373+
Task {
374+
375+
let context = try await loadContext(modelId: modelId, hub: hub, directory: directory)
376+
377+
// Build chat history similar to respond() to prime the cache effectively
378+
var chat: [MLXLMCommon.Chat.Message] = []
379+
380+
// Add system instructions if present
381+
if let instructions, !instructions.isEmpty {
382+
chat.append(.init(role: .system, content: instructions))
383+
}
384+
385+
// Add prompt prefix or minimal user message
386+
let promptText = promptPrefix?.description ?? "."
387+
chat.append(.init(role: .user, content: promptText))
388+
389+
// Convert tools to MLX format
390+
let toolSpecs: [ToolSpec]? =
391+
tools.isEmpty
392+
? nil
393+
: tools.map { convertToolToMLXSpec($0) }
394+
395+
let userInput = MLXLMCommon.UserInput(
396+
chat: chat,
397+
processing: .init(resize: .init(width: 512, height: 512)),
398+
tools: toolSpecs
399+
)
400+
401+
// Prepare input - triggers tokenization and processor initialization
402+
_ = try await context.processor.prepare(input: userInput)
403+
}
404+
}
360405
}
361406

362407
// MARK: - Options Mapping

0 commit comments

Comments
 (0)