Centralize file download patterns

Author: DePasqualeOrg

Libraries/BenchmarkHelpers/BenchmarkHelpers.swift

@@ -196,8 +196,6 @@ public func loadDecodingBenchmarkText(
     )
 }
 
-private let tokenizerDownloadPatterns = ["*.json", "*.jinja"]
-
 private func resolveTokenizerDirectory(
     from downloader: any Downloader,
     configuration: MLXLMCommon.ModelConfiguration,
@@ -444,7 +442,7 @@ public func benchmarkDownloadCacheHit(
     modelId: String = "mlx-community/Qwen3-0.6B-4bit",
     runs: Int = BenchmarkDefaults.downloadRuns
 ) async throws -> BenchmarkStats {
-    let patterns = ["*.safetensors", "*.json", "*.jinja"]
+    let patterns = modelDownloadPatterns
 
     // Warm-up: ensure the model is cached
     _ = try await downloader.download(

Libraries/MLXEmbedders/Load.swift

@@ -83,7 +83,7 @@ func resolveDirectories(
     case .id(let id, let revision):
         modelDirectory = try await downloader.download(
             id: id, revision: revision,
-            matching: ["*.safetensors", "*.json"],
+            matching: modelDownloadPatterns,
             useLatest: useLatest,
             progressHandler: progressHandler)
     case .directory(let directory):
@@ -95,7 +95,7 @@ func resolveDirectories(
     case .id(let id, let revision):
         tokenizerDirectory = try await downloader.download(
             id: id, revision: revision,
-            matching: ["*.json"],
+            matching: tokenizerDownloadPatterns,
             useLatest: useLatest,
             progressHandler: { _ in })
     case .directory(let directory):

Libraries/MLXLMCommon/Downloader.swift

@@ -21,7 +21,7 @@ public protocol Downloader: Sendable {
     ///   - revision: Optional revision (branch, tag, commit hash, version number).
     ///     Providers without versioning receive `nil`.
     ///   - patterns: Glob patterns to filter which files to download
-    ///     (e.g. `["*.safetensors", "*.json"]`)
+    ///     (e.g. `["*.safetensors", "*.json", "*.jinja"]`)
     ///   - useLatest: When `true`, check the provider for updates even if a cached
     ///     version exists. When `false`, return the cached version if available.
     ///   - progressHandler: Callback for download progress

Libraries/MLXLMCommon/ModelFactory.swift

@@ -2,6 +2,10 @@
 
 import Foundation
 
+/// File patterns required to resolve a tokenizer without downloading model weights.
+package let tokenizerDownloadPatterns = ["*.json", "*.jinja"]
+package let modelDownloadPatterns = ["*.safetensors"] + tokenizerDownloadPatterns
+
 public enum ModelFactoryError: LocalizedError {
     case unsupportedModelType(String)
     case unsupportedProcessorType(String)
@@ -197,7 +201,7 @@ public func resolve(
     case .id(let id, let revision):
         modelDirectory = try await downloader.download(
             id: id, revision: revision,
-            matching: ["*.safetensors", "*.json", "*.jinja"],
+            matching: modelDownloadPatterns,
             useLatest: useLatest,
             progressHandler: progressHandler)
     case .directory(let directory):
@@ -209,7 +213,7 @@ public func resolve(
     case .id(let id, let revision):
         tokenizerDirectory = try await downloader.download(
             id: id, revision: revision,
-            matching: ["*.json", "*.jinja"],
+            matching: tokenizerDownloadPatterns,
             useLatest: useLatest,
             progressHandler: { _ in })
     case .directory(let directory):

Tests/MLXLMTests/ResolveTests.swift

@@ -67,6 +67,7 @@ private final class LockIsolated<Value: Sendable>: @unchecked Sendable {
         #expect(downloader.calls.value.count == 1)
         #expect(downloader.calls.value[0].id == "org/model")
         #expect(downloader.calls.value[0].revision == "abc123")
+        #expect(downloader.calls.value[0].patterns.contains("*.jinja"))
 
         // No separate tokenizer download, so both point to the model directory.
         #expect(resolved.modelDirectory == resolved.tokenizerDirectory)
@@ -91,6 +92,7 @@ private final class LockIsolated<Value: Sendable>: @unchecked Sendable {
         // Tokenizer download uses nil revision (provider default).
         #expect(downloader.calls.value[1].id == "org/tokenizer")
         #expect(downloader.calls.value[1].revision == nil)
+        #expect(downloader.calls.value[1].patterns.contains("*.jinja"))
 
         // Model and tokenizer come from different repos, so directories differ.
         #expect(resolved.modelDirectory != resolved.tokenizerDirectory)
@@ -113,6 +115,7 @@ private final class LockIsolated<Value: Sendable>: @unchecked Sendable {
 
         #expect(downloader.calls.value[1].id == "org/tokenizer")
         #expect(downloader.calls.value[1].revision == "tok-v2")
+        #expect(downloader.calls.value[1].patterns.contains("*.jinja"))
 
         // Model and tokenizer come from different repos, so directories differ.
         #expect(resolved.modelDirectory != resolved.tokenizerDirectory)
@@ -150,6 +153,7 @@ private final class LockIsolated<Value: Sendable>: @unchecked Sendable {
         #expect(downloader.calls.value.count == 1)
         #expect(downloader.calls.value[0].id == "org/tokenizer")
         #expect(downloader.calls.value[0].revision == "v3")
+        #expect(downloader.calls.value[0].patterns.contains("*.jinja"))
 
         #expect(resolved.modelDirectory == localDir)
         #expect(resolved.tokenizerDirectory != localDir)