Merge pull request #35 from filipw/feature/0.8.3

updated to candle 0.8.3 and added support for Phi-4

Author: filipw

.github/workflows/ci.yml

@@ -29,7 +29,7 @@ jobs:
         
     - name: Upload default artifacts
       if: github.event_name != 'pull_request'
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
          name: native-library-${{ matrix.os }}
          path: |
@@ -39,7 +39,7 @@ jobs:
     
     - name: Upload bindings
       if: matrix.os == 'ubuntu-latest' && github.event_name != 'pull_request'
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
          name: language-bindings
          path: |
@@ -63,7 +63,7 @@ jobs:
       run: rm -rf packages/swift/Strathweb.Phi.Engine/.build
     - name: Upload artifact
       if: github.event_name != 'pull_request'
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: Strathweb.Phi.Engine-SwiftPackage
         path: packages/swift/Strathweb.Phi.Engine
@@ -79,22 +79,22 @@ jobs:
       with:
         dotnet-version: '9.0.x'
     - name: Download Linux artifacts
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: native-library-ubuntu-latest
         path: artifacts/linux
     - name: Download macOS artifacts
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: native-library-macos-15
         path: artifacts/macos
     - name: Download Windows artifacts
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: native-library-windows-latest
         path: artifacts/windows
     - name: Download bindings
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: language-bindings
         path: artifacts/bindings
@@ -113,17 +113,17 @@ jobs:
         dotnet build -c Release
         dotnet pack -c Release
     - name: Upload NuGet package (Strathweb.Phi.Engine)
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: Strathweb.Phi.Engine
         path: packages/csharp/Strathweb.Phi.Engine/bin/Release/*.nupkg
     - name: Upload NuGet package (Strathweb.Phi.Engine.AutoGen)
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: Strathweb.Phi.Engine.AutoGen
         path: packages/csharp/Strathweb.Phi.Engine.AutoGen/bin/Release/*.nupkg
     - name: Upload NuGet package (Strathweb.Phi.Engine.Microsoft.Extensions.AI)
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: Strathweb.Phi.Engine.Microsoft.Extensions.AI
         path: packages/csharp/Strathweb.Phi.Engine.Microsoft.Extensions.AI/bin/Release/*.nupkg

README.md

@@ -1,6 +1,6 @@
 # Strathweb Phi Engine
 
-A cross-platform library for running Microsoft's [Phi-3](https://azure.microsoft.com/en-us/blog/introducing-phi-3-redefining-whats-possible-with-slms/) locally using [candle](https://github.com/huggingface/candle) in GGUF (quantized) and safe tensors (full models) format.
+A cross-platform library for running Microsoft's [Phi-3](https://azure.microsoft.com/en-us/blog/introducing-phi-3-redefining-whats-possible-with-slms/) locally using [candle](https://github.com/huggingface/candle) in GGUF and safe tensors format.
 
 ## Supported platforms
 

samples/swift/main.swift

@@ -1,56 +1,100 @@
 import Foundation
 
-let isQuantizedMode = CommandLine.arguments.contains("--quantized")
+let (isGgufMode, isPhi4, isCpuMode) = (
+    CommandLine.arguments.contains("--gguf"),
+    CommandLine.arguments.contains("--phi-4"),
+    CommandLine.arguments.contains("--cpu")
+)
 
-if isQuantizedMode {
-    print(" 🍃 Quantized mode is enabled.")
-} else {
-    print(" 💪 Safe tensors mode is enabled.")
-}
+let formatMode = isGgufMode ? " 🍃 GGUF" : " 💪 Safe tensors"
+let modelMode = isPhi4 ? " 🚀 Phi-4" : " 🚗 Phi-3"
 
-let modelProvider = isQuantizedMode ? 
-    PhiModelProvider.huggingFaceGguf(modelRepo: "microsoft/Phi-3-mini-4k-instruct-gguf", modelFileName: "Phi-3-mini-4k-instruct-q4.gguf", modelRevision: "main") : 
-    PhiModelProvider.huggingFace(modelRepo: "microsoft/Phi-3-mini-4k-instruct", modelRevision: "main")
+print("\(formatMode) mode is enabled.\n\(modelMode) mode is enabled.")
+
+let modelProvider = switch (isGgufMode, isPhi4) {
+    case (true, true):
+        PhiModelProvider.huggingFaceGguf(
+            modelRepo: "microsoft/phi-4-gguf",
+            modelFileName: "phi-4-q4.gguf",
+            modelRevision: "main"
+        )
+    case (true, false):
+        PhiModelProvider.huggingFaceGguf(
+            modelRepo: "microsoft/Phi-3-mini-4k-instruct-gguf",
+            modelFileName: "Phi-3-mini-4k-instruct-q4.gguf",
+            modelRevision: "main"
+        )
+    case (false, true):
+        PhiModelProvider.huggingFace(
+            modelRepo: "microsoft/phi-4",
+            modelRevision: "main"
+        )
+    case (false, false):
+        PhiModelProvider.huggingFace(
+            modelRepo: "microsoft/Phi-3-mini-4k-instruct",
+            modelRevision: "main"
+        )
+}
 
 let inferenceOptionsBuilder = InferenceOptionsBuilder()
 try! inferenceOptionsBuilder.withTemperature(temperature: 0.9)
 try! inferenceOptionsBuilder.withSeed(seed: 146628346)
+if isPhi4 {
+    try! inferenceOptionsBuilder.withChatFormat(chatFormat: ChatFormat.chatMl)
+}
 let inferenceOptions = try! inferenceOptionsBuilder.build()
 
 let cacheDir = FileManager.default.currentDirectoryPath.appending("/.cache")
 
 class ModelEventsHandler: PhiEventHandler {
-    func onInferenceStarted() {}
-
-    func onInferenceEnded() {}
-
+    func onInferenceStarted() {
+        print(" ℹ️ Inference started...")
+    }
+    func onInferenceEnded() {
+        print("\n ℹ️ Inference ended.")
+    }
     func onInferenceToken(token: String) {
         print(token, terminator: "")
     }
-
     func onModelLoaded() {
         print("""
- 🧠 Model loaded!
-****************************************
-""")
+         🧠 Model loaded!
+        ****************************************
+        """)
     }
 }
 
 let modelBuilder = PhiEngineBuilder()
 try! modelBuilder.withEventHandler(eventHandler: BoxedPhiEventHandler(handler: ModelEventsHandler()))
-let gpuEnabled = try! modelBuilder.tryUseGpu()
 try! modelBuilder.withModelProvider(modelProvider: modelProvider)
 
-let model = try! modelBuilder.buildStateful(cacheDir: cacheDir, systemInstruction: "You are a hockey poet. Be brief and polite.")
+if isPhi4 {
+    try! modelBuilder.withTokenizerProvider(tokenizerProvider: .huggingFace(
+        tokenizerRepo: "microsoft/phi-4",
+        tokenizerFileName: "tokenizer.json"
+    ))
+}
+
+if !isCpuMode {
+    let gpuEnabled = try! modelBuilder.tryUseGpu()
+    print(gpuEnabled ? " 🎮 GPU mode enabled." : " 💻 Tried GPU, but falling back to CPU.")
+} else {
+    print(" 💻 CPU mode enabled.")
+}
 
-// Run inference
-let result = try! model.runInference(promptText: "Write a haiku about ice hockey", inferenceOptions: inferenceOptions)
+let model = try! modelBuilder.buildStateful(
+    cacheDir: cacheDir,
+    systemInstruction: "You are a hockey poet. Be brief and polite."
+)
 
-print("""
+let result = try! model.runInference(
+    promptText: "Write a haiku about ice hockey",
+    inferenceOptions: inferenceOptions
+)
 
+print("""
 ****************************************
  📝 Tokens Generated: \(result.tokenCount)
  🖥️ Tokens per second: \(result.tokensPerSecond)
  ⏱️ Duration: \(result.duration)s
- 🏎️ GPU enabled: \(gpuEnabled)
 """)

samples/swift/run.sh

@@ -15,7 +15,9 @@ swiftc *.swift \
     -framework MetalPerformanceShaders \
     -framework SystemConfiguration \
     -lc++ \
-    -O \
+    -O -whole-module-optimization \
+    -cross-module-optimization \
+    -enforce-exclusivity=unchecked \
     -o phi-engine-swift-sample
 
 ./phi-engine-swift-sample "$@"