chore(CoreMLPredictionsPlugin): Performing simulator work on CPU and surfacing errors

Author: jcjimenez

AmplifyPlugins/Predictions/CoreMLPredictionsPlugin/CoreMLPredictionsPlugin+ClientBehavior.swift

@@ -43,7 +43,7 @@ extension CoreMLPredictionsPlugin {
                 .detectLabels(.moderation, _):
             throw predictionsError
         case let .detectText(lift):
-            guard  let result = coreMLVisionAdapter.detectText(image) else {
+            guard  let result = try coreMLVisionAdapter.detectText(image) else {
                 let errorDescription = CoreMLPluginErrorString.detectTextNoResult.errorDescription
                 let recovery = CoreMLPluginErrorString.detectTextNoResult.recoverySuggestion
                 let predictionsError = PredictionsError.service(
@@ -53,7 +53,7 @@ extension CoreMLPredictionsPlugin {
             }
             return lift.outputSpecificToGeneric(result)
         case let .detectLabels(_, lift):
-            guard let result = coreMLVisionAdapter.detectLabels(image) else {
+            guard let result = try coreMLVisionAdapter.detectLabels(image) else {
                 let errorDescription = CoreMLPluginErrorString.detectLabelsNoResult.errorDescription
                 let recovery = CoreMLPluginErrorString.detectLabelsNoResult.recoverySuggestion
                 let predictionsError = PredictionsError.service(
@@ -90,14 +90,18 @@ extension CoreMLPredictionsPlugin {
             )
             let stream = AsyncThrowingStream<Predictions.Convert.SpeechToText.Result, Error> { continuation in
                 Task {
-                    let result = try await coreMLSpeech.getTranscription(
-                        request.speechToText
-                    )
-                    continuation.yield(
-                        .init(transcription: result.bestTranscription.formattedString)
-                    )
-                    if result.isFinal {
-                        continuation.finish()
+                    do {
+                        let result = try await coreMLSpeech.getTranscription(
+                            request.speechToText
+                        )
+                        continuation.yield(
+                            .init(transcription: result.bestTranscription.formattedString)
+                        )
+                        if result.isFinal {
+                            continuation.finish()
+                        }
+                    } catch {
+                        continuation.yield(with: .failure(error))
                     }
                 }
             }

AmplifyPlugins/Predictions/CoreMLPredictionsPlugin/Dependency/CoreMLSpeechAdapter.swift

@@ -12,12 +12,24 @@ class CoreMLSpeechAdapter: CoreMLSpeechBehavior {
     func getTranscription(_ audioData: URL) async throws -> SFSpeechRecognitionResult {
         let request = SFSpeechURLRecognitionRequest(url: audioData)
         request.requiresOnDeviceRecognition = true
-        let recognizer = SFSpeechRecognizer()
+        guard let recognizer = SFSpeechRecognizer() else {
+            throw PredictionsError.client(
+                .init(
+                    description: "CoreML Service is not configured",
+                    recoverySuggestion: "Ensure that dictation is enabled on your device."
+                )
+            )
+        }
 
         let result = try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<SFSpeechRecognitionResult, Error>) in
-            recognizer?.recognitionTask(
+            recognizer.recognitionTask(
                 with: request,
-                resultHandler: { result, _ in
+                resultHandler: { (result, error) in
+                    if let error = error {
+                        continuation.resume(with: .failure(error))
+                        return
+                    }
+
                     guard let result = result else {
                         continuation.resume(with: .failure(
                             PredictionsError.client(

AmplifyPlugins/Predictions/CoreMLPredictionsPlugin/Dependency/CoreMLVisionAdapter.swift

@@ -10,11 +10,14 @@ import Vision
 
 class CoreMLVisionAdapter: CoreMLVisionBehavior {
 
-    public func detectLabels(_ imageURL: URL) -> Predictions.Identify.Labels.Result? {
+    func detectLabels(_ imageURL: URL) throws -> Predictions.Identify.Labels.Result? {
         var labelsResult = [Predictions.Label]()
         let handler = VNImageRequestHandler(url: imageURL, options: [:])
         let request = VNClassifyImageRequest()
-        try? handler.perform([request])
+#if targetEnvironment(simulator)
+        request.usesCPUOnly = true
+#endif
+        try handler.perform([request])
         guard let observations = request.results else { return nil }
 
         let categories = observations.filter { $0.hasMinimumRecall(0.01, forPrecision: 0.9) }
@@ -26,11 +29,14 @@ class CoreMLVisionAdapter: CoreMLVisionBehavior {
         return Predictions.Identify.Labels.Result(labels: labelsResult)
     }
 
-    public func detectText(_ imageURL: URL) -> Predictions.Identify.Text.Result? {
+    public func detectText(_ imageURL: URL) throws -> Predictions.Identify.Text.Result? {
         let handler = VNImageRequestHandler(url: imageURL, options: [:])
         let request = VNRecognizeTextRequest()
+#if targetEnvironment(simulator)
+        request.usesCPUOnly = true
+#endif
         request.recognitionLevel = .accurate
-        try? handler.perform([request])
+        try handler.perform([request])
         guard let observations = request.results else { return nil }
 
         var identifiedLines = [Predictions.IdentifiedLine]()
@@ -62,10 +68,13 @@ class CoreMLVisionAdapter: CoreMLVisionBehavior {
         )
     }
 
-    func detectEntities(_ imageURL: URL) -> Predictions.Identify.Entities.Result? {
+    func detectEntities(_ imageURL: URL) throws -> Predictions.Identify.Entities.Result? {
         let handler = VNImageRequestHandler(url: imageURL, options: [:])
         let faceLandmarksRequest = VNDetectFaceLandmarksRequest()
-        try? handler.perform([faceLandmarksRequest])
+#if targetEnvironment(simulator)
+        faceLandmarksRequest.usesCPUOnly = true
+#endif
+        try handler.perform([faceLandmarksRequest])
         guard let observations = faceLandmarksRequest.results else { return nil }
 
         var entities: [Predictions.Entity] = []

AmplifyPlugins/Predictions/CoreMLPredictionsPlugin/Dependency/CoreMLVisionBehavior.swift

@@ -9,7 +9,7 @@ import Foundation
 import Amplify
 
 protocol CoreMLVisionBehavior: AnyObject {
-    func detectLabels(_ imageURL: URL) -> Predictions.Identify.Labels.Result?
-    func detectText(_ imageURL: URL) -> Predictions.Identify.Text.Result?
-    func detectEntities(_ imageURL: URL) -> Predictions.Identify.Entities.Result?
+    func detectLabels(_ imageURL: URL) throws -> Predictions.Identify.Labels.Result?
+    func detectText(_ imageURL: URL) throws -> Predictions.Identify.Text.Result?
+    func detectEntities(_ imageURL: URL) throws -> Predictions.Identify.Entities.Result?
 }

AmplifyPlugins/Predictions/Tests/PredictionsHostApp/CoreMLPredictionsPluginIntegrationTests/CoreMLPredictionsPluginIntegrationTest.swift

@@ -8,17 +8,44 @@
 import XCTest
 import Amplify
 import XCTest
+import AVFoundation
 
 class CoreMLPredictionsPluginIntegrationTest: AWSPredictionsPluginTestBase {
+
     func testIdentify() async throws {
         let testBundle = Bundle(for: type(of: self))
         let url = try XCTUnwrap(testBundle.url(forResource: "people", withExtension: "jpg"))
 
-        let result = try await Amplify.Predictions.identify(
+        let result: Predictions.Identify.Labels.Result = try await Amplify.Predictions.identify(
             .labels(type: .all),
             in: url
         )
 
-        XCTAssertNotNil(result, "Result should contain value")
+        XCTAssertEqual(result.labels.count, 0, String(describing: result))
+        XCTAssertNil(result.unsafeContent, String(describing: result))
+    }
+
+    func testConvertSpeechToText() async throws {
+        let testBundle = Bundle(for: type(of: self))
+        let url = try XCTUnwrap(testBundle.url(forResource: "audio", withExtension: "wav"))
+
+        let options = Predictions.Convert.SpeechToText.Options(
+            defaultNetworkPolicy: .auto,
+            language: .usEnglish
+        )
+
+        let result = try await Amplify.Predictions.convert(
+            .speechToText(url: url), options: options
+        )
+        let responses = result.map(\.transcription)
+
+        do {
+            for try await response in responses {
+                XCTFail("Expecting failure but got: \(response)")
+            }
+        } catch let error as NSError {
+            XCTAssertEqual(error.code, 201)
+            XCTAssertEqual(error.localizedDescription, "Siri and Dictation are disabled")
+        }
     }
 }