up

Author: metascroy

examples/models/llama/coreml_enumerated_shape.py

@@ -49,7 +49,11 @@ def get_example_inputs(max_batch_size, args, coreml=False, use_enumerated_shapes
 
 
 # Batch with kv cache runs into issues
-max_batch_size = args.max_seq_length
+if args.use_kv_cache and not args.enable_dynamic_shape:
+    max_batch_size = 1
+else:
+    max_batch_size = args.max_seq_length
+
 example_inputs = get_example_inputs(max_batch_size, args)
 
 

extension/benchmark/apple/Benchmark/Tests/CoreMLTests.mm

@@ -85,20 +85,20 @@ @implementation CoreMLTests
                 XCTFail(@"Failed to load model: %@", error.localizedDescription);
                 return;
             }
-            //      NSMutableDictionary *inputs = DummyInputsForModel(model, &error);
-            //      if (error || !inputs) {
-            //        XCTFail(@"Failed to prepare inputs: %@", error.localizedDescription);
-            //        return;
-            //      }
-            //      MLDictionaryFeatureProvider *featureProvider = [[MLDictionaryFeatureProvider alloc] initWithDictionary:inputs error:&error];
-            //      if (error || !featureProvider) {
-            //          if (error) {
-            //              XCTFail(@"Failed to create input provider: %@", error.localizedDescription);
-            //          } else {
-            //              XCTFail(@"Failed with unknown error");
-            //          }
-            //        return;
-            //      }
+            NSMutableDictionary *inputs = DummyInputsForModel(model, &error);
+            if (error || !inputs) {
+                XCTFail(@"Failed to prepare inputs: %@", error.localizedDescription);
+                return;
+            }
+            MLDictionaryFeatureProvider *featureProvider = [[MLDictionaryFeatureProvider alloc] initWithDictionary:inputs error:&error];
+            if (error || !featureProvider) {
+                if (error) {
+                    XCTFail(@"Failed to create input provider: %@", error.localizedDescription);
+                } else {
+                    XCTFail(@"Failed with unknown error");
+                }
+                return;
+            }
 
 
             MLMultiArray *tokensArray1x1 = [[MLMultiArray alloc] initWithShape:@[@1, @1] dataType:MLMultiArrayDataTypeInt32 error:&error];
@@ -122,6 +122,8 @@ @implementation CoreMLTests
                 id<MLFeatureProvider> prediction;
                 for (int i = 0; i < 50; i++) {
                     // prediction = [model predictionFromFeatures:featureProvider usingState:state error:&error];
+
+
                     if (i % 2 == 0) {
                         prediction = [model predictionFromFeatures:features1x128 error:&error];
                     } else {