Skip to content

Commit 73b3303

Browse files
shoumikhinshoumikhin
authored
Update LLMTests with new APIs (pytorch#14665)
1 parent 6f9c5b1 commit 73b3303

File tree

6 files changed

+36
-33
lines changed

6 files changed

+36
-33
lines changed

‎extension/benchmark/apple/Benchmark/Benchmark.xcodeproj/project.pbxproj‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88

99
/* Begin PBXBuildFile section */
1010
0314AE3A2E2AAEE700DDE821 /* executorch_llm in Frameworks */ = {isa = PBXBuildFile; productRef = 0314AE392E2AAEE700DDE821 /* executorch_llm */; };
11-
032A73CA2CAFBA8600932D36 /* LLaMATests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 032A73C82CAFBA8600932D36 /* LLaMATests.mm */; };
11+
032A73CA2CAFBA8600932D36 /* LLMTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 032A73C82CAFBA8600932D36 /* LLMTests.mm */; };
1212
0351D9D72CAFC9A200607121 /* Resources in Resources */ = {isa = PBXBuildFile; fileRef = 03C7FA322C8AA24200E6E9AE /* Resources */; };
1313
03B0118E2CAC567900054791 /* DynamicTestCase.m in Sources */ = {isa = PBXBuildFile; fileRef = 03B0118C2CAC567900054791 /* DynamicTestCase.m */; };
1414
03B011912CAD114E00054791 /* ResourceTestCase.m in Sources */ = {isa = PBXBuildFile; fileRef = 03B011902CAD114E00054791 /* ResourceTestCase.m */; };
@@ -35,7 +35,7 @@
3535
/* End PBXContainerItemProxy section */
3636

3737
/* Begin PBXFileReference section */
38-
032A73C82CAFBA8600932D36 /* LLaMATests.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = LLaMATests.mm; sourceTree = "<group>"; };
38+
032A73C82CAFBA8600932D36 /* LLMTests.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = LLMTests.mm; sourceTree = "<group>"; };
3939
037C96A02C8A570B00B3DF38 /* Tests.xctestplan */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = Tests.xctestplan; sourceTree = "<group>"; };
4040
03B0118B2CAC567900054791 /* DynamicTestCase.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = DynamicTestCase.h; sourceTree = "<group>"; };
4141
03B0118C2CAC567900054791 /* DynamicTestCase.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = DynamicTestCase.m; sourceTree = "<group>"; };
@@ -76,12 +76,12 @@
7676
/* End PBXFrameworksBuildPhase section */
7777

7878
/* Begin PBXGroup section */
79-
032A73C92CAFBA8600932D36 /* LLaMA */ = {
79+
032A73C92CAFBA8600932D36 /* LLM */ = {
8080
isa = PBXGroup;
8181
children = (
82-
032A73C82CAFBA8600932D36 /* LLaMATests.mm */,
82+
032A73C82CAFBA8600932D36 /* LLMTests.mm */,
8383
);
84-
path = LLaMA;
84+
path = LLM;
8585
sourceTree = "<group>";
8686
};
8787
03B0118D2CAC567900054791 /* TestUtils */ = {
@@ -127,7 +127,7 @@
127127
03B2D3782C8A515C0046936E /* Tests */ = {
128128
isa = PBXGroup;
129129
children = (
130-
032A73C92CAFBA8600932D36 /* LLaMA */,
130+
032A73C92CAFBA8600932D36 /* LLM */,
131131
03E7E6782CBDC1C900205E71 /* CoreMLTests.mm */,
132132
03B2D3792C8A515C0046936E /* GenericTests.mm */,
133133
037C96A02C8A570B00B3DF38 /* Tests.xctestplan */,
@@ -241,7 +241,7 @@
241241
03B011912CAD114E00054791 /* ResourceTestCase.m in Sources */,
242242
03E7E6792CBDCAE900205E71 /* CoreMLTests.mm in Sources */,
243243
03B2D37A2C8A515C0046936E /* GenericTests.mm in Sources */,
244-
032A73CA2CAFBA8600932D36 /* LLaMATests.mm in Sources */,
244+
032A73CA2CAFBA8600932D36 /* LLMTests.mm in Sources */,
245245
);
246246
runOnlyForDeploymentPostprocessing = 0;
247247
};

‎extension/benchmark/apple/Benchmark/Tests/LLaMA/LLaMATests.mm‎ renamed to ‎extension/benchmark/apple/Benchmark/Tests/LLM/LLMTests.mm‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,10 @@ - (id)copyWithZone:(NSZone *)zone {
4242

4343
@end
4444

45-
@interface LLaMATests : ResourceTestCase
45+
@interface LLMTests : ResourceTestCase
4646
@end
4747

48-
@implementation LLaMATests
48+
@implementation LLMTests
4949

5050
+ (NSArray<NSString *> *)directories {
5151
return @[
@@ -87,9 +87,10 @@ @implementation LLaMATests
8787
for (NSUInteger index = 2; specialTokens.count < 256; ++index) {
8888
[specialTokens addObject:[NSString stringWithFormat:@"<|reserved_special_token_%zu|>", index]];
8989
}
90-
auto __block runner = [[ExecuTorchLLMTextRunner alloc] initWithModelPath:modelPath
91-
tokenizerPath:tokenizerPath
92-
specialTokens:specialTokens];
90+
ExecuTorchLLMTextRunner *__block runner =
91+
[[ExecuTorchLLMTextRunner alloc] initWithModelPath:modelPath
92+
tokenizerPath:tokenizerPath
93+
specialTokens:specialTokens];
9394
NSError *error;
9495
BOOL status = [runner loadWithError:&error];
9596
if (!status) {
@@ -100,12 +101,14 @@ @implementation LLaMATests
100101
[testCase measureWithMetrics:@[ tokensPerSecondMetric, [XCTClockMetric new], [XCTMemoryMetric new] ]
101102
block:^{
102103
tokensPerSecondMetric.tokenCount = 0;
103-
BOOL status = [runner generate:@"Once upon a time"
104-
sequenceLength:50
105-
withTokenCallback:^(NSString *token) {
106-
tokensPerSecondMetric.tokenCount++;
104+
BOOL status = [runner generateWithPrompt:@"Once upon a time"
105+
config:[[ExecuTorchLLMConfig alloc] initWithBlock:^(ExecuTorchLLMConfig *config) {
106+
config.sequenceLength = 50;
107+
}]
108+
tokenCallback:^(NSString *token) {
109+
++tokensPerSecondMetric.tokenCount;
107110
}
108-
error:NULL];
111+
error:NULL];
109112
XCTAssertTrue(status);
110113
}];
111114
},

‎extension/llm/apple/ExecuTorchLLM/Exported/ExecuTorchLLMMultimodalRunner.h‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -193,10 +193,10 @@ __attribute__((objc_subclassing_restricted))
193193
@param error On failure, populated with an NSError explaining the issue.
194194
@return YES if generation completes successfully, NO if an error occurred.
195195
*/
196-
- (BOOL)generate:(NSArray<ExecuTorchLLMMultimodalInput *> *)inputs
197-
config:(ExecuTorchLLMConfig *)config
198-
withTokenCallback:(nullable void (^)(NSString *))callback
199-
error:(NSError **)error
196+
- (BOOL)generateWithInputs:(NSArray<ExecuTorchLLMMultimodalInput *> *)inputs
197+
config:(ExecuTorchLLMConfig *)config
198+
tokenCallback:(nullable void (^)(NSString *))callback
199+
error:(NSError **)error
200200
NS_SWIFT_NAME(generate(_:_:tokenCallback:));
201201

202202
/**

‎extension/llm/apple/ExecuTorchLLM/Exported/ExecuTorchLLMMultimodalRunner.mm‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -162,10 +162,10 @@ - (BOOL)loadWithError:(NSError**)error {
162162
return YES;
163163
}
164164

165-
- (BOOL)generate:(NSArray<ExecuTorchLLMMultimodalInput *> *)inputs
166-
config:(ExecuTorchLLMConfig *)config
167-
withTokenCallback:(nullable void (^)(NSString *))callback
168-
error:(NSError **)error {
165+
- (BOOL)generateWithInputs:(NSArray<ExecuTorchLLMMultimodalInput *> *)inputs
166+
config:(ExecuTorchLLMConfig *)config
167+
tokenCallback:(nullable void (^)(NSString *))callback
168+
error:(NSError **)error {
169169
if (![self loadWithError:error]) {
170170
return NO;
171171
}

‎extension/llm/apple/ExecuTorchLLM/Exported/ExecuTorchLLMTextRunner.h‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,10 +60,10 @@ __attribute__((deprecated("This API is experimental.")))
6060
@param error On failure, populated with an NSError explaining the issue.
6161
@return YES if generation completes successfully, NO if an error occurred.
6262
*/
63-
- (BOOL)generate:(NSString *)prompt
64-
config:(ExecuTorchLLMConfig *)config
65-
withTokenCallback:(nullable void (^)(NSString *token))callback
66-
error:(NSError **)error
63+
- (BOOL)generateWithPrompt:(NSString *)prompt
64+
config:(ExecuTorchLLMConfig *)config
65+
tokenCallback:(nullable void (^)(NSString *token))callback
66+
error:(NSError **)error
6767
NS_SWIFT_NAME(generate(_:_:tokenCallback:));
6868

6969
/**

‎extension/llm/apple/ExecuTorchLLM/Exported/ExecuTorchLLMTextRunner.mm‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -74,10 +74,10 @@ - (BOOL)loadWithError:(NSError**)error {
7474
return YES;
7575
}
7676

77-
- (BOOL)generate:(NSString*)prompt
78-
config:(ExecuTorchLLMConfig *)config
79-
withTokenCallback:(nullable void (^)(NSString*))callback
80-
error:(NSError**)error {
77+
- (BOOL)generateWithPrompt:(NSString*)prompt
78+
config:(ExecuTorchLLMConfig *)config
79+
tokenCallback:(nullable void (^)(NSString*))callback
80+
error:(NSError**)error {
8181
if (![self loadWithError:error]) {
8282
return NO;
8383
}

0 commit comments

Comments
 (0)