99#import < XCTest/XCTest.h>
1010
1111#import < objc/runtime.h>
12+ #import < UIKit/UIDevice.h>
1213
1314#import < executorch/extension/module/module.h>
1415#import < executorch/extension/tensor/tensor.h>
1516
1617using namespace ::executorch::extension;
1718using namespace ::executorch::runtime;
1819
20+ @interface BenchmarkModel : NSObject
21+
22+ // The model name, i.e. stories110M
23+ @property NSString *name;
24+ @property NSString *backend;
25+ @property NSString *quantization;
26+
27+ -(instancetype )init : (NSString *)name
28+ backend : (NSString *)backend
29+ quantization : (NSString *)quantization ;
30+
31+ +(instancetype )initWithModelName : (NSString *)model ;
32+
33+ @end
34+
35+ @implementation BenchmarkModel
36+
37+ -(instancetype )init : (NSString *)name
38+ backend : (NSString *)backend
39+ quantization : (NSString *)quantization {
40+ if (self = [super init ]) {
41+ self.name = name;
42+ self.backend = backend;
43+ self.quantization = quantization;
44+ }
45+ return self;
46+ }
47+
48+ +(instancetype )initWithModelName : (NSString *)model {
49+ NSRegularExpression *regex = [NSRegularExpression regularExpressionWithPattern: @" (\\ w+)_(\\ w+)_(\\ w+)" options: 0 error: nil ];
50+ NSArray * matches = [regex matchesInString: model options: 0 range: NSMakeRange (0 , [model length ])];
51+
52+ for (NSTextCheckingResult * match in matches) {
53+ return [[BenchmarkModel alloc ] init: [model substringWithRange: [match rangeAtIndex: 1 ]]
54+ backend: [model substringWithRange: [match rangeAtIndex: 2 ]]
55+ quantization: [model substringWithRange: [match rangeAtIndex: 3 ]]
56+ ];
57+ }
58+
59+ return [[BenchmarkModel alloc ] init: model
60+ backend: @" "
61+ quantization: @" "
62+ ];
63+ }
64+
65+ @end
66+
67+ @interface DeviceInfo : NSObject
68+
69+ // The metric name, i.e. TPS
70+ @property NSString *device;
71+
72+ // The phone model and Android release version
73+ @property NSString *arch;
74+ @property NSString *os;
75+ // TODO (huydhn): These are not used yet
76+ @property NSNumber *totalMem;
77+ @property NSNumber *availMem;
78+
79+ -(instancetype )init ;
80+
81+ @end
82+
83+ @implementation DeviceInfo
84+
85+ -(instancetype )init {
86+ self.device = [[UIDevice currentDevice ] model ];
87+ self.arch = [[UIDevice currentDevice ] name ];
88+ self.os = [NSString stringWithFormat: @" %@ / %@ " currentDevice ] systemName ], [[UIDevice currentDevice ] systemVersion ]];
89+ self.totalMem = [[NSNumber alloc ] initWithInt: 0 ];
90+ self.availMem = [[NSNumber alloc ] initWithInt: 0 ];
91+ return self;
92+ }
93+
94+ @end
95+
96+ @interface BenchmarkMetric : NSObject
97+
98+ @property BenchmarkModel *benchmarkModel;
99+
100+ // The metric name, i.e. TPS
101+ @property NSString *metric;
102+
103+ // The actual value and the option target value
104+ @property NSNumber *actualValue;
105+ @property NSNumber *targetValue;
106+
107+ @property DeviceInfo *deviceInfo;
108+
109+ -(instancetype )init : (BenchmarkModel*)benchmarkModel
110+ metric : (NSString *)metric
111+ actualValue : (NSNumber *)actualValue
112+ targetValue : (NSNumber *)targetValue
113+ deviceInfo : (DeviceInfo*)deviceInfo ;
114+
115+ @end
116+
117+ @implementation BenchmarkMetric
118+
119+ -(instancetype )init : (BenchmarkModel*)benchmarkModel
120+ metric : (NSString *)metric
121+ actualValue : (NSNumber *)actualValue
122+ targetValue : (NSNumber *)targetValue
123+ deviceInfo : (DeviceInfo*)deviceInfo {
124+
125+ self.benchmarkModel = benchmarkModel;
126+ self.metric = metric;
127+ self.actualValue = actualValue;
128+ self.targetValue = targetValue;
129+ self.deviceInfo = deviceInfo;
130+ return self;
131+ }
132+
133+ @end
134+
19135@interface Tests : XCTestCase
20136@end
21137
@@ -25,6 +141,9 @@ + (void)initialize {
25141 if (self != [self class ]) {
26142 return ;
27143 }
144+ DeviceInfo *deviceInfo = [[DeviceInfo alloc ] init ];
145+ NSMutableArray *benchmarkMetrics = [[NSMutableArray alloc ] init ];
146+
28147 for (NSBundle *bundle in @[
29148 [NSBundle mainBundle ],
30149 [NSBundle bundleForClass: [self class ]],
@@ -50,6 +169,7 @@ + (void)initialize {
50169 .lowercaseString ;
51170 NSString *modelName =
52171 modelPath.lastPathComponent .stringByDeletingPathExtension ;
172+ BenchmarkModel *benchmarkModel = [BenchmarkModel initWithModelName: modelName];
53173
54174 SEL testLoadSelector = NSSelectorFromString ([NSString
55175 stringWithFormat: @" test_load_%@ _%@ "
@@ -61,8 +181,14 @@ + (void)initialize {
61181 ]
62182 options: XCTMeasureOptions.defaultOptions
63183 block: ^{
64- XCTAssertEqual (module ->load_method (" forward"
65- Error::Ok);
184+ auto status = module ->load_method (" forward"
185+ BenchmarkMetric *benchmarkMetric = [[BenchmarkMetric alloc ] init: benchmarkModel
186+ metric: @" load_status"
187+ actualValue: [[NSNumber alloc ] initWithInt: module ->is_loaded () ? 0 : 1 ]
188+ targetValue: 0
189+ deviceInfo: deviceInfo];
190+ [benchmarkMetrics addObject: benchmarkMetric];
191+ XCTAssertEqual (status, Error::Ok);
66192 }];
67193 });
68194 class_addMethod (
0 commit comments