Re-work

Author: huydhn

extension/apple/Benchmark/Tests/Tests.mm

@@ -17,117 +17,20 @@
 using namespace ::executorch::extension;
 using namespace ::executorch::runtime;
 
-@interface BenchmarkModel : NSObject
-
-// The model name, i.e. stories110M
-@property NSString *name;
-@property NSString *backend;
-@property NSString *quantization;
-
--(instancetype)init: (NSString*)name
-            backend: (NSString*)backend
-       quantization: (NSString*)quantization;
-
-+(instancetype)initWithModelName: (NSString*)model;
-
-@end
-
-@implementation BenchmarkModel
-
--(instancetype)init: (NSString*)name
-            backend: (NSString*)backend
-       quantization: (NSString*)quantization {
-  if (self = [super init]) {
-    self.name = name;
-    self.backend = backend;
-    self.quantization = quantization;
-  }
-  return self;
-}
-
-+(instancetype)initWithModelName: (NSString*)model {
-  NSRegularExpression *regex = [NSRegularExpression regularExpressionWithPattern:@"(\\w+)_(\\w+)_(\\w+)" options:0 error:nil];
-  NSArray* matches = [regex matchesInString:model options:0 range:NSMakeRange(0, [model length])];
-
-  for (NSTextCheckingResult* match in matches) {
-    return [[BenchmarkModel alloc] init:[model substringWithRange:[match rangeAtIndex:1]]
-            backend:[model substringWithRange:[match rangeAtIndex:2]]
-            quantization:[model substringWithRange:[match rangeAtIndex:3]]
-    ];
-  }
-
-  return [[BenchmarkModel alloc] init:model
-          backend:@""
-          quantization:@""
-  ];
-}
-
-@end
-
 @interface DeviceInfo : NSObject
 
-// The metric name, i.e. TPS
-@property NSString *device;
-
-// The phone model and Android release version
-@property NSString *arch;
-@property NSString *os;
-// TODO (huydhn): These are not used yet
-@property NSNumber *totalMem;
-@property NSNumber *availMem;
-
--(instancetype)init;
++(NSString *)getDeviceInfo;
 
 @end
 
 @implementation DeviceInfo
 
--(instancetype)init {
-  self.device = [[UIDevice currentDevice] model];
-  self.arch = [[UIDevice currentDevice] name];
-  self.os = [NSString stringWithFormat:@"%@ / %@", [[UIDevice currentDevice] systemName], [[UIDevice currentDevice] systemVersion]];
-  self.totalMem = [[NSNumber alloc] initWithInt:0];
-  self.availMem = [[NSNumber alloc] initWithInt:0];
-  return self;
-}
-
-@end
-
-@interface BenchmarkMetric : NSObject
-
-@property BenchmarkModel *benchmarkModel;
-
-// The metric name, i.e. TPS
-@property NSString *metric;
-
-// The actual value and the option target value
-@property NSNumber *actualValue;
-@property NSNumber *targetValue;
-
-@property DeviceInfo *deviceInfo;
-
--(instancetype)init: (BenchmarkModel*)benchmarkModel
-             metric: (NSString*)metric
-        actualValue: (NSNumber*)actualValue
-        targetValue: (NSNumber*)targetValue
-         deviceInfo: (DeviceInfo*)deviceInfo;
-
-@end
-
-@implementation BenchmarkMetric
-
--(instancetype)init: (BenchmarkModel*)benchmarkModel
-             metric: (NSString*)metric
-        actualValue: (NSNumber*)actualValue
-        targetValue: (NSNumber*)targetValue
-         deviceInfo: (DeviceInfo*)deviceInfo {
-    
-  self.benchmarkModel = benchmarkModel;
-  self.metric = metric;
-  self.actualValue = actualValue;
-  self.targetValue = targetValue;
-  self.deviceInfo = deviceInfo;
-  return self;
++(NSString *)getDeviceInfo {
+  NSString *device = [[UIDevice currentDevice] model];
+  NSString *name = [[UIDevice currentDevice] name];
+  NSString *systemName = [[UIDevice currentDevice] systemName];
+  NSString *systemVersion = [[UIDevice currentDevice] systemVersion];
+  return [NSString stringWithFormat:@"%@_%@_%@_%@", device, name, systemName, systemVersion];
 }
 
 @end
@@ -141,9 +44,8 @@ + (void)initialize {
   if (self != [self class]) {
     return;
   }
-  DeviceInfo *deviceInfo = [[DeviceInfo alloc] init];
-  NSMutableArray *benchmarkMetrics = [[NSMutableArray alloc] init];
-  
+  NSString *deviceInfo = [DeviceInfo getDeviceInfo];
+
   for (NSBundle *bundle in @[
          [NSBundle mainBundle],
          [NSBundle bundleForClass:[self class]],
@@ -169,10 +71,9 @@ + (void)initialize {
                 .lowercaseString;
         NSString *modelName =
             modelPath.lastPathComponent.stringByDeletingPathExtension;
-        BenchmarkModel *benchmarkModel = [BenchmarkModel initWithModelName:modelName];
 
         SEL testLoadSelector = NSSelectorFromString([NSString
-            stringWithFormat:@"test_load_%@_%@", directoryName, modelName]);
+            stringWithFormat:@"test_load_%@_%@_%@", directoryName, modelName, deviceInfo]);
         IMP testLoadImplementation = imp_implementationWithBlock(^(id _self) {
           auto __block module = std::make_unique<Module>(modelPath.UTF8String);
           [_self measureWithMetrics:@[
@@ -181,14 +82,8 @@ + (void)initialize {
           ]
                             options:XCTMeasureOptions.defaultOptions
                               block:^{
-                                auto status = module->load_method("forward");
-                                BenchmarkMetric *benchmarkMetric = [[BenchmarkMetric alloc] init:benchmarkModel 
-                                                                                          metric:@"load_status"
-                                                                                     actualValue:[[NSNumber alloc] initWithInt:module->is_loaded() ? 0 : 1]
-                                                                                     targetValue:0
-                                                                                      deviceInfo:deviceInfo];
-                                [benchmarkMetrics addObject:benchmarkMetric];
-                                XCTAssertEqual(status, Error::Ok);
+                                XCTAssertEqual(module->load_method("forward"),
+                                               Error::Ok);
                               }];
         });
         class_addMethod(