From 93bdc9b38eaffb14011773cea5b4a6e3386be66d Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 18 Sep 2024 11:26:31 -0700 Subject: [PATCH] Add DeviceInfo in iOS benchmark run (#5410) Summary: Given the way iOS benchmark app measures model load time, inference time, and memory usage using `measureWithMetrics` with `XCTClockMetric` and `XCTMemoryMetric`. I think the easiest way to gather the benchmark metric is to do it after the test finishes and parse the output. In the same spirit as https://github.com/pytorch/executorch/pull/5332, this PR adds more information about the device so that it can be parsed later. I add the information into the test name, shoumikhin plz let me know if you know a better way to pass this information around. The output looks like this with the information in the test case name, i.e. `test_forward_models_llama2_iPhone_iPhone14,2_iOS_17.6.1` ``` Test Case '-[Tests test_forward_models_llama2_iPhone_iPhone14,2_iOS_17.6.1]' started. /Users/huydo/Storage/mine/executorch/extension/apple/Benchmark/Tests/Tests.mm:134: Test Case '-[Tests test_forward_models_llama2_iPhone_iPhone14,2_iOS_17.6.1]' measured [Memory Peak Physical, kB] average: 125171.731, relative standard deviation: 0.010%, values: [125158.624000, 125175.008000, 125175.008000, 125158.624000, 125191.392000], performanceMetricID:com.apple.dt.XCTMetric_Memory.physical_peak, baselineName: "", baselineAverage: , polarity: prefers smaller, maxPercentRegression: 10.000%, maxPercentRelativeStandardDeviation: 10.000%, maxRegression: 0.000, maxStandardDeviation: 0.000 /Users/huydo/Storage/mine/executorch/extension/apple/Benchmark/Tests/Tests.mm:134: Test Case '-[Tests test_forward_models_llama2_iPhone_iPhone14,2_iOS_17.6.1]' measured [Memory Physical, kB] average: -29.491, relative standard deviation: -228.792%, values: [-49.152000, -16.384000, 16.384000, 49.152000, -147.456000], performanceMetricID:com.apple.dt.XCTMetric_Memory.physical, baselineName: "", baselineAverage: , polarity: prefers smaller, maxPercentRegression: 10.000%, maxPercentRelativeStandardDeviation: 10.000%, maxRegression: 0.000, maxStandardDeviation: 0.000 /Users/huydo/Storage/mine/executorch/extension/apple/Benchmark/Tests/Tests.mm:134: Test Case '-[Tests test_forward_models_llama2_iPhone_iPhone14,2_iOS_17.6.1]' measured [Clock Monotonic Time, s] average: 0.160, relative standard deviation: 3.460%, values: [0.163377, 0.165837, 0.164974, 0.152334, 0.154970], performanceMetricID:com.apple.dt.XCTMetric_Clock.time.monotonic, baselineName: "", baselineAverage: , polarity: prefers smaller, maxPercentRegression: 10.000%, maxPercentRelativeStandardDeviation: 10.000%, maxRegression: 0.000, maxStandardDeviation: 0.000 Test Case '-[Tests test_forward_models_llama2_iPhone_iPhone14,2_iOS_17.6.1]' passed (1.322 seconds). Test Case '-[Tests test_load_models_llama2_iPhone_iPhone14,2_iOS_17.6.1]' started. /Users/huydo/Storage/mine/executorch/extension/apple/Benchmark/Tests/Tests.mm:85: Test Case '-[Tests test_load_models_llama2_iPhone_iPhone14,2_iOS_17.6.1]' measured [Memory Peak Physical, kB] average: 127403.280, relative standard deviation: 0.000%, values: [127403.280000, 127403.280000, 127403.280000, 127403.280000, 127403.280000], performanceMetricID:com.apple.dt.XCTMetric_Memory.physical_peak, baselineName: "", baselineAverage: , polarity: prefers smaller, maxPercentRegression: 10.000%, maxPercentRelativeStandardDeviation: 10.000%, maxRegression: 0.000, maxStandardDeviation: 0.000 /Users/huydo/Storage/mine/executorch/extension/apple/Benchmark/Tests/Tests.mm:85: Test Case '-[Tests test_load_models_llama2_iPhone_iPhone14,2_iOS_17.6.1]' measured [Memory Physical, kB] average: 0.000, relative standard deviation: 0.000%, values: [0.000000, 0.000000, 0.000000, 0.000000, 0.000000], performanceMetricID:com.apple.dt.XCTMetric_Memory.physical, baselineName: "", baselineAverage: , polarity: prefers smaller, maxPercentRegression: 10.000%, maxPercentRelativeStandardDeviation: 10.000%, maxRegression: 0.000, maxStandardDeviation: 0.000 /Users/huydo/Storage/mine/executorch/extension/apple/Benchmark/Tests/Tests.mm:85: Test Case '-[Tests test_load_models_llama2_iPhone_iPhone14,2_iOS_17.6.1]' measured [Clock Monotonic Time, s] average: 0.000, relative standard deviation: 41.029%, values: [0.000001, 0.000001, 0.000001, 0.000001, 0.000001], performanceMetricID:com.apple.dt.XCTMetric_Clock.time.monotonic, baselineName: "", baselineAverage: , polarity: prefers smaller, maxPercentRegression: 10.000%, maxPercentRelativeStandardDeviation: 10.000%, maxRegression: 0.000, maxStandardDeviation: 0.000 Test Case '-[Tests test_load_models_llama2_iPhone_iPhone14,2_iOS_17.6.1]' passed (0.132 seconds). ``` Reviewed By: guangy10 Differential Revision: D62902327 Pulled By: huydhn --- .../Benchmark.xcodeproj/project.pbxproj | 4 +++ extension/apple/Benchmark/Tests/Tests.mm | 34 ++++++++++++++++--- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/extension/apple/Benchmark/Benchmark.xcodeproj/project.pbxproj b/extension/apple/Benchmark/Benchmark.xcodeproj/project.pbxproj index 41b2bd16a53..fb7146aa937 100644 --- a/extension/apple/Benchmark/Benchmark.xcodeproj/project.pbxproj +++ b/extension/apple/Benchmark/Benchmark.xcodeproj/project.pbxproj @@ -24,6 +24,7 @@ 03ED6D152C8AAFFF00F2D6EE /* Metal.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 03ED6D142C8AAFFF00F2D6EE /* Metal.framework */; }; 03ED6D172C8AB00500F2D6EE /* CoreML.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 03ED6D162C8AB00500F2D6EE /* CoreML.framework */; }; 03ED6D192C8AB00A00F2D6EE /* Accelerate.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 03ED6D182C8AB00A00F2D6EE /* Accelerate.framework */; }; + 8493389C2C9918950071ABAD /* UIKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 8493389B2C9918950071ABAD /* UIKit.framework */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -59,6 +60,7 @@ 03ED6D142C8AAFFF00F2D6EE /* Metal.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Metal.framework; path = Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS17.5.sdk/System/Library/Frameworks/Metal.framework; sourceTree = DEVELOPER_DIR; }; 03ED6D162C8AB00500F2D6EE /* CoreML.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CoreML.framework; path = Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS17.5.sdk/System/Library/Frameworks/CoreML.framework; sourceTree = DEVELOPER_DIR; }; 03ED6D182C8AB00A00F2D6EE /* Accelerate.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Accelerate.framework; path = Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS17.5.sdk/System/Library/Frameworks/Accelerate.framework; sourceTree = DEVELOPER_DIR; }; + 8493389B2C9918950071ABAD /* UIKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = UIKit.framework; path = Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS17.5.sdk/System/Library/Frameworks/UIKit.framework; sourceTree = DEVELOPER_DIR; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -77,6 +79,7 @@ 03ED6D172C8AB00500F2D6EE /* CoreML.framework in Frameworks */, 03ED6D152C8AAFFF00F2D6EE /* Metal.framework in Frameworks */, 03ED6D132C8AAFF700F2D6EE /* MetalPerformanceShaders.framework in Frameworks */, + 8493389C2C9918950071ABAD /* UIKit.framework in Frameworks */, 03ED6D112C8AAFF200F2D6EE /* MetalPerformanceShadersGraph.framework in Frameworks */, 03ED6D0F2C8AAFE900F2D6EE /* libsqlite3.0.tbd in Frameworks */, 03DD00A92C8FE44600FE4619 /* backend_coreml.xcframework in Frameworks */, @@ -135,6 +138,7 @@ 03ED6CEB2C8AAF5300F2D6EE /* Frameworks */ = { isa = PBXGroup; children = ( + 8493389B2C9918950071ABAD /* UIKit.framework */, 03ED6D182C8AB00A00F2D6EE /* Accelerate.framework */, 03ED6D162C8AB00500F2D6EE /* CoreML.framework */, 03ED6D142C8AAFFF00F2D6EE /* Metal.framework */, diff --git a/extension/apple/Benchmark/Tests/Tests.mm b/extension/apple/Benchmark/Tests/Tests.mm index 2730cd8963f..74388b49d54 100644 --- a/extension/apple/Benchmark/Tests/Tests.mm +++ b/extension/apple/Benchmark/Tests/Tests.mm @@ -8,7 +8,9 @@ #import +#import #import +#import #import #import @@ -17,14 +19,32 @@ using namespace ::executorch::runtime; @interface Tests : XCTestCase + @end @implementation Tests ++ (NSString *)getDeviceInfo { + NSString *device = [[UIDevice currentDevice] model]; + + // i.e. iPhone15 + struct utsname systemInfo; + uname(&systemInfo); + NSString *name = [NSString stringWithCString:systemInfo.machine + encoding:NSUTF8StringEncoding]; + + NSString *systemName = [[UIDevice currentDevice] systemName]; + NSString *systemVersion = [[UIDevice currentDevice] systemVersion]; + return [NSString + stringWithFormat:@"%@_%@_%@_%@", device, name, systemName, systemVersion]; +} + + (void)initialize { if (self != [self class]) { return; } + NSString *deviceInfo = [Tests getDeviceInfo]; + for (NSBundle *bundle in @[ [NSBundle mainBundle], [NSBundle bundleForClass:[self class]], @@ -51,8 +71,11 @@ + (void)initialize { NSString *modelName = modelPath.lastPathComponent.stringByDeletingPathExtension; - SEL testLoadSelector = NSSelectorFromString([NSString - stringWithFormat:@"test_load_%@_%@", directoryName, modelName]); + SEL testLoadSelector = NSSelectorFromString( + [NSString stringWithFormat:@"test_load_%@_%@_%@", + directoryName, + modelName, + deviceInfo]); IMP testLoadImplementation = imp_implementationWithBlock(^(id _self) { auto __block module = std::make_unique(modelPath.UTF8String); [_self measureWithMetrics:@[ @@ -68,8 +91,11 @@ + (void)initialize { class_addMethod( [self class], testLoadSelector, testLoadImplementation, "v@:"); - SEL testForwardSelector = NSSelectorFromString([NSString - stringWithFormat:@"test_forward_%@_%@", directoryName, modelName]); + SEL testForwardSelector = NSSelectorFromString( + [NSString stringWithFormat:@"test_forward_%@_%@_%@", + directoryName, + modelName, + deviceInfo]); IMP testForwardImplementation = imp_implementationWithBlock(^( id _self) { auto __block module = std::make_unique(modelPath.UTF8String);