Skip to content

Commit 98e2ea9

Browse files
zhou9584zhou9584
committed
Merge branch 'main' into zhoule/analysisSmartTestResult
2 parents 268d1e5 + 4d24a70 commit 98e2ea9

File tree

16 files changed

+33853
-13
lines changed

16 files changed

+33853
-13
lines changed

‎.github/CODEOWNERS‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ ehcache.xml @zhou9584 @olivershen-wow
1212
*.md @zhou9584 @hydraxman @dexterdreeeam
1313

1414
# Hydra Lab Android client
15-
/android_client/ @lifesaver0129 @TedaLIEz @dexterdreeeam @taoran6
15+
/android_client/app/ @lifesaver0129 @TedaLIEz @dexterdreeeam @taoran6
1616

1717
# TestTaskSpec
1818
/common/src/main/java/com/microsoft/hydralab/common/entity/ @olivershen-wow @zhou9584 @dexterdreeeam

‎agent/src/main/java/com/microsoft/hydralab/agent/runner/scanner/ApkCanaryExecutor.java‎

Lines changed: 267 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,267 @@
1+
package com.microsoft.hydralab.agent.runner.scanner;
2+
3+
import com.alibaba.fastjson.JSON;
4+
import com.alibaba.fastjson.JSONArray;
5+
import com.alibaba.fastjson.JSONObject;
6+
import com.microsoft.hydralab.common.entity.common.scanner.ApkManifest;
7+
import com.microsoft.hydralab.common.entity.common.scanner.ApkReport;
8+
import com.microsoft.hydralab.common.entity.common.scanner.ApkSizeReport;
9+
import com.microsoft.hydralab.common.util.FileUtil;
10+
import org.apache.commons.io.FileUtils;
11+
import org.apache.commons.io.IOUtils;
12+
import org.slf4j.Logger;
13+
import org.slf4j.LoggerFactory;
14+
15+
import java.io.BufferedReader;
16+
import java.io.File;
17+
import java.io.FileInputStream;
18+
import java.io.FileOutputStream;
19+
import java.io.IOException;
20+
import java.io.InputStream;
21+
import java.io.InputStreamReader;
22+
import java.nio.charset.Charset;
23+
import java.nio.charset.StandardCharsets;
24+
import java.util.ArrayList;
25+
import java.util.List;
26+
27+
/**
28+
*
29+
*/
30+
public class ApkCanaryExecutor {
31+
/**
32+
* https://github.com/Tencent/matrix/blob/master/matrix/matrix-android/matrix-apk-canary/src/main/java/com/tencent/matrix/apk/model/task/TaskFactory.java
33+
*/
34+
public static final int TASK_TYPE_UNZIP = 1;
35+
public static final int TASK_TYPE_MANIFEST = 2;
36+
public static final int TASK_TYPE_SHOW_FILE_SIZE = 3;
37+
public static final int TASK_TYPE_COUNT_METHOD = 4;
38+
public static final int TASK_TYPE_CHECK_RESGUARD = 5;
39+
public static final int TASK_TYPE_FIND_NON_ALPHA_PNG = 6;
40+
public static final int TASK_TYPE_CHECK_MULTILIB = 7;
41+
public static final int TASK_TYPE_UNCOMPRESSED_FILE = 8;
42+
public static final int TASK_TYPE_COUNT_R_CLASS = 9;
43+
public static final int TASK_TYPE_DUPLICATE_FILE = 10;
44+
public static final int TASK_TYPE_CHECK_MULTISTL = 11;
45+
public static final int TASK_TYPE_UNUSED_RESOURCES = 12;
46+
public static final int TASK_TYPE_UNUSED_ASSETS = 13;
47+
public static final int TASK_TYPE_UNSTRIPPED_SO = 14;
48+
public static final int TASK_TYPE_COUNT_CLASS = 15;
49+
public static final int DUP_FILE_MAX_COUNT = 5;
50+
public static final int ASSETS_LIST_MAX_COUNT = 20;
51+
private static final Logger LOGGER = LoggerFactory.getLogger(ApkCanaryExecutor.class);
52+
static final Charset CHARSET = StandardCharsets.UTF_8;
53+
54+
public ApkReport analyzeApk(File canaryJar, File configTemplate, String apkPath, String workingDirPath) {
55+
File apk = new File(apkPath);
56+
if (!apk.exists()) {
57+
throw new RuntimeException("apk not exist");
58+
}
59+
File workingDirFile = new File(workingDirPath);
60+
if (!workingDirFile.exists()) {
61+
if (!workingDirFile.mkdirs()) {
62+
throw new RuntimeException("mkdir fail!");
63+
}
64+
}
65+
66+
int code = -1;
67+
68+
String name = apk.getName();
69+
String itemName = name.replace(".apk", "");
70+
String reportPrefix = itemName + "-report";
71+
String buildDirName = name.substring(0, name.indexOf('.')) + "_unzip";
72+
73+
File reportFile = new File(workingDirFile, reportPrefix + ".json");
74+
if (reportFile.exists()) {
75+
reportFile.delete();
76+
}
77+
Runtime runtime = Runtime.getRuntime();
78+
Process process = null;
79+
try {
80+
String error = "";
81+
FileInputStream input = new FileInputStream(configTemplate);
82+
String content = IOUtils.toString(input, CHARSET);
83+
input.close();
84+
85+
String newConfigFileName = itemName + "-config.json";
86+
File newConfigFile = new File(workingDirFile, newConfigFileName);
87+
if (newConfigFile.exists()) {
88+
newConfigFile.delete();
89+
}
90+
91+
FileOutputStream fileOutputStream = new FileOutputStream(newConfigFile);
92+
IOUtils.write(
93+
content.replace("@NAME_HOLDER", apk.getAbsolutePath().replace("\\", "\\\\"))
94+
.replace("@REPORT_NAME_HOLDER", new File(workingDirFile, reportPrefix).getAbsolutePath().replace("\\", "\\\\")),
95+
fileOutputStream, CHARSET);
96+
97+
fileOutputStream.close();
98+
99+
String command = String.format("java -jar %s --config %s", canaryJar.getAbsoluteFile(), newConfigFile.getAbsolutePath());
100+
LOGGER.info(command);
101+
process = runtime.exec(command, null, workingDirFile);
102+
103+
try (InputStream inputStream = process.getInputStream();
104+
InputStream errorStream = process.getErrorStream();
105+
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream))) {
106+
String line;
107+
while ((line = bufferedReader.readLine()) != null) {
108+
LOGGER.info(line);
109+
}
110+
code = process.waitFor();
111+
error = IOUtils.toString(errorStream, CHARSET);
112+
} finally {
113+
if (process != null) {
114+
process.destroy();
115+
}
116+
}
117+
FileUtils.deleteDirectory(new File(workingDirFile, buildDirName));
118+
119+
if (code != 0) {
120+
LOGGER.info(error);
121+
throw new RuntimeException("failed for " + name);
122+
}
123+
124+
return getApkReportFromJsonReport(reportFile);
125+
} catch (InterruptedException e) {
126+
LOGGER.error("Interrupted in analyzeApk", e);
127+
} catch (IOException e) {
128+
LOGGER.error("error in analyzeApk", e);
129+
throw new RuntimeException(e);
130+
}
131+
return null;
132+
}
133+
134+
public static ApkReport getApkReportFromJsonReport(File file) {
135+
String json = FileUtil.getStringFromFilePath(file.getAbsolutePath());
136+
JSONArray objects = JSON.parseArray(json);
137+
138+
ApkReport apkReport = new ApkReport("apkReport");
139+
140+
for (int i = 0; i < objects.size(); i++) {
141+
JSONObject jsonObject = objects.getJSONObject(i);
142+
int taskType = jsonObject.getIntValue("taskType");
143+
// Task type definition is in:
144+
// https://github.com/Tencent/matrix/blob/master/matrix/matrix-android/matrix-apk-canary/src/main/java/com/tencent/matrix/apk/model/task/TaskFactory.java
145+
// Sample json search MicrosoftLauncherAPKReport.json
146+
if (taskType == TASK_TYPE_UNZIP) {
147+
// Size distribution among each format
148+
parseSizeInfo(apkReport.getApkSizeReport(), jsonObject);
149+
} else if (taskType == TASK_TYPE_MANIFEST) {
150+
// APK manifest information
151+
parseManifest(apkReport.getApkManifest(), jsonObject);
152+
} else if (taskType == TASK_TYPE_DUPLICATE_FILE) {
153+
// Find out the duplicated files
154+
parseDupFile(apkReport.getApkSizeReport(), jsonObject);
155+
} else if (taskType == TASK_TYPE_UNUSED_ASSETS) {
156+
// Find out the unused assets
157+
parseUnusedAssets(apkReport.getApkSizeReport(), jsonObject);
158+
} else if (taskType == TASK_TYPE_SHOW_FILE_SIZE) {
159+
// Show files whose size exceed limit size in order
160+
parseBigFiles(apkReport.getApkSizeReport(), jsonObject);
161+
}
162+
}
163+
return apkReport;
164+
}
165+
166+
private static void parseBigFiles(ApkSizeReport apkSizeReport, JSONObject jsonObject) {
167+
JSONArray bigFiles = jsonObject.getJSONArray("files");
168+
if (bigFiles != null) {
169+
for (int n = 0; n < bigFiles.size(); n++) {
170+
JSONObject fileObject = bigFiles.getJSONObject(n);
171+
ApkSizeReport.FileItem fileItem = new ApkSizeReport.FileItem();
172+
fileItem.fileName = fileObject.getString("entry-name");
173+
if (fileItem.fileName != null && fileItem.fileName.equals("resources.arsc")) {
174+
continue;
175+
}
176+
fileItem.size = fileObject.getLongValue("entry-size");
177+
apkSizeReport.bigSizeFileList.add(fileItem);
178+
}
179+
}
180+
}
181+
182+
private static void parseUnusedAssets(ApkSizeReport apkSizeReport, JSONObject jsonObject) {
183+
String unusedStr = jsonObject.getString("unused-assets");
184+
apkSizeReport.unusedAssetsList.addAll(parseUnusedAssetsList(JSONArray.parseArray(unusedStr, String.class)));
185+
if (apkSizeReport.unusedAssetsList.size() > ASSETS_LIST_MAX_COUNT) {
186+
apkSizeReport.unusedAssetsList = apkSizeReport.unusedAssetsList.subList(0, ASSETS_LIST_MAX_COUNT);
187+
}
188+
}
189+
190+
private static void parseDupFile(ApkSizeReport apkSizeReport, JSONObject jsonObject) {
191+
JSONArray dupFiles = jsonObject.getJSONArray("files");
192+
if (dupFiles != null) {
193+
for (int n = 0; n < Math.min(dupFiles.size(), DUP_FILE_MAX_COUNT); n++) {
194+
ApkSizeReport.DuplicatedFile duplicatedFile = new ApkSizeReport.DuplicatedFile();
195+
JSONObject dupFileObject = dupFiles.getJSONObject(n);
196+
duplicatedFile.md5 = dupFileObject.getString("md5");
197+
duplicatedFile.size = dupFileObject.getLongValue("size");
198+
String fileString = dupFileObject.getString("files");
199+
duplicatedFile.fileList = JSONArray.parseArray(fileString, String.class);
200+
apkSizeReport.getDuplicatedFileList().add(duplicatedFile);
201+
}
202+
}
203+
}
204+
205+
private static void parseManifest(ApkManifest apkManifest, JSONObject jsonObject) {
206+
JSONObject manifest = jsonObject.getJSONObject("manifest");
207+
apkManifest.setPackageName(manifest.getString("package"));
208+
apkManifest.setMinSDKVersion(Integer.parseInt(manifest.getString("android:minSdkVersion")));
209+
apkManifest.setTargetSDKVersion(Integer.parseInt(manifest.getString("android:targetSdkVersion")));
210+
apkManifest.setVersionCode(Integer.parseInt(manifest.getString("android:versionCode")));
211+
apkManifest.setVersionName(manifest.getString("android:versionName"));
212+
}
213+
214+
private static void parseSizeInfo(ApkSizeReport apkSizeReport, JSONObject jsonObject) {
215+
long total = jsonObject.getLongValue("total-size");
216+
apkSizeReport.setTotalSize(total);
217+
apkSizeReport.setTotalSizeInMB(total * 1f / 1024 / 1024);
218+
219+
long otherSize = total;
220+
JSONArray entries = jsonObject.getJSONArray("entries");
221+
222+
for (int n = 0; n < entries.size(); n++) {
223+
JSONObject en = entries.getJSONObject(n);
224+
String suffix = en.getString("suffix");
225+
long entryTotalSize = en.getLongValue("total-size");
226+
switch (suffix) {
227+
case ".arsc":
228+
apkSizeReport.setArscSize(entryTotalSize);
229+
otherSize -= entryTotalSize;
230+
break;
231+
case ".dex":
232+
apkSizeReport.setDexSize(entryTotalSize);
233+
otherSize -= entryTotalSize;
234+
break;
235+
case ".png":
236+
apkSizeReport.setPngSize(entryTotalSize);
237+
otherSize -= entryTotalSize;
238+
break;
239+
case ".xml":
240+
apkSizeReport.setXmlSize(entryTotalSize);
241+
otherSize -= entryTotalSize;
242+
break;
243+
case ".so":
244+
apkSizeReport.setSoSize(entryTotalSize);
245+
otherSize -= entryTotalSize;
246+
break;
247+
case ".webp":
248+
apkSizeReport.setWebpSize(entryTotalSize);
249+
otherSize -= entryTotalSize;
250+
break;
251+
default:
252+
break;
253+
}
254+
}
255+
apkSizeReport.setOtherSize(otherSize);
256+
}
257+
258+
public static List<ApkSizeReport.FileItem> parseUnusedAssetsList(List<String> unusedFileNameList) {
259+
List<ApkSizeReport.FileItem> unusedAssetsList = new ArrayList<>();
260+
for (String fileName : unusedFileNameList) {
261+
ApkSizeReport.FileItem fileItem = new ApkSizeReport.FileItem();
262+
unusedAssetsList.add(fileItem);
263+
fileItem.fileName = "assets/" + fileName;
264+
}
265+
return unusedAssetsList;
266+
}
267+
}

‎agent/src/main/resources/apk_canary/apk_canary_config_template.json‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
{
2+
"--apk": "@NAME_HOLDER",
3+
"--format": "mm.html,mm.json",
4+
"--output": "@REPORT_NAME_HOLDER",
5+
"options": [
6+
{
7+
"name": "-manifest"
8+
},
9+
{
10+
"name": "-fileSize",
11+
"--min": "10",
12+
"--order": "desc",
13+
"--suffix": "png, jpg, webp, jpeg, gif, arsc"
14+
},
15+
{
16+
"name": "-countMethod",
17+
"--group": "package"
18+
},
19+
{
20+
"name": "-checkResProguard"
21+
},
22+
{
23+
"name": "-findNonAlphaPng",
24+
"--min": "10"
25+
},
26+
{
27+
"name": "-checkMultiLibrary"
28+
},
29+
{
30+
"name": "-uncompressedFile",
31+
"--suffix": "png, jpg, jpeg, gif, arsc"
32+
},
33+
{
34+
"name": "-countR"
35+
},
36+
{
37+
"name": "-duplicatedFile"
38+
},
39+
{
40+
"name": "-unusedAssets"
41+
}
42+
]
43+
}

‎agent/src/main/resources/apk_canary/matrix-apk-canary-2.1.0.jar‎

43.6 MB
Binary file not shown.

‎agent/src/test/java/com/microsoft/hydralab/agent/runner/scanner/ApkCanaryExecutorTest.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
package com.microsoft.hydralab.agent.runner.scanner;
2+
3+
import com.alibaba.fastjson.JSON;
4+
import com.alibaba.fastjson.serializer.SerializerFeature;
5+
import com.microsoft.hydralab.common.entity.common.scanner.ApkReport;
6+
import org.junit.jupiter.api.Assertions;
7+
import org.junit.jupiter.api.Test;
8+
9+
import java.io.File;
10+
11+
public class ApkCanaryExecutorTest {
12+
13+
@Test
14+
public void testApkCanaryExecutor() {
15+
File apkFilePath = new File("../common/src/test/resources/record_release.apk");
16+
Assertions.assertTrue(apkFilePath.exists(), "apk file not exist: " + apkFilePath.getAbsolutePath());
17+
ApkCanaryExecutor apkCanaryExecutor = new ApkCanaryExecutor();
18+
ApkReport apkReport = apkCanaryExecutor.analyzeApk(
19+
new File("src/main/resources/apk_canary/matrix-apk-canary-2.1.0.jar"),
20+
new File("src/main/resources/apk_canary/apk_canary_config_template.json"),
21+
apkFilePath.getAbsolutePath(),
22+
new File("").getAbsolutePath()
23+
);
24+
System.out.println(JSON.toJSONString(apkReport, SerializerFeature.PrettyFormat));
25+
}
26+
27+
@Test
28+
public void testApkReportParsing() {
29+
File apkJsonReportFile = new File("src/test/resources/MicrosoftLauncherAPKReport.json");
30+
Assertions.assertTrue(apkJsonReportFile.exists(), "apkJsonReportFile does not exist: " + apkJsonReportFile.getAbsolutePath());
31+
ApkReport apkReport = ApkCanaryExecutor.getApkReportFromJsonReport(apkJsonReportFile);
32+
System.out.println(JSON.toJSONString(apkReport, SerializerFeature.PrettyFormat));
33+
}
34+
35+
}

0 commit comments

Comments
 (0)