APK main feature into DB

For repackage detection.

Author: pkumza

LibRadar/_settings.py

@@ -40,6 +40,7 @@
 DB_HOST = 'localhost'
 DB_PORT = 6379
 DB_ID = 2
+DB_ID_REP = 1
 # if you don't have Password, delete DB_PSWD
 DB_PSWD = ''
 

LibRadar/dex_tree.py

@@ -20,12 +20,13 @@
 #   This script is used to implement the tree node and tree structure.
 
 
-
-
 from _settings import *
+from collections import Counter
 import hashlib
 import csv
 import redis
+import zlib
+import rputil
 
 
 # tag_rules
@@ -62,23 +63,27 @@ def __init__(self, n_weight=-1, n_pn="", n_parent=None):
         self.match = list()
         self.permissions = set()
         self.db = redis.StrictRedis(host=DB_HOST, port=DB_PORT, db=DB_ID, password=DB_PSWD)
+        self.api_id_list = []
 
-    def insert(self, package_name, weight, sha256, permission_list):
+    def insert(self, package_name, weight, sha256, permission_list, api_id_list):
         # no matter how deep the package is, add permissions here.
         for permission in permission_list:
             self.permissions.add(permission)
+        # no matter how deep the package is, add api_id_list
+        # self.api_id_list = self.api_id_list + api_id_list
         current_depth = 0 if self.pn == "" else self.pn.count('/') + 1
         target_depth = package_name.count('/') + 1
         if current_depth == target_depth:
             self.sha256 = sha256
+            self.api_id_list = api_id_list
             return "F: %s" % package_name
         target_package_name = '/'.join(package_name.split('/')[:current_depth + 1])
         if target_package_name in self.children:
             self.children[target_package_name].weight += weight
-            return self.children[target_package_name].insert(package_name, weight, sha256, permission_list)
+            return self.children[target_package_name].insert(package_name, weight, sha256, permission_list, api_id_list)
         else:
             self.children[target_package_name] = TreeNode(n_weight=weight, n_pn=target_package_name, n_parent=self)
-            return self.children[target_package_name].insert(package_name, weight, sha256, permission_list)
+            return self.children[target_package_name].insert(package_name, weight, sha256, permission_list, api_id_list)
 
     def brand(self, package_name, standard_package):
         current_depth = 0 if self.pn == "" else self.pn.count('/') + 1
@@ -108,22 +113,15 @@ class Tree(object):
     """
     Tree
     """
-    def __init__(self, lite=True):
-        self.lite = lite
+    def __init__(self):
         self.root = TreeNode()
         self.db = None
         self.feature = None
-        if not self.lite:
-            self.db = redis.StrictRedis(host=DB_HOST, port=DB_PORT, db=DB_ID, password=DB_PSWD)
-        else:
-            self.feature = dict()
-            with open(LITE_DATASET_10, 'r') as file_rules:
-                csv_rules_reader = csv.reader(file_rules, delimiter=',', quotechar='|')
-                for row in csv_rules_reader:
-                    self.feature[row[0]] = row[1:5]
+        self.db = redis.StrictRedis(host=DB_HOST, port=DB_PORT, db=DB_ID, password=DB_PSWD)
+        self.db_rep = redis.StrictRedis(host=DB_HOST, port=DB_PORT, db=DB_ID_REP, password=DB_PSWD)
 
-    def insert(self, package_name, weight, sha256, permission_list):
-        self.root.insert(package_name, weight, sha256, permission_list)
+    def insert(self, package_name, weight, sha256, permission_list, api_id_list):
+        self.root.insert(package_name, weight, sha256, permission_list, api_id_list)
 
     def brand(self, package_name, standard_package):
         return self.root.brand(package_name, standard_package)
@@ -139,6 +137,17 @@ def _pre_order_res(self, node, visit, res):
             for child_pn in node.children:
                 self._pre_order_res(node.children[child_pn], visit, res)
 
+    def pre_order_res_ret(self, visit, res, ret):
+        self._pre_order_res_ret(node=self.root, visit=visit, res=res, ret=ret)
+
+    def _pre_order_res_ret(self, node, visit, res, ret):
+        retu = visit(node, res, ret)
+        if retu < 0:
+            return
+        else:
+            for child_pn in node.children:
+                self._pre_order_res_ret(node.children[child_pn], visit, res, ret)
+
     def pre_order(self, visit):
         self._pre_order(self.root, visit)
 
@@ -183,17 +192,12 @@ def cal_sha256(self):
 
     def _match(self, node):
         a, c, u = None, None, None
-        if not self.lite:
-            pipe = self.db.pipeline()
-            pipe.hget(name=DB_UN_OB_PN, key=node.sha256)
-            pipe.hget(name=DB_FEATURE_CNT, key=node.sha256)
-            pipe.hget(name=DB_UN_OB_CNT, key=node.sha256)
-            pipe_res = pipe.execute()
-            a, c, u = pipe_res
-        else:
-            if node.sha256 in self.feature:
-                acu_cur = self.feature[node.sha256]
-                a, c, u = acu_cur[3], acu_cur[0], acu_cur[2]
+        pipe = self.db.pipeline()
+        pipe.hget(name=DB_UN_OB_PN, key=node.sha256)
+        pipe.hget(name=DB_FEATURE_CNT, key=node.sha256)
+        pipe.hget(name=DB_UN_OB_CNT, key=node.sha256)
+        pipe_res = pipe.execute()
+        a, c, u = pipe_res
 
         # if could not find this package in database, search its children.
         if a is None:
@@ -308,17 +312,13 @@ def _find_untagged(self, node, res):
         a, c, u = None, None, None
         if len(node.match) != 0:
             return -1
-        if not self.lite:
-            pipe = self.db.pipeline()
-            pipe.hget(name=DB_UN_OB_PN, key=node.sha256)
-            pipe.hget(name=DB_FEATURE_CNT, key=node.sha256)
-            pipe.hget(name=DB_UN_OB_CNT, key=node.sha256)
-            pipe_res = pipe.execute()
-            a, c, u = pipe_res
-        else:
-            if node.sha256 in self.feature:
-                acu_cur = self.feature[node.sha256]
-                a, c, u = acu_cur[3], acu_cur[0], acu_cur[2]
+        pipe = self.db.pipeline()
+        pipe.hget(name=DB_UN_OB_PN, key=node.sha256)
+        pipe.hget(name=DB_FEATURE_CNT, key=node.sha256)
+        pipe.hget(name=DB_UN_OB_CNT, key=node.sha256)
+        pipe_res = pipe.execute()
+        a, c, u = pipe_res
+
 
         if a is None:
             return 1
@@ -387,3 +387,25 @@ def _get_lib(node, res):
     def get_lib(self, res):
         self.pre_order_res(visit=self._get_lib, res=res)
 
+    @staticmethod
+    def _get_repackage_main(node, res, ret):
+        if node.pn in res:
+            return -1
+        if len(node.children) == 0:
+            ret.extend(node.api_id_list)
+            ret += node.api_id_list
+        return 0
+
+    def get_repackage_main(self, res, hex_sha256):
+        # res is a list of libraries. Result.
+        pn_list = list()
+        for item in res:
+            pn_list.append(item["Package"])
+        ret = list()
+        self.pre_order_res_ret(visit=self._get_repackage_main, res=pn_list, ret=ret)
+        ret_length = len(ret)
+        kvd = dict(Counter(ret))
+        str = rputil.Util.dict2str(kvd)
+        zstr = zlib.compress(str,1)
+        self.db_rep.hset(name="apk_feature", key=hex_sha256, value=zstr)
+        self.db_rep.zadd("apk_weight", ret_length, hex_sha256 )

LibRadar/libradar.py

@@ -27,20 +27,21 @@
 import hashlib
 import zipfile
 import json
+from collections import Counter
 
 class LibRadar(object):
     """
     LibRadar
     """
-    def __init__(self, apk_path, lite=True):
+    def __init__(self, apk_path):
         """
         Init LibRadar instance with apk_path as a basestring.
         Create a Tree for every LibRadar instance. The tree describe the architecture of the apk. Every package is a
         node.
         :param apk_path: basestring
         """
         self.apk_path = apk_path
-        self.tree = dex_tree.Tree(lite=lite)
+        self.tree = dex_tree.Tree()
         self.dex_name = ""
         # Instance of Dex Object in dex_parser.
         self.dex = None
@@ -50,6 +51,7 @@ def __init__(self, apk_path, lite=True):
         """
         self.k_api_v_permission = dict()
         with open(SCRIPT_PATH + "/Data/IntermediateData/strict_api.csv", 'r') as api_and_permission:
+            api_id = 0
             for line in api_and_permission:
                 api, permission_with_colon = line.split(",")
                 permissions = permission_with_colon[:-2].split(":")
@@ -58,7 +60,8 @@ def __init__(self, apk_path, lite=True):
                 for permission in permissions:
                     if permission != "":
                         permission_list.append(permission)
-                self.k_api_v_permission[api] = permission_list
+                self.k_api_v_permission[api] = (permission_list, api_id)
+                api_id += 1
         """
         invoke_file = open(SCRIPT_PATH +"/Data/IntermediateData/invokeFormat.txt", 'r')
         self.invokes = set()
@@ -125,7 +128,7 @@ def get_api_list(self, dex_method, api_list, permission_list):
             if 0x6e <= op_code <= 0x72:
                 if decoded_instruction.getApi in self.k_api_v_permission:
                     api_list.append(decoded_instruction.getApi)
-                    for permission in self.k_api_v_permission[decoded_instruction.getApi]:
+                    for permission in self.k_api_v_permission[decoded_instruction.getApi][0]:
                         permission_list.add(permission)
         return
 
@@ -154,7 +157,12 @@ def extract_class(self, dex_class_def_obj):
             class_sha256.update(api)
         if not IGNORE_ZERO_API_FILES or len(api_list) != 0:
             pass
-        return len(api_list), class_sha256.hexdigest(), class_sha256.hexdigest(), sorted(list(permission_list))
+        # api_id_list
+        api_id_list = []
+        for api in api_list:
+            api_id_list.append(self.k_api_v_permission[api][1])
+        return len(api_list), class_sha256.hexdigest(), class_sha256.hexdigest(), sorted(list(permission_list)),\
+               api_id_list
 
     def extract_dex(self):
         # Log Start
@@ -166,7 +174,8 @@ def extract_dex(self):
         # Create a Dex object
         self.dex = dex_parser.DexFile(self.dex_name)
         for dex_class_def_obj in self.dex.dexClassDefList:
-            weight, raw_sha256, hex_sha256, permission_list = self.extract_class(dex_class_def_obj=dex_class_def_obj)
+            weight, raw_sha256, hex_sha256, permission_list, api_id_list = \
+                self.extract_class(dex_class_def_obj=dex_class_def_obj)
             class_name = self.dex.getDexTypeId(dex_class_def_obj.classIdx)
             """
             I got many \x01 here before the class name.
@@ -180,7 +189,8 @@ def extract_dex(self):
                 class_name = class_name[l_index:]
             if IGNORE_ZERO_API_FILES and weight == 0:
                 continue
-            self.tree.insert(package_name=class_name, weight=weight, sha256=raw_sha256, permission_list=permission_list)
+            self.tree.insert(package_name=class_name, weight=weight, sha256=raw_sha256,
+                             permission_list=permission_list, api_id_list=api_id_list)
         return 0
 
     def analyse(self):
@@ -206,17 +216,18 @@ def compare(self):
         self.tree.get_lib(res)
         # Step 6: traverse the tree, find potential libraries that has not been tagged.
         self.tree.find_untagged(res)
+        # Step 7: repackage feature store.
+        self.tree.get_repackage_main(res, self.hex_sha256)
         return res
 
 
-
 if __name__ == '__main__':
     if len(sys.argv) != 2:
         print("LibRadar only takes 1 arguments.")
         print("Usage:")
         print("    $ python libradar.py example.apk")
         exit(1)
     apk_path = sys.argv[1]
-    lrd = LibRadar(apk_path, lite=True)
+    lrd = LibRadar(apk_path)
     res = lrd.compare()
     print(json.dumps(res, indent=4, sort_keys=True))

LibRadar/util.py LibRadar/rputil.pyLibRadar/util.py renamed to LibRadar/rputil.py

@@ -19,6 +19,21 @@
 from _settings import *
 
 class Util(object):
+    """
+    Format of String Storage:
+
+    {1:2}
+    |--------|--------|--------|---------|
+    |00000000|00000001|00000000|000000010|
+    |--------|--------|--------|---------|
+
+
+    {2:3,4:128}
+    |--------|--------|--------|---------|--------|--------|--------|---------|
+    |00000000|00000010|00000000|000000011|00000000|00000100|00000000|100000000|
+    |--------|--------|--------|---------|--------|--------|--------|---------|
+
+    """
     @staticmethod
     def dict2str(kvd):
         """
@@ -70,3 +85,49 @@ def str2dict(feature_str):
             value_int = i3 * 256 + i4
             kvd[key_int] = value_int
         return kvd
+
+    @staticmethod
+    def get_key(feature_str, offset):
+        """
+
+        :param feature_str:
+        :param offset:
+        :return:
+        """
+        return ord(feature_str[offset]) * 256 + ord(feature_str[offset + 1])
+
+    @staticmethod
+    def get_value(feature_str, offset):
+        return ord(feature_str[offset + 2]) * 256 + ord(feature_str[offset + 3])
+
+    @staticmethod
+    def comp_str(str1, str2):
+        assert len(str1) % 4 == 0 and len(str2) % 4 == 0, "Feature_str length is not 4X"
+        feature_length1, feature_length2 = len(str1), len(str2)
+        cur1, cur2 = 0, 0
+        diff = 0
+        sum1, sum2 = 0, 0
+        while cur1 < feature_length1 or cur2 < feature_length2:
+            if cur1 == feature_length1 or Util.get_key(str2, cur2) < Util.get_key(str1, cur1):
+                v2 = Util.get_value(str2, cur2)
+                sum2 += v2
+                diff += v2
+                cur2 += 4
+                continue
+            if cur2 == feature_length2 or Util.get_key(str2, cur2) > Util.get_key(str1, cur1):
+                v1 = Util.get_value(str1, cur1)
+                sum1 += v1
+                diff += v1
+                cur1 += 4
+                continue
+            if Util.get_key(str2, cur2) == Util.get_key(str1, cur1):
+                v1 = Util.get_value(str1, cur1)
+                v2 = Util.get_value(str2, cur2)
+                diff += abs(v1 - v2)
+                sum1 += v1
+                sum2 += v2
+                cur1 += 4
+                cur2 += 4
+                continue
+            assert False, "Not reachable."
+        return float(diff) / (sum1 + sum2)