added test environment

Author: andreadesim

.gitignore

@@ -5,3 +5,6 @@ __pycache__/
 .idea/
 
 path_bpatch.py
+
+/test/*
+!/test/*.py

bpatch.py

@@ -642,7 +642,7 @@ def generate_hex_fw(filename, fw_path, fw_header_path, header_fw_size):
             # check if the patched firmware is equal to the new firmware
             if os.system(f"diff patched_fw.tmp new_fw.tmp"):
                 print("The txt patch is not correct")
-                exit()
+                exit(2)
             else:
                 print("The txt patch is correct")
 
@@ -653,7 +653,7 @@ def generate_hex_fw(filename, fw_path, fw_header_path, header_fw_size):
             # check if the patched firmware is equal to the new firmware
             if os.system(f"diff {path_patch_txt} patch_txt_new.tmp"):
                 print("The bin patch is not correct")
-                exit()
+                exit(3)
             else:
                 print("The bin patch is correct")
 
@@ -665,13 +665,16 @@ def generate_hex_fw(filename, fw_path, fw_header_path, header_fw_size):
             # check if the patched firmware is equal to the new firmware
             if os.system(f"diff patched_fw.tmp {new_fw}"):
                 print("The patch is not correct")
+                exit(1)
             else:
                 print("The patch is correct")
 
         # remove temporary files
         for tmp in to_remove:
             os.remove(tmp)
 
+        exit(0)
+
     # decode function
     elif sys.argv[1] == 'decode':
         # check the number of arguments
@@ -708,7 +711,7 @@ def generate_hex_fw(filename, fw_path, fw_header_path, header_fw_size):
             print("Error during parsing arguments")
 
         # apply the patch
-        os.system(f"{path_patch_exec} {old_fw} {p} {new_fw} {read_buffer} {patch_buffer} {verbose}")
+        exit(os.system(f"{path_patch_exec} {old_fw} {p} {new_fw} {read_buffer} {patch_buffer} {verbose}"))
 
     # help function
     elif sys.argv[1] == 'help':

test/plot.py

@@ -0,0 +1,87 @@
+import matplotlib.pyplot as plt
+import numpy as np
+from math import ceil
+import os
+
+from test import extract_last_decimal
+
+for d in os.listdir():
+    if d.startswith('f'):
+        r = {}
+
+        fw_size = np.array([])
+        patch_size = np.array([])
+        n_frag_patch = np.array([])
+        n_frag_fw = np.array([])
+        ovh_nbd = np.array([])
+        ovh_nbc = np.array([])
+        ovh_nba = np.array([])
+        ovh_tot = np.array([])
+        n_nba = np.array([])
+
+        if os.path.exists(d + '/testout'):
+            for report in os.listdir(d + '/testout'):
+                if report.startswith('report'):
+                    # extract version
+                    version = extract_last_decimal(report)
+                    # add to dictionary
+                    r[version] = report
+
+        # check if there are report in the folder
+        if r:
+            fw_versions = sorted(r.keys())
+            for v in fw_versions:
+                with open(d + '/testout/' + r[v], 'r') as f:
+                    f.readline()
+                    fields = f.readline().split(',')
+
+                    fw_size         = np.append(fw_size, int(fields[0]))
+                    patch_size      = np.append(patch_size, int(fields[1]))
+                    n_frag_patch    = np.append(n_frag_patch, ceil(int(fields[1])/112))
+                    n_frag_fw       = np.append(n_frag_fw, ceil(int(fields[0])/112))
+                    n_nba           = np.append(n_nba, int(fields[4]))
+                    ovh_nbd         = np.append(ovh_nbd, float(fields[5]))
+                    ovh_nbc         = np.append(ovh_nbc, float(fields[6]))
+                    ovh_nba         = np.append(ovh_nba, float(fields[7]))
+                    ovh_tot         = np.append(ovh_tot, float(fields[8]))
+
+            # fig, axs = plt.subplots(2, 2, figsize=(15, 10))
+            fig, axs = plt.subplots(2, 2)
+            fig.suptitle(f'Statistics folder: {d}')
+
+            axs[0, 0].set_title('Compression')
+            axs[0, 0].plot(fw_versions, patch_size / fw_size * 100)
+            axs[0, 0].set_xlabel('FW version')
+            axs[0, 0].set_ylabel('Patch compression [%]')
+            axs[0, 0].grid()
+
+            axs[0, 1].set_title('New Bytes')
+            axs[0, 1].plot(fw_versions, np.ceil(n_nba / 1024))
+            axs[0, 1].set_xlabel('FW version')
+            axs[0, 1].set_ylabel('kB')
+            axs[0, 1].grid()
+
+            axs[1, 0].set_title('Overhead')
+            axs[1, 0].plot(fw_versions, ovh_nbd, fw_versions, ovh_nbc, fw_versions, ovh_nba, fw_versions, ovh_tot)
+            axs[1, 0].set_xlabel('FW version')
+            axs[1, 0].set_ylabel('Overhead respect new bytes [%]')
+            axs[1, 0].legend(['nbd', 'nbc', 'nba', 'total'])
+            axs[1, 0].grid()
+
+            width = 0.3  # the width of the bars
+            species = np.arange(len(fw_versions))
+
+            p1 = axs[1, 1].bar(species - width / 2, n_frag_fw, width, label='FW')
+            axs[1, 1].bar_label(p1, label_type='edge')
+            p2 = axs[1, 1].bar(species + width / 2, n_frag_patch, width, label='Patch')
+            axs[1, 1].bar_label(p2, label_type='edge')
+
+            axs[1, 1].set_title('Number of fragment comparison')
+            axs[1, 1].set_xticks(species)
+            axs[1, 1].set_xticklabels(fw_versions)
+            axs[1, 1].legend()
+
+            plt.tight_layout()
+            plt.show()
+
+exit()

test/test.py

@@ -0,0 +1,52 @@
+import os
+import subprocess
+import re
+import sys
+
+def extract_last_decimal(s):
+    # find all numbers in the string
+    numbers = re.findall(r'\d+', s)
+    # restur last number if exists
+    return numbers[-1]if numbers else None
+
+def test_bpatch():
+    error_count = 0
+
+    for d in os.listdir():
+        if d.startswith('f'):
+            print(f'### Start test folder {d} ###')
+            # create output directory if not exists
+            if not os.path.exists(d + '/testout'):
+                os.makedirs(d + '/testout')
+            # create firmware dict with fw name and version
+            f = {}
+            for el in os.listdir(d):
+                # check if the file is a binary file
+                if el.endswith('.bin'):
+                    version = extract_last_decimal(el)
+                    # check if the version is not None and not in the dict
+                    if version is None:
+                        print(f'Error: cannot find version in {el}', file=sys.stderr)
+                        exit(-1)
+                    if version in f:
+                        print(f'Error: duplicate version {version} in {el}', file=sys.stderr)
+                        exit(-1)
+                    # add fw in dict
+                    f[version] = el
+
+            for version in sorted(f.keys())[:-1]:
+                print(f'Start test bpatch for {d}/{f[version]} and {d}/{f[str(int(version) + 1)]}')
+                # run bpatch
+                if os.system(f'python3 ../bpatch.py encode {d}/{f[version]} {d}/{f[str(int(version) + 1)]} {d}/testout/patch_{int(version) + 1}.bin -v -R {d}/testout/report_{int(version) + 1}.csv> /dev/null') != 0:
+                    print(f'Error while running bpatch for {d}/{f[version]} and {d}/{f[str(int(version) + 1)]}', file=sys.stderr)
+                    error_count += 1
+                else:
+                    print('Test OK')
+
+            print(f'### End test folder {d} ###\n\n')
+
+    return error_count
+
+
+if __name__ == '__main__':
+    exit(test_bpatch())