Addressed review comments.

Signed-off-by: L Lakshmanan <s2760012@ed.ac.uk>

Author: KarthikL1729

.github/workflows/tools-tests.yaml

@@ -64,18 +64,19 @@ jobs:
 
       - name: Profile load check
         run: |
-          tar -xzvf tools/mapper/profile.tar.gz -C tools/mapper/
+          gzip -d tools/mapper/profile.json.gz -c > tools/mapper/profile.json
           python3 -c "import json; json.load(open('tools/mapper/profile.json'))"
   
       - name: Test traces load check
         run: |
-          tar -xzvf tools/mapper/test_files/trace.tar.gz -C tools/mapper/test_files/
           python3 tools/mapper/trace_load_test.py -t tools/mapper/test_files/extremes/
 
       - name: Extreme mapping tests
         run: |
           python3 tools/mapper/mapper.py -t tools/mapper/test_files/extremes -p tools/mapper/profile.json
+          diff tools/mapper/test_files/extremes/mapper_output.json tools/mapper/test_files/extremes/correct_mapper_output.json
 
-      - name: Run mapper tool on sample trace
+      - name: Run mapper tool on example trace
         run: |
-          python3 tools/mapper/mapper.py -t tools/mapper/test_files/20 -p tools/mapper/profile.json
+          python3 tools/mapper/mapper.py -t data/traces/example -p tools/mapper/profile.json
+          diff data/traces/example/mapper_output.json data/traces/example/correct_mapper_output.json

data/traces/example/correct_mapper_output.json

@@ -0,0 +1,14 @@
+{
+    "c13acdc7567b225971cef2416a3a2b03c8a4d8d154df48afe75834e2f5c59ddf": {
+        "proxy-function": "video-processing-python-10"
+    },
+    "a2faad786b3c813b12ce57d349d5e62f6d0f22ceecfa86cd72a962853383b600": {
+        "proxy-function": "image-rotate-go-11"
+    },
+    "7dc5aeabc131669912e8c793c8925cc9928321f45f13a4af031592b4611630d7": {
+        "proxy-function": "video-processing-python-100"
+    },
+    "ae8a1640fa932024f59b38a0b001808b5c64612bd60c6f3eb80ba9461ba2d091": {
+        "proxy-function": "video-processing-python-10"
+    }
+}

tools/mapper/.gitattributes

@@ -58,24 +58,29 @@ def get_closest_proxy_function(
         proxy_list.append(proxy_functions[function_name])
         proxy_functions[function_name]["index"] = len(proxy_list) - 1
 
-    for function_name in trace_functions:
+    for id in trace_functions:
         min_error = math.inf
         min_error_index = -1
         for i in range(0, len(proxy_list)):
-            error = get_error(trace_functions[function_name], proxy_list[i])
+            error = get_error(trace_functions[id], proxy_list[i])
             if error < min_error:
                 min_error = error
                 min_error_index = i
 
         if min_error == math.inf:
-            log.warning(f"Proxy function for function {function_name} not found. Using InVitro trace function.")
-            trace_functions[function_name]["proxy-function"] = "trace-func-go"
+            log.warning(f"Proxy function for unique id (HashFunction + HashOwner + HashApp) {id} not found. Using InVitro trace function.")
+            trace_functions[id]["proxy-function"] = "trace-func-go"
             continue
 
-        trace_functions[function_name]["proxy-function"] = proxy_list[
+        trace_functions[id]["proxy-function"] = proxy_list[
             min_error_index
         ]["name"]
 
+        if abs(trace_functions[id]["duration"]["50-percentile"] - proxy_functions[trace_functions[id]["proxy-function"]]["duration"]["50-percentile"]) > 0.4*trace_functions[id]["duration"]["50-percentile"]:
+            log.warning(f"Duration error for id {id} above 40%. Using InVitro trace function.")
+            trace_functions[id]["proxy-function"] = "trace-func-go"
+            continue
+
     for function_name in proxy_functions:
         del proxy_functions[function_name]["index"]
 

tools/mapper/find_proxy_function.py

@@ -55,52 +55,16 @@ def load_trace(trace_directorypath):
     trace_functions = {}
 
     for _, row in duration_info.iterrows():
-        hash_function = row["HashFunction"]
-        trace_functions[hash_function] = {}
-        trace_functions[hash_function]["duration"] = row
+        unique_id = row["HashFunction"] + row["HashOwner"] + row["HashApp"]
+        trace_functions[unique_id] = {}
+        trace_functions[unique_id]["duration"] = row
 
     for _, row in memory_info.iterrows():
-        hash_function = row["HashFunction"]
-        trace_functions[hash_function]["memory"] = row
+        unique_id = row["HashFunction"] + row["HashOwner"] + row["HashApp"]
+        trace_functions[unique_id]["memory"] = row
 
     return trace_functions, 0
 
-def generate_plot(trace_directorypath, trace_functions, proxy_functions, mapped_trace, invocation_statistics=True):
-    
-    if invocation_statistics:
-        invocations = pd.read_csv(trace_directorypath+"/invocations.csv")
-        inv_df = {}
-        for i in range(len(invocations)):
-            inv_df[invocations["HashFunction"][i]] = sum([invocations[col][i] if col not in invocations.columns[0:INVOCATION_COLUMN] else 0 for col in invocations.columns])
-        
-        trace_durations = []
-        mapped_durations = []
-        dropped_functions, dropped_invocations, total_invocations = 0, 0, 0
-        for trace in trace_functions:
-            duration = trace_functions[trace]["duration"]["50-percentile"]
-            if duration > VSWARM_MAX_DUR:
-                dropped_functions += 1
-                dropped_invocations += inv_df[trace]
-                continue
-            total_invocations += inv_df[trace]
-            proxy_name = mapped_trace[trace]["proxy-function"]
-            profile_duration = proxy_functions[proxy_name]["duration"]["50-percentile"]
-            trace_durations.append(duration)
-            mapped_durations.append(profile_duration)
-        return trace_durations, mapped_durations, dropped_functions, (dropped_invocations/total_invocations)*100
-    else:
-        trace_durations = []
-        mapped_durations = []
-        for trace in trace_functions:
-            duration = trace_functions[trace]["duration"]["50-percentile"]
-            if duration > VSWARM_MAX_DUR:
-                continue
-            proxy_name = mapped_trace[trace]["proxy-function"]
-            profile_duration = proxy_functions[proxy_name]["duration"]["50-percentile"]
-            trace_durations.append(duration)
-            mapped_durations.append(profile_duration)
-        return trace_durations, mapped_durations
-
 def main():
     # Parse the arguments
     parser = argparse.ArgumentParser(description="Mapper")
@@ -160,9 +124,9 @@ def main():
 
     # Only give function name and proxy name
     trace_json = {}
-    for function in trace_functions:
-        trace_json[function] = {}
-        trace_json[function]["proxy-function"] = trace_functions[function]["proxy-function"] 
+    for id in trace_functions:
+        trace_json[id] = {}
+        trace_json[id]["proxy-function"] = trace_functions[id]["proxy-function"] 
 
     try:
         with open(output_filepath, "w") as jf:
@@ -173,15 +137,7 @@ def main():
         return
 
     log.info(f"Output file {output_filepath} written")
-    log.info(f"Preliminary load Generation successful. Checking error levels in mapping...")
-
-    # Read the mapper output
-    try:
-        with open(output_filepath, "r") as jf:
-            mapper_output = json.load(jf)
-    except Exception as e:
-        log.critical(f"Error in loading mapper output file {e}")
-        return
+    log.info(f"Load Generation successful. Mapper output generated.")
 
     # Check the memory and duration errors
 
@@ -195,11 +151,14 @@ def main():
     abs_rel_mem_error = 0
     abs_rel_dur_error = 0
     zero_duration = 0
-    for function in mapper_output:
-        trace_mem = trace_functions[function]["memory"]["50-percentile"]
-        trace_dur = trace_functions[function]["duration"]["50-percentile"]
-        proxy_dur = proxy_functions[mapper_output[function]["proxy-function"]]["duration"]["50-percentile"]
-        proxy_mem = proxy_functions[mapper_output[function]["proxy-function"]]["memory"]["50-percentile"]
+    for id in trace_functions:
+        if trace_functions[id]["proxy-function"] == "trace-func-go":
+            dur_count += 1
+            continue
+        trace_mem = trace_functions[id]["memory"]["50-percentile"]
+        trace_dur = trace_functions[id]["duration"]["50-percentile"]
+        proxy_dur = proxy_functions[trace_functions[id]["proxy-function"]]["duration"]["50-percentile"]
+        proxy_mem = proxy_functions[trace_functions[id]["proxy-function"]]["memory"]["50-percentile"]
         mem_error += trace_mem - proxy_mem
         rel_mem_error += (trace_mem - proxy_mem)/trace_mem
         abs_mem_error += abs(trace_mem - proxy_mem)
@@ -211,11 +170,8 @@ def main():
         else:
             rel_dur_error += (trace_dur - proxy_dur)/trace_dur
             abs_rel_dur_error += abs((trace_dur - proxy_dur)/trace_dur)
-        if abs(trace_dur - proxy_dur) > 0.4*trace_dur:
-            mapper_output[function]["proxy-function"] = "trace-func-go"
-            dur_count += 1
 
-    total_functions = len(mapper_output)
+    total_functions = len(trace_functions)
     log.info(f"Average memory error: {mem_error/total_functions} MB per invocation")
     log.info(f"Average duration error: {dur_error/total_functions} ms per invocation")
     log.info(f"Average absolute memory error: {abs_mem_error/total_functions} MB per invocation")
@@ -227,18 +183,5 @@ def main():
     log.info(f"Duration errors: {dur_count}")
     log.info(f"Functions with 0 duration: {zero_duration}")
 
-    log.info(f"Replacing the functions with high duration error with invitro functions.")
-
-    ## Write the updated mapper output
-
-    try:
-        with open(output_filepath, "w") as jf:
-            json.dump(mapper_output, jf, indent=4)
-            log.info(f"Updated output file {output_filepath} written. Load generated.")
-    except Exception as e:
-        log.critical(f"Output file {output_filepath} cannot be written. Error: {e}")
-        log.critical(f"Load Generation failed")
-        return
-
 if __name__ == "__main__":
     main()

tools/mapper/mapper.py

@@ -0,0 +1,11 @@
+{
+    "fdd2e3c3758d30f677499f1ba3b6c39eb0d7df625a947568503e46e2e60b94e5": {
+        "proxy-function": "trace-func-go"
+    },
+    "f8620c125fb6e77e03d179c91045ae6b1bb69d6a692e75766b5959bff7f7e595": {
+        "proxy-function": "fibonacci-nodejs-100000-101000"
+    },
+    "4dd18b8612337a03958c30308bd81a96b46d24dc6ef32f60eb1cb687f6291f50": {
+        "proxy-function": "fibonacci-nodejs-10000-10100"
+    }
+}

tools/mapper/profile.json.gz

@@ -0,0 +1,4 @@
+HashOwner,HashApp,HashFunction,Average,Count,Minimum,Maximum,percentile_Average_0,percentile_Average_1,percentile_Average_25,percentile_Average_50,percentile_Average_75,percentile_Average_99,percentile_Average_100
+cf6b17f1e703bc9aa5dde7707333a17f94bf48c2f73c5a37ad046b81e552d261,7668725e5528728a76d7e74849fb7c9db7e37c6ba196d294ec53c35887758d0f,fdd2e3c3758d30f677499f1ba3b6c39eb0d7df625a947568503e46e2e60b94e5,93983,8,90640.0,98756.0,90640,90640,91138,93192,94213,98756,98756
+215b790f4b1b38b801826384739038dd419f60f8cab9d677e16a628f53848d50,82ace503c9cd11b2a34ee63767b2587b15c43ad4c2a60db4d4a038831f3dc009,f8620c125fb6e77e03d179c91045ae6b1bb69d6a692e75766b5959bff7f7e595,200,1,200.0,200.0,200,200,200,200,200,200,200
+710c5bac43705eb08b3434a2d00984a77710c6c19208a73ec62f8680d98a11b2,13a2239a8f7a35fe6aeda427c30cb982d928922e9366c2b2b4f00cfe9ccb8e35,4dd18b8612337a03958c30308bd81a96b46d24dc6ef32f60eb1cb687f6291f50,43,227,0.0,2031.0,0,14,15,17,31,625,2031

tools/mapper/profile.tar.gz

@@ -0,0 +1,4 @@
+HashOwner,HashApp,HashFunction,Trigger,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,622,623,624,625,626,627,628,629,630,631,632,633,634,635,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,662,663,664,665,666,667,668,669,670,671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689
+cf6b17f1e703bc9aa5dde7707333a17f94bf48c2f73c5a37ad046b81e552d261,7668725e5528728a76d7e74849fb7c9db7e37c6ba196d294ec53c35887758d0f,fdd2e3c3758d30f677499f1ba3b6c39eb0d7df625a947568503e46e2e60b94e5,timer,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+215b790f4b1b38b801826384739038dd419f60f8cab9d677e16a628f53848d50,82ace503c9cd11b2a34ee63767b2587b15c43ad4c2a60db4d4a038831f3dc009,f8620c125fb6e77e03d179c91045ae6b1bb69d6a692e75766b5959bff7f7e595,http,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+710c5bac43705eb08b3434a2d00984a77710c6c19208a73ec62f8680d98a11b2,13a2239a8f7a35fe6aeda427c30cb982d928922e9366c2b2b4f00cfe9ccb8e35,4dd18b8612337a03958c30308bd81a96b46d24dc6ef32f60eb1cb687f6291f50,http,1,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,1,2,1,0,0,0,0,0,0,0,0,0,2,1,2,2,1,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,1,1,1,0,1,0,0,0,2,2,0,0,0,0,1,0,3,1,0,0,0,0,0,0,0,0,2,0,0,0,2,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1