Added the mapper tool to map trace to vSwarm proxies

.github/configs/wordlist.txt

@@ -798,3 +798,19 @@ TracesFormat
 baseLoaderConfig
 iats
 RpsDataSizeMB
+DIRECTORYPATH
+FILEPATH
+directorypath
+filepath
+HashApp
+HashFunction
+HashOwner
+AverageAllocatedMb
+SampleCount
+Jonker
+Pipelined
+SciPy
+Volgenant
+injective
+py
+DDR

.github/workflows/tools-tests.yaml

@@ -35,3 +35,48 @@ jobs:
     - name: Run tests
       working-directory: ${{ matrix.module }}
       run: go test -cover -race
+
+  mapper_e2e:
+    name: Mapper E2E Test
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        module: [ tools/mapper, ]
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+        with:
+          lfs: 'true'
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+
+      - uses: actions/cache@v4
+        with:
+          path: ${{ env.pythonLocation }}
+          key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('requirements.txt') }}
+
+      - name: Install requirements
+        run: pip install -r ./requirements.txt
+
+      - name: Profile load check
+        run: |
+          gzip -d tools/mapper/profile.json.gz -c > tools/mapper/profile.json
+          python3 -c "import json; json.load(open('tools/mapper/profile.json'))"
+
+      - name: Test traces load check
+        run: |
+          python3 tools/mapper/trace_load_test.py -t tools/mapper/test_files/extremes/
+
+      - name: Extreme mapping tests
+        run: |
+          python3 tools/mapper/mapper.py -t tools/mapper/test_files/extremes -p tools/mapper/profile.json
+          diff tools/mapper/test_files/extremes/mapper_output.json tools/mapper/test_files/extremes/correct_mapper_output.json
+
+      - name: Run mapper tool on example trace
+        run: |
+          python3 tools/mapper/mapper.py -t data/traces/example -p tools/mapper/profile.json
+          diff data/traces/example/mapper_output.json data/traces/example/correct_mapper_output.json

data/traces/example/correct_mapper_output.json

@@ -0,0 +1,14 @@
+{
+    "c13acdc7567b225971cef2416a3a2b03c8a4d8d154df48afe75834e2f5c59ddfc455703077a17a9b8d0fc655d939fcc6d24d819fa9a1066b74f710c35a43cbc868baea05aa0c3619b6feb78c80a07e27e4e68f921d714b8125f916c3b3370bf2": {
+        "proxy-function": "video-processing-python-10"
+    },
+    "a2faad786b3c813b12ce57d349d5e62f6d0f22ceecfa86cd72a962853383b600d7028b01f2422ea9d4f695cae4085800eb34540674081a46bb4e4388e7995f7ac8b8b9160f181010f509ee0af3266cbcb5a5f4c7700ba8d4396f1147e1ad3bbd": {
+        "proxy-function": "image-rotate-go-11"
+    },
+    "7dc5aeabc131669912e8c793c8925cc9928321f45f13a4af031592b4611630d70728f480194febeddbc0d2a69a2cb38344e495f264d52c62102e7bd99505dbf22cc1c836a3b32265b4a36b6e6f56b2d7c0588a926df5d03a76b7b4388cbf2258": {
+        "proxy-function": "video-processing-python-100"
+    },
+    "ae8a1640fa932024f59b38a0b001808b5c64612bd60c6f3eb80ba9461ba2d09101d1a13d091b0cfc764a125ead054ad6a720cb29b0b1fc2c187e9be3311f09fe90ba6ab2b1b9f5b5925d1e9ee6242d0f840ebcfb3221567bc2b5f24cb864b016": {
+        "proxy-function": "video-processing-python-10"
+    }
+}

docs/mapper.md

@@ -0,0 +1,138 @@
+# Mapper
+
+Use the mapper tool to map functions in a specific trace directory (with memory and duration traces) to proxy functions in the [`vSwarm`](https://github.com/vhive-serverless/vSwarm/tree/main/) benchmark suite. The benchmarks in the vSwarm suite have been profiled, and their memory utilization and duration traces are stored in the `profile.json` file. The tool maps each function in the trace to its closest proxy in the benchmark suite based on memory and duration correlation.
+
+The [`profiler` tool](https://github.com/vhive-serverless/vSwarm/tree/load-generator/tools/profiler#profiler) generates the `profile.json` JSON output file to profile the benchmark suite functions. We provide the `profile.json` file as a compressed file in the `tools/mapper` directory.
+
+### Usage
+
+```bash
+usage: mapper.py [-h] -t TRACE_DIRECTORYPATH -p PROFILE_FILEPATH
+
+Arguments:
+    -h, --help            show this help message and exit
+    -t TRACE_DIRECTORYPATH, --trace-directorypath TRACE_DIRECTORYPATH
+                                                Path to the directory containing the trace files (required)
+    -p PROFILE_FILEPATH, --profile-filepath PROFILE_FILEPATH
+                                                Path to the profile file containing the proxy functions
+```
+
+The tool reads trace information (memory and duration details) from the `trace/` directory, configurable using `-t` or `--trace-directorypath` flags. The `trace/` directory must contain `memory.csv` and `durations.csv` files with trace information in the format mentioned in the [*Azure Functions Dataset 2019*](https://github.com/Azure/AzurePublicDataset/blob/master/AzureFunctionsDataset2019.md).
+
+#### Function Execution Duration `durations.csv` Schema
+
+|Field|Description  |
+|--|--|
+| HashOwner | unique id of the application owner |
+| HashApp | unique id for application name  |
+| HashFunction | unique id for the function name within the app | 
+|Average | Average execution time (ms) across all invocations of the 24-period|  
+|Count | Number of executions used in computing the average|  
+|Minimum | Minimum execution time|  
+|Maximum | Maximum execution time|  
+|percentile_Average_0| Weighted 0th-percentile of the execution time *average*|  
+|percentile_Average_1| Weighted 1st-percentile of the execution time *average*|  
+|percentile_Average_25 | Weighted 25th-percentile of the execution time *average*|  
+|percentile_Average_50 | Weighted 50th-percentile of the execution time *average*|  
+|percentile_Average_75 | Weighted 75th-percentile of the execution time *average*|  
+|percentile_Average_99 | Weighted 99th-percentile of the execution time *average*|  
+|percentile_Average_100 | Weighted 100th-percentile of the execution time *average*|
+Execution time is in milliseconds. 
+
+#### Function Memory Usage `memory.csv` Schema
+
+|Field|Description  |
+|--|--|
+| HashOwner | unique id of the application owner |
+| HashApp | unique id for application name  |
+| HashFunction | unique id for the function name within the app |
+|SampleCount | Number of samples used for computing the average |  
+|AverageAllocatedMb | Average allocated memory across all SampleCount measurements|  
+|AverageAllocatedMb_pct1 | 1st percentile of the average allocated memory|  
+|AverageAllocatedMb_pct5 | 5th percentile of the average allocated memory|  
+|AverageAllocatedMb_pct25 | 25th percentile of the average allocated memory|  
+|AverageAllocatedMb_pct50 | 50th percentile of the average allocated memory|  
+|AverageAllocatedMb_pct75 | 75th percentile of the average allocated memory|  
+|AverageAllocatedMb_pct95 | 95th percentile of the average allocated memory|  
+|AverageAllocatedMb_pct99 | 99th percentile of the average allocated memory|  
+|AverageAllocatedMb_pct100 | 100th percentile of the average allocated memory|
+
+Use the [`sampler`](https://github.com/vhive-serverless/invitro/tree/main/sampler) tool in InVitro to generate sampled traces from the original Azure traces.
+
+For every function in the trace, the tool sets the closest function in the [`vSwarm`](https://github.com/vhive-serverless/vSwarm/tree/main/) benchmark suite as its proxy (considering 50-percentile memory and 50-percentile duration for the highest correlation). The 50th percentile ensures that the mapping corresponds not only to the peak values of the workload but also to a representative proxy function. Currently, the tool uses only _Serving Functions_ that are _NOT Pipelined_ as proxy functions.
+
+vSwarm currently *does not* fully cover the Azure trace functions. To avoid high error in function mapping, we set a hard threshold of 40% as the maximum absolute error (from the actual trace function duration) that a proxy function can have to be mapped to a trace function. If no eligible function is found for mapping from vSwarm, we use standard InVitro trace functions for those functions alone.
+
+The mapper requires the profiles of the vSwarm benchmark functions to identify proxies. The tool uses the `profile.json` JSON output file generated by the [`profiler` tool](https://github.com/vhive-serverless/vSwarm/tree/load-generator/tools/profiler#profiler) to obtain the profile of the benchmark suite functions. We provide a profile file in the `tools/mapper` directory as a compressed file and once decompressed, this is used as the default profile. Users can configure the path of the profile file to be used through the `-p` (or `--profile-filepath`) flag.
+
+An example of a generated output file is as follows:
+
+```json
+{
+        "c13acdc7567b225971cef2416a3a2b03c8a4d8d154df48afe75834e2f5c59ddf": {
+                "proxy-function": "video-processing-python-10"
+        },
+        "a2faad786b3c813b12ce57d349d5e62f6d0f22ceecfa86cd72a962853383b600": {
+                "proxy-function": "image-rotate-go-11"
+        },
+        "7dc5aeabc131669912e8c793c8925cc9928321f45f13a4af031592b4611630d7": {
+                "proxy-function": "video-processing-python-70"
+        },
+        "ae8a1640fa932024f59b38a0b001808b5c64612bd60c6f3eb80ba9461ba2d091": {
+                "proxy-function": "video-processing-python-20"
+        }
+}
+```
+
+The mapper stores the output file in the trace directory with the name `mapper_output.json` by default. The output file contains the mapping of the trace functions to the proxy functions in the vSwarm benchmark suite.
+
+### vSwarm statistics
+
+The vSwarm benchmark suite has been profiled, and the statistics of the functions are stored in the `profile.json` file. The statistics include the memory and duration details of the functions **(as profiled on two Intel Xeon Silver 4114 10-core CPUs at 2.20 GHz with 192GB of DDR4 RAM)**. The table below gives an overview of broad statistics of the functions in the vSwarm benchmark suite.
+
+| Metric | Value |
+| --- | --- |
+| Minimum Duration (ms) | 0.7466532679738556 ms |
+| Maximum Duration (ms) | 27459.274496825394 ms |
+| Average Duration (ms) | 2612.325542552247 ms |
+| Minimum Memory (MB) | 24.235294117647058 MB |
+| Maximum Memory (MB) | 1293.3715789473683 MB |
+| Average Memory (MB) | 142.66478734351986 MB |
+
+### Error metrics and mapping accuracy 
+
+We evaluated the accuracy and effectiveness of the mapper tool using the following metrics:
+
+- Duration CDF plot of the trace functions and the generated mapping
+- Wasserstein distance (WD) plot between the trace functions and the mapped functions
+
+As mentioned previously, the maximum function duration achievable with vSwarm functions is near 27 seconds. For the WD plot, we exclude trace functions beyond this duration to better compare the mapping accuracy.
+
+The CDF plot of the trace functions and the mapped functions is as follows:
+
+![CDF plot of trace functions and mapped functions](mapper_cdf.png)
+
+As shown, the mapping is quite accurate for the majority of the functions. The WD plot is as follows:
+
+![Wasserstein distance plot between trace functions and mapped functions](WD.png)
+
+The dropped functions plot is shown below:
+
+![Functions dropped from the WD plot](WD_dropped_data.png)
+
+Additionally, we display the following error metrics whenever the mapper runs with a trace (the specified values below are results for the full Azure trace mapping):
+
+| Metric | Value |
+| --- | --- |
+| Average memory error | -7.638341413565764 MB per invocation |
+| Average duration error | 4174.5554028958695 ms per invocation |
+| Average absolute memory error | 24.24782794856284 MB per invocation |
+| Average absolute duration error | 4414.451828203135 ms per invocation |
+| Average relative memory error | -0.8412999109296387 |
+| Average relative duration error | 0.004934168605729668 |
+| Average absolute relative memory error | 1.0028566557523266 |
+| Average absolute relative duration error | 0.20141343497568448 |
+| Functions with 0 duration | 1596 |
+| Number of mapped functions with higher than 40% duration error (replaced by InVitro trace functions) | 5258 |
+
+---

requirements.txt

@@ -6,4 +6,5 @@ pytest==8.3.2
 cloudpickle==3.1.0
 seaborn==0.13.0
 tqdm==4.67.0
-statsmodels==0.14.0
+statsmodels==0.14.0
+argparse==1.4.0

tools/mapper/find_proxy_function.py

@@ -0,0 +1,89 @@
+import math
+from log_config import *
+
+def get_error(trace_function, proxy_function) -> float:
+    """
+    Returns a float value on how close the trace function is to the proxy function. Lower the value, better the correlation.
+    Euclidean distance between normalized memory and duration is considered.
+
+    Parameters:
+    - `trace_function` (dict): Dictionary containing information regarding trace function
+    - `proxy_function` (dict): Dictionary containing information regarding proxy function
+
+    Returns:
+    - `float`: closeness value
+    """
+
+    try:
+        trace_memory = trace_function["memory"]["50-percentile"]
+        proxy_memory = proxy_function["memory"]["50-percentile"]
+        trace_duration = trace_function["duration"]["50-percentile"]
+        proxy_duration = proxy_function["duration"]["50-percentile"]
+    except KeyError as e:
+        log.warning(f"Correlation cannot be found. Error: {e}")
+        return math.inf
+
+    # NOTE: Better Error mechanisms can be considered to improve the correlation
+    # Currently only the 50%tile memory and duration are considered.
+    # Euclidean distance between normalized memory and duration is considered
+    try:
+        if trace_memory == 0: trace_memory += 0.01
+        if trace_duration == 0: trace_duration += 0.01
+        diff_memory = (math.log(trace_memory) - math.log(proxy_memory)) 
+        diff_duration = (math.log(trace_duration) - math.log(proxy_duration)) 
+        error = math.sqrt((diff_memory) ** 2 + (diff_duration) ** 2)
+        return error
+    except ValueError as e:
+        log.warning(f"Correlation cannot be found. Error: {e}")
+        return math.inf
+
+
+def get_closest_proxy_function(
+    trace_functions: dict, proxy_functions: dict
+) -> dict:
+    """
+    Obtains the closest proxy function for every trace function
+
+    Parameters:
+    - `trace_functions` (dict): Dictionary containing information regarding trace functions
+    - `proxy_functions` (dict): Dictionary containing information regarding proxy functions
+
+    Returns:
+    - `dict`: Dictionary containing information regarding trace functions with the associated proxy functions
+    - `int`: 0 if no error. -1 if error
+    """
+
+    proxy_list = []
+    for function_name in proxy_functions:
+        proxy_list.append(proxy_functions[function_name])
+        proxy_functions[function_name]["index"] = len(proxy_list) - 1
+
+    for id in trace_functions:
+        min_error = math.inf
+        min_error_index = -1
+        for i in range(0, len(proxy_list)):
+            error = get_error(trace_functions[id], proxy_list[i])
+            if error < min_error:
+                min_error = error
+                min_error_index = i
+
+        if min_error == math.inf:
+            log.warning(f"Proxy function for unique id (HashFunction + HashOwner + HashApp) {id} not found. Using InVitro trace function.")
+            trace_functions[id]["proxy-function"] = "trace-func-go"
+            continue
+
+        trace_functions[id]["proxy-function"] = proxy_list[
+            min_error_index
+        ]["name"]
+
+        if abs(trace_functions[id]["duration"]["50-percentile"] - proxy_functions[trace_functions[id]["proxy-function"]]["duration"]["50-percentile"]) > 0.4*trace_functions[id]["duration"]["50-percentile"]:
+            log.warning(f"Duration error for id {id} above 40%. Using InVitro trace function.")
+            trace_functions[id]["proxy-function"] = "trace-func-go"
+            continue
+
+    for function_name in proxy_functions:
+        del proxy_functions[function_name]["index"]
+
+    log.info("Proxy functions found for all trace functions.")
+
+    return trace_functions

tools/mapper/log_config.py

@@ -0,0 +1,33 @@
+import logging
+
+class CustomFormatter(logging.Formatter):
+
+    blue = "\x1b[34;20m"
+    green = "\x1b[32;20m"
+    grey = "\x1b[38;20m"
+    yellow = "\x1b[33;20m"
+    red = "\x1b[31;20m"
+    bold_red = "\x1b[31;1m"
+    bold_red_white_bg = "\x1b[1;37;41m"
+    reset = "\x1b[0m"
+    format = "%(asctime)s - %(levelname)s - (%(filename)s:%(lineno)d) - %(message)s"
+
+    FORMATS = {
+        logging.DEBUG: grey + format + reset,
+        logging.INFO: green + format + reset,
+        logging.WARNING: yellow + format + reset,
+        logging.ERROR: red + format + reset,
+        logging.CRITICAL: bold_red_white_bg + format + reset,
+    }
+
+    def format(self, record):
+        log_fmt = self.FORMATS.get(record.levelno)
+        formatter = logging.Formatter(log_fmt)
+        return formatter.format(record)
+
+
+log = logging.getLogger("Benchmark")
+log.setLevel(logging.INFO)
+ch = logging.StreamHandler()
+ch.setFormatter(CustomFormatter())
+log.addHandler(ch)