Merge branch 'main' into mapper-tool

Author: KarthikL1729

.github/actions/ci_knative_setup/action.yml

@@ -0,0 +1,45 @@
+name: CI Knative Setup
+description: Sets up the CI environment with Kind, Serving, and Kourier for Knative platform type
+inputs:
+  go-version:
+    description: Version of Go to set up
+    required: true
+    default: '1.22'
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        lfs: "true"
+
+    - name: Checkout LFS objects
+      shell: bash
+      run: git lfs checkout
+
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: ${{ inputs.go-version }}
+
+    - name: Create k8s Kind Cluster
+      shell: bash
+      run: bash ./scripts/konk-ci/01-kind.sh
+
+    - name: Install Serving
+      shell: bash
+      run: bash ./scripts/konk-ci/02-serving.sh
+
+    - name: Install Kourier
+      shell: bash
+      run: bash ./scripts/konk-ci/02-kourier.sh
+
+    - name: Setup domain and autoscaler
+      shell: bash
+      run: |
+        INGRESS_HOST="127.0.0.1"
+        KNATIVE_DOMAIN=$INGRESS_HOST.sslip.io
+        kubectl patch configmap -n knative-serving config-domain -p "{\"data\": {\"$KNATIVE_DOMAIN\": \"\"}}"
+        kubectl patch configmap -n knative-serving config-autoscaler -p "{\"data\": {\"allow-zero-initial-scale\": \"true\"}}"
+        kubectl patch configmap -n knative-serving config-features -p "{\"data\": {\"kubernetes.podspec-affinity\": \"enabled\"}}"
+        kubectl label node knative-control-plane loader-nodetype=worker

.github/configs/wordlist.txt

@@ -798,3 +798,18 @@ AsyncMode
 AsyncResponseURL
 AsyncWaitToCollectMin
 Knative's
+BaseConfigPath
+LoaderStudy
+LoaderExperiment
+PostScript
+PreScript
+IatGeneration
+LoaderConfiguration
+PostScript
+PreScript
+TraceFormat
+TraceValues
+TracesDir
+TracesFormat
+baseLoaderConfig
+iats

.github/workflows/e2e.yaml .github/workflows/e2e_loader.yaml.github/workflows/e2e.yaml renamed to .github/workflows/e2e_loader.yaml

@@ -1,13 +1,13 @@
-name: End-to-End Tests
+name: End-to-End Loader Test
 
 on:
- schedule:
-   - cron: "0 9 * * 1"
- workflow_dispatch:
- push:
-   branches: [main]
- pull_request:
-   branches: [main]
+  schedule:
+    - cron: "0 9 * * 1"
+  workflow_dispatch:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
 
 env:
   GOOS: linux
@@ -20,7 +20,6 @@ jobs:
       KIND_VERSION: v0.22.0
       K8S_VERSION: v1.29
       YAML_DIR: workloads/container
-
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
@@ -30,33 +29,11 @@ jobs:
             trace_func_go,
           ]
     steps:
-      - uses: actions/checkout@v4
-        with:
-          lfs: "true"
-      - name: Checkout LFS objects
-        run: git lfs checkout
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version: 1.22
-
-      - name: Create k8s Kind Cluster
-        run: bash ./scripts/konk-ci/01-kind.sh
-
-      - name: Install Serving
-        run: bash ./scripts/konk-ci/02-serving.sh
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
-      - name: Install Kourier
-        run: bash ./scripts/konk-ci/02-kourier.sh
-
-      - name: Setup domain and autoscaler
-        run: |
-          INGRESS_HOST="127.0.0.1"
-          KNATIVE_DOMAIN=$INGRESS_HOST.sslip.io
-          kubectl patch configmap -n knative-serving config-domain -p "{\"data\": {\"$KNATIVE_DOMAIN\": \"\"}}"
-          kubectl patch configmap -n knative-serving config-autoscaler -p "{\"data\": {\"allow-zero-initial-scale\": \"true\"}}"
-          kubectl patch configmap -n knative-serving config-features -p "{\"data\": {\"kubernetes.podspec-affinity\": \"enabled\"}}"
-          kubectl label node knative-control-plane loader-nodetype=worker
+      - name: Set up Kubernetes KinD Cluster and install Knative
+        uses: ./.github/actions/ci_knative_setup
 
       - name: Build and run loader
         run: go run cmd/loader.go --config pkg/config/test_config.json
@@ -73,6 +50,7 @@ jobs:
           do
             kubectl logs -n default -c $container_name -l serving.knative.dev/service=${{ matrix.service }}
           done
+
       - name: Down
         if: ${{ always() }}
         run: |

.github/workflows/e2e_multi_loader.yaml

@@ -0,0 +1,68 @@
+name: End-to-End Multi-Loader Tests
+
+on:
+  schedule:
+    - cron: "0 9 * * 1"
+  workflow_dispatch:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+env:
+  GOOS: linux
+  GO111MODULE: on
+
+jobs:
+  test-multi-loader:
+    name: Test Multi-Loader with Knative Deployment
+    env:
+      KIND_VERSION: v0.22.0
+      K8S_VERSION: v1.29
+      YAML_DIR: workloads/container
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Kubernetes KinD Cluster and install Knative
+        uses: ./.github/actions/ci_knative_setup
+      
+      - name: Setup traces for multi trace test
+        shell: bash
+        run: bash ./scripts/setup/setup_multi_test_trace.sh
+
+      - name: Build and run multi-loader
+        run: go run tools/multi_loader/multi_loader.go --multiLoaderConfigPath tools/multi_loader/multi_loader_config.json
+
+      - name: Check multi_loader output
+        run: |
+          # check if there are 4 output folders: *_example_1_test, *_example_2_test, *_example_3_test & dry_run
+          folder_count=$(find data/out/multi-test/test-experiment -mindepth 1 -maxdepth 1 -type d | wc -l)
+          if [ "$folder_count" -ne 4 ]; then
+            echo "Output folder count is $folder_count, expected 4"
+            exit 1
+          else
+            echo "Output correct number of folders"
+          fi
+
+          # Check for errors in each output CSV file
+          for file in $(find . -name "*_example_1_test/example_1_test_duration_1.csv" -o -name "*_example_2_test/example_2_test_duration_1.csv" -o -name "*_example_3_test/example_3_test_duration_1.csv"); do
+            if [ ! -f "$file" ]; then
+              echo "File $file not found!"
+              exit 1
+            fi
+
+            if [ $(grep true "$file" | wc -l) -ne 0 ]; then
+              echo "Error found in $file"
+              exit 1
+            fi
+          done
+          echo "No errors found in output files"
+
+      - name: Down
+        if: ${{ always() }}
+        run: |
+          kn service delete --all

.github/workflows/tools-tests.yaml

@@ -22,7 +22,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        module: [ tools/plotter, ]
+        module: [ tools/plotter, tools/multi_loader/runner]
     steps:
     - name: Set up Golang
       uses: actions/setup-go@v5

docs/multi_loader.md

@@ -0,0 +1,112 @@
+# Multi-Loader
+
+A wrapper around loader to run multiple experiments in sequence with additional features like validation, dry-run, log collection
+
+## Prerequisites
+As a wrapper around loader, multi-loader requires the initial cluster setup to be completed. See [vHive Loader to create a cluster](https://github.com/vhive-serverless/invitro/blob/main/docs/loader.md#create-a-cluster)
+
+## Configuration
+### Multi-Loader Configuration
+| Parameter name      | Data type          | Possible values | Default value | Description                                                |
+|---------------------|--------------------|-----------------|---------------|------------------------------------------------------------|
+| Studies         | []LoaderStudy | N/A             | N/A           | A list of loader studies with their respective configurations. See [LoaderStudy](#loaderstudy) |
+| BaseConfigPath      | string             | "tools/multi_loader/base_loader_config.json" | N/A           | Path to the base configuration file                         |
+| IatGeneration         | bool                   | true, false                   | false         | (Optional) Whether to Generate iats only and skip invocations |
+| Generated             | bool                   | true, false                   | false         | (Optional) if iats were already generated         |
+| PreScript           | string             | any bash command | ""           | (Optional) A global script that runs once before all experiments |
+| PostScript          | string             | any bash command | ""           | (Optional) A global script that runs once after all experiments  |
+
+### LoaderStudy
+| Parameter name        | Data type              | Possible values               | Default value | Description                                                        |
+|-----------------------|------------------------|-------------------------------|---------------|--------------------------------------------------------------------|
+| Config                | map[string]interface{} | Any field in [LoaderConfiguration](https://github.com/vhive-serverless/invitro/blob/main/docs/configuration.md#loader-configuration-file-format) except `Platform`| N/A           | The configuration for each loader experiment which overrides configurations in baseLoaderConfig                      |
+| Name                  | string                 | N/A                           | N/A           | The name of the loader experiment                                  |
+| TracesDir             | string                 | N/A                           | N/A           | Directory containing the traces for the experiment                 |
+| TracesFormat          | string                 | "data/traces/example_{}"      | N/A           | Format of the trace files **The format string "{}" is required** |
+| TraceValues           | []interface{}          | ["any", 0, 1.1]               | N/A           | Values of the trace files Replaces the "{}" in TraceFormat             |
+| OutputDir             | string                 | any                           | data/out/{Name} | (Optional) Output directory for experiment results                 |
+| Verbosity             | string                 | "info", "debug", "trace"      | "info"        | (Optional) Verbosity level for logging the experiment             |
+| PreScript             | string                 | any bash Command              | ""           | (Optional) Local script that runs this specific experiment |
+| PostScript            | string                 | any bash Command              | ""           | (Optional) Local script that runs this specific experiment |
+
+> **_Important_**: 
+>
+> Only one of the following is required:
+> 1. `TracesDir`, or
+> 2. `TracesFormat` and `TraceValues`, or
+> 3. `TracePath` within the `LoaderExperiment`'s `Config` field
+>
+> If more than one is defined, the order of precedence is as follows:  
+> 1. `TracesDir`,  
+> 2. `TracesFormat` and `TraceValues`,  
+> 3. `TracePath`
+>
+> The `Platform` field must not be overridden and should only be defined in the base config.
+>
+> The `IatGeneration` and `Generated` fields may not function as expected when handling multiple experiments due to limitations in the loader implementation.
+
+> **_Note_**: 
+>
+> The `Config` field follows the same structure as the [LoaderConfiguration](https://github.com/vhive-serverless/invitro/blob/main/docs/configuration.md#loader-configuration-file-format). 
+> Any field defined in `Config` will override the corresponding value from the configuration in `BaseConfigPath`, but only for that specific experiment. 
+> For example, if `BaseConfigPath` has `ExperimentDuration` set to 5 minutes, and you define `ExperimentDuration` as 10 minutes in `Config`, that particular experiment will run for 10 minutes instead.
+
+## Command Flags
+
+The multi-loader accepts the following command-line flags. 
+
+> **_Note_**: These flags will subsequently be used during the execution of loader.go for **<u>every experiment</u>**. If you would like to define these flag for specific experiments only, define it in [LoaderStudy](#loaderstudy)
+
+Available flags:
+
+- **`--multiLoaderConfig`** *(default: `tools/multi_loader/multi_loader_config.json`)*:  
+  Specifies the path to the multi-loader configuration file. This file contains settings and parameters that define how the multi-loader operates [see above](#multi-loader-configuration)
+
+- **`--verbosity`** *(default: `info`)*:  
+    Sets the logging verbosity level. You can choose from the following options:
+    - `info`: Standard information messages.
+    - `debug`: Detailed debugging messages.
+    - `trace`: Extremely verbose logging, including detailed execution traces.
+
+- **`--failFast`** *(default: `false`)*:  
+  Determines whether the multi-loader should skip the study immediately after a failure. By default, the loader retries a failed experiment once with debug verbosity and skips the study only if the second attempt also fails. Setting this flag to `true` prevents the retry and skips the study after the first failure.
+
+
+
+## Multi-loader Overall Flow
+
+1. **Initialization**
+    - Flags for configuration file path, verbosity, IAT generation, and execution mode are parsed
+    - Logger is initialized based on verbosity level
+
+3. **Experiment Execution Flow**
+    - The multi-loader runner is instantiated with the provided configuration path.
+    - A dry run is executed to validate the setup for all studies:
+        - If any dry run fails, the execution terminates.
+    - If all dry runs succeed, proceed to actual runs:
+        - Global pre-scripts are executed.
+        - Each experiment undergoes the following steps:
+            1. **Pre-Execution Setup**
+                - Experiment-specific pre-scripts are executed.
+                - Necessary directories and folders are created.
+                - Each sub-experiment is unpacked and prepared
+            2. **Experiment Invocation**
+                - The loader is executed with generated configurations and related flags
+            3. **Post-Execution Steps**
+                - Experiment-specific post-scripts are executed
+                - Cleanup tasks are performed
+  
+4. **Completion**
+    - Global post-scripts are executed.
+    - Run Make Clean
+
+### How the Dry Run Works
+
+The dry run mode executes the loader with the `--dryRun` flag set to true after the unpacking of experiments defined in the multi-loader configurations.
+
+In this mode, the loader performs the following actions:
+
+- **Configuration Validation**: It verifies the experiment configurations without deploying any functions or generating invocations.
+- **Error Handling**: If a fatal error occurs at any point, the experiment will halt immediately.
+
+The purpose is to ensure that your configurations are correct and to identify any potential issues before actual execution.

pkg/common/utilities.go

@@ -25,11 +25,15 @@
 package common
 
 import (
+	"encoding/json"
 	"hash/fnv"
 	"log"
 	"math/rand"
+	"os/exec"
 	"strconv"
 	"strings"
+
+	logger "github.com/sirupsen/logrus"
 )
 
 type Pair struct {
@@ -146,4 +150,42 @@ func GetName(function *Function) int {
 		log.Fatal(err)
 	}
 	return functionId
-}
+}
+
+func DeepCopy[T any](a T) (T, error) {
+	var b T
+	byt, err := json.Marshal(a)
+	if err != nil {
+		return b, err
+	}
+	err = json.Unmarshal(byt, &b)
+	return b, err
+}
+
+func RunScript(command string) {
+	if command == "" {
+		return
+	}
+	logger.Debug("Running command ", command)
+	cmd, err := exec.Command("/bin/sh", command).Output()
+	if err != nil {
+		log.Fatal(err)
+	}
+	logger.Debug(string(cmd))
+}
+
+func ParseLogType(logString string) string {
+	logTypeArr := strings.Split(logString, "level=")
+	if len(logTypeArr) > 1 {
+		return strings.Split(logTypeArr[1], " ")[0]
+	}
+	return "info"
+}
+
+func ParseLogMessage(logString string) string {
+	message := strings.Split(logString, "msg=")
+	if len(message) > 1 {
+		return message[1][1 : len(message[1])-1]
+	}
+	return logString
+}