Added deployment for vSwarm functions, end to end pipeline working

Signed-off-by: aryans1204 <arshar1204@gmail.com>

Author: aryans1204

cmd/config_vswarm_trace.json

@@ -0,0 +1,32 @@
+{
+  "Seed": 42,
+
+  "Platform": "Knative",
+  "InvokeProtocol" : "grpc",
+  "YAMLSelector": "container",
+  "EndpointPort": 80,
+
+  "BusyLoopOnSandboxStartup": false,
+
+  "TracePath": "data/traces/example/",
+  "Granularity": "minute",
+  "OutputPathPrefix": "data/out/experiment",
+  "IATDistribution": "exponential",
+  "CPULimit": "1vCPU",
+  "ExperimentDuration": 3,
+  "WarmupDuration": 0,
+
+  "IsPartiallyPanic": false,
+  "EnableZipkinTracing": false,
+  "EnableMetricsScrapping": false,
+  "MetricScrapingPeriodSeconds": 15,
+  "AutoscalingMetric": "concurrency",
+
+  "GRPCConnectionTimeoutSeconds": 15,
+  "GRPCFunctionTimeoutSeconds": 900,
+  "DAGMode": false,
+  "EnableDAGDataset": true,
+  "Width": 2,
+  "Depth": 2,
+  "VSwarm": true
+}

cmd/loader.go

@@ -180,8 +180,7 @@ func runTraceMode(cfg *config.LoaderConfiguration, readIATFromFile bool, writeIA
 	if !cfg.VSwarm {
 		traceParser := trace.NewAzureParser(cfg.TracePath, durationToParse, yamlPath)
 		functions = traceParser.Parse()
-	}
-	else {
+	} else {
 		traceParser := trace.NewMapperParser(cfg.TracePath, durationToParse)
 		functions = traceParser.Parse()
 	}

data/traces/example/mapper_output.json

@@ -0,0 +1,14 @@
+{
+    "c13acdc7567b225971cef2416a3a2b03c8a4d8d154df48afe75834e2f5c59ddf": {
+        "proxy-function": "video-processing-python-10"
+    },
+    "a2faad786b3c813b12ce57d349d5e62f6d0f22ceecfa86cd72a962853383b600": {
+        "proxy-function": "image-rotate-go-11"
+    },
+    "7dc5aeabc131669912e8c793c8925cc9928321f45f13a4af031592b4611630d7": {
+        "proxy-function": "video-processing-python-100"
+    },
+    "ae8a1640fa932024f59b38a0b001808b5c64612bd60c6f3eb80ba9461ba2d091": {
+        "proxy-function": "video-processing-python-10"
+    }
+}

pkg/driver/deployment/knative.go

@@ -26,7 +26,6 @@ var (
 type knativeDeployer struct{}
 
 type knativeDeploymentConfiguration struct {
-	YamlPath          string
 	IsPartiallyPanic  bool
 	EndpointPort      int
 	AutoscalingMetric string
@@ -38,7 +37,6 @@ func newKnativeDeployer() *knativeDeployer {
 
 func newKnativeDeployerConfiguration(cfg *config.Configuration) knativeDeploymentConfiguration {
 	return knativeDeploymentConfiguration{
-		YamlPath:          cfg.YAMLPath,
 		IsPartiallyPanic:  cfg.LoaderConfiguration.IsPartiallyPanic,
 		EndpointPort:      cfg.LoaderConfiguration.EndpointPort,
 		AutoscalingMetric: cfg.LoaderConfiguration.AutoscalingMetric,
@@ -61,7 +59,7 @@ func (*knativeDeployer) Deploy(cfg *config.Configuration) {
 
 			knativeDeploySingleFunction(
 				cfg.Functions[i],
-				knativeConfig.YamlPath,
+				cfg.Functions[i].YAMLPath,
 				knativeConfig.IsPartiallyPanic,
 				knativeConfig.EndpointPort,
 				knativeConfig.AutoscalingMetric,

pkg/trace/mapper_trace_parser.go

@@ -28,6 +28,11 @@ func NewMapperParser(directoryPath string, totalDuration int) *MapperTraceParser
 func (p* MapperTraceParser) extractFunctions(mapperOutput map[string]map[string]string, deploymentInfo map[string]map[string]interface{}, dirPath string) []*common.Function {
 	var result []*common.Function
 	invocations := parseInvocationTrace(dirPath+"/invocations.csv", p.duration)
+	runtime := parseRuntimeTrace(dirPath+"/durations.csv")
+	memory := parseMemoryTrace(dirPath+"/memory.csv")
+
+	runtimeByHashFunction := createRuntimeMap(runtime)
+	memoryByHashFunction := createMemoryMap(memory)
 	for i := 0; i < len(*invocations); i++ {
 		invocationStats := (*invocations)[i]
 		hashFunction := invocationStats.HashFunction
@@ -39,6 +44,8 @@ func (p* MapperTraceParser) extractFunctions(mapperOutput map[string]map[string]
 			Name: fmt.Sprintf("%s-%d-%d", proxyFunction, i, p.functionNameGenerator.Uint64()),
 
 			InvocationStats: &invocationStats,
+			RuntimeStats: runtimeByHashFunction[hashFunction],
+			MemoryStats: memoryByHashFunction[hashFunction],
 			YAMLPath: yamlPath,
 			PreDeploymentCommands: preDeploymentCommands,
 		}
@@ -51,8 +58,8 @@ func (p *MapperTraceParser) Parse() []*common.Function {
 	var mapperOutput map[string]map[string]string // HashFunction mapped to vSwarm function yaml.
 	var deploymentInfo map[string]map[string]interface{}
 	// Read the deployment info file for yaml locations and predeployment commands if any
-
-	deploymentInfoFile, err := os.ReadFile("deploy_info.json")
+	homeDir := os.Getenv("HOME")
+	deploymentInfoFile, err := os.ReadFile(homeDir+"/invitro/pkg/trace/deploy_info.json")
 	if err != nil {
 		log.Warn("No deployment info file")
 	}

workloads/container/.gitattributes

@@ -0,0 +1 @@
+yamls.tar.gz filter=lfs diff=lfs merge=lfs -text