Added docs for deploy info file generation

Signed-off-by: aryans1204 <arshar1204@gmail.com>

Author: aryans1204

.gitattributes

@@ -0,0 +1 @@
+workloads/container/vSwarm_deploy_metadata.tar.gz filter=lfs diff=lfs merge=lfs -text

.github/workflows/e2e_loader.yaml

@@ -59,7 +59,7 @@ jobs:
       - name: Untar vSwarm YAMLs
         if: ${{ always() }}
         run: |
-          tar -xzvf workloads/container/yamls.tar.gz -C workloads/container/ 
+          tar -xzvf workloads/container/vSwarm_deploy_metadata.tar.gz -C workloads/container/ 
       - name: Run vSwarm loader
         run: go run cmd/loader.go --config pkg/config/test_vswarm_config.json
 

docs/generate_deploy_info_docs.md

@@ -1,10 +1,21 @@
 # Deploy Info JSON
 
-## The `deploy_info.json` file in the `yamls.tar.gz`  is used to identify the relative file paths for the Knative YAML manifests for deploying vSwarm functions. It is generated using `generate_deploy_info.py` Python script, also present under `yamls.tar.gz`, which outputs a JSON that embeds the yaml-location and pre-deployment commands for every vSwarm function.
+## The `deploy_info.json` file in the `yamls.tar.gz`  is used to identify the relative file paths for the Knative YAML manifests for deploying vSwarm functions. It is generated using `generate_deploy_info.py` Python script, also present under `yamls.tar.gz`, which outputs a JSON that embeds the yaml-location and pre-deployment commands for every vSwarm function. zit also contains the path of YAML files needed as part of the predeployment commands to run certain vSwarm benchmakrs, for example the `online-shop-database` which requires to be deployed before running `cartservice` benchmark.
 
 ## While the `deploy_info.json` file ships with the `yamls.tar.gz`, In order to regenerate the `deploy_info.json` run:
 ```console
-tar -xzvf workloads/container/yamls.tar.gz -C workloads/container
+tar -xzvf workloads/container/vSwarm_deploy_metadata.tar.gz -C workloads/container
 cd workloads/container/yamls/
 python3 generate_deploy_info.py
+```
+
+The `deploy_info.json` has the following schema:
+```console
+{
+    vswarm-function-name:
+        {
+            YamlLocation: /path/to/yaml
+            PredeploymentPath: [/path/to/predeployment/yaml]
+        }
+}
 ```

docs/loader.md

@@ -123,7 +123,7 @@ git lfs checkout
 This retrieves the `yamls.tar.gz` from Git LFS. Then, untar this tarball by running the following command from the root of this directory:
 
 ```bash
-$ tar -xzvf workloads/container/yamls.tar.gz -C workloads/container/
+$ tar -xzvf workloads/container/vSwarm_deploy_metadata.tar.gz -C workloads/container/
 ```
 
 ## Single execution

workloads/container/generate_all_yamls.py

@@ -0,0 +1,20 @@
+# Write a python script to run go through each directory and generate the yaml files by running the python script inside each directory
+
+import os
+import subprocess
+
+# Get the current working directory
+yaml_dir = os.getcwd()+"/yamls"
+
+# Get the list of directories
+directories = [d for d in os.listdir(yaml_dir) if os.path.isdir(d)]
+
+# Go through each directory and run the python script
+for directory in directories:
+    os.chdir(directory)
+    # Run the python script
+    subprocess.run(['python3', 'generate-yamls.py'])
+    print(f"Generated yaml files for {directory}")
+    os.chdir(current_dir)
+
+print("Generated all yaml files")

workloads/container/generate_deploy_info.py

@@ -0,0 +1,74 @@
+# Write a script to generate the deploy info for each yaml file based on the directory and write to a json calleds deploy_info.json
+
+# Example
+# "aes-python-20000-20200": {
+#        "yaml-location": "./yamls/aes-python/kn-aes-python-20000-20200.yaml",
+#        "predeployment-commands": []
+#    },
+#
+import os
+import glob
+import json
+
+# Get the current working directory
+current_dir = os.getcwd()+"/yamls"
+
+# Get the list of directories
+directories = [d for d in os.listdir(current_dir) if os.path.isdir(d)]
+
+predeployment_required = ["hotel-app", "image-rotate-python", "image-rotate-go", "video-processing", "video-analytics-standalone", "online-shop"]
+
+deploy_info = {}
+for directory in directories:
+    os.chdir(directory)
+    # Get the list of yaml files
+    yaml_files = glob.glob("*.yaml")
+    function_name = {}
+    for yaml_file in yaml_files:
+        function_name[yaml_file] = yaml_file.replace(".yaml", "")
+        function_name[yaml_file] = function_name[yaml_file].replace("kn-", "")
+    for yaml_file in yaml_files:
+        if directory in predeployment_required:
+            if directory == "hotel-app":
+                deploy_info[directory + '-' + function_name[yaml_file]] = {
+                    "YamlLocation": f"workloads/container/yamls/{directory}/{yaml_file}",
+                    "PredeploymentPath": [f"workloads/container/yamls/hotel-app/database.yaml"]
+                }
+            elif directory == "online-shop":
+                deploy_info[function_name[yaml_file]] = {
+                    "YamlLocation": f"workloads/container/yamls/{directory}/{yaml_file}",
+                    "PredeploymentPath": [f"workloads/container/yamls/online-shop/database.yaml"]
+                }
+            elif directory == "image-rotate-go" or directory == "image-rotate-python":
+                deploy_info[function_name[yaml_file]] = {
+                    "YamlLocation": f"workloads/container/yamls/{directory}/{yaml_file}",
+                    "PredeploymentPath": [f"workloads/container/yamls/{directory}/image-rotate-database.yaml"]
+                }
+            elif directory == "video-processing":
+                deploy_info[function_name[yaml_file]] = {
+                    "YamlLocation": f"workloads/container/yamls/{directory}/{yaml_file}",
+                    "PredeploymentPath": [f"workloads/container/yamls/{directory}/video-processing-database.yaml"]
+                }
+            elif directory == "video-analytics-standalone":
+                deploy_info[function_name[yaml_file]] = {
+                    "YamlLocation": f"workloads/container/yamls/{directory}/{yaml_file}",
+                    "PredeploymentPath": [f"workloads/container/yamls/{directory}/video-analytics-standalone-database.yaml"]
+                }
+        else:
+            deploy_info[function_name[yaml_file]] = {
+                "YamlLocation": f"workloads/container/yamls/{directory}/{yaml_file}",
+                "PredeploymentPath": []
+            }
+
+
+    os.chdir(current_dir)
+
+# Write the deploy info to a json file
+deploy_keys = sorted(deploy_info)
+sorted_deploy_info = {}
+for key in deploy_keys:
+    sorted_deploy_info[key] = deploy_info[key]
+with open("deploy_info.json", "w") as f:
+    f.write(json.dumps(sorted_deploy_info, indent=4))
+print("Generated deploy info json file")
+