Skip to content

Commit d5ca597

Browse files
committed
Added docs pertaining to the usage of vSwarm functionality in the loader
Signed-off-by: aryans1204 <arshar1204@gmail.com> Beautified code according to Invitro standards Signed-off-by: aryans1204 <arshar1204@gmail.com> Added deployment of predeployment commands for vSwarm benchmarks Signed-off-by: aryans1204 <arshar1204@gmail.com> Removed erroneous E2E testfile Signed-off-by: aryans1204 <arshar1204@gmail.com> Added cleanup for kubectl pods Signed-off-by: aryans1204 <arshar1204@gmail.com> Fixed trace unit test Signed-off-by: aryans1204 <arshar1204@gmail.com> Changed function identifier to include hashOwner and hashApp Signed-off-by: aryans1204 <arshar1204@gmail.com> Added docs for deploy_info Signed-off-by: aryans1204 <arshar1204@gmail.com> Fixed spelling errors Signed-off-by: aryans1204 <arshar1204@gmail.com> Added docs for deploy info file generation Signed-off-by: aryans1204 <arshar1204@gmail.com> Created spelling fix for loader docs Signed-off-by: aryans1204 <arshar1204@gmail.com> Changed pre-deployment to Go command from bash script Signed-off-by: aryans1204 <arshar1204@gmail.com> Impoved documentation Signed-off-by: aryans1204 <arshar1204@gmail.com> Impoved documentation with spellcheck Signed-off-by: aryans1204 <arshar1204@gmail.com> Improved documentation Signed-off-by: aryans1204 <arshar1204@gmail.com> Fixed spellcheck in documentation Signed-off-by: aryans1204 <arshar1204@gmail.com> Updated docs Signed-off-by: aryans1204 <arshar1204@gmail.com> Changed log.Warn to log.Fatal Signed-off-by: aryans1204 <arshar1204@gmail.com>
1 parent 00d2d18 commit d5ca597

23 files changed

+258
-157
lines changed

.gitattributes

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
workloads/container/vSwarm_deploy_metadata.tar.gz filter=lfs diff=lfs merge=lfs -text

.github/workflows/e2e.yaml

Lines changed: 0 additions & 104 deletions
This file was deleted.

.github/workflows/e2e_loader.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ jobs:
5959
- name: Untar vSwarm YAMLs
6060
if: ${{ always() }}
6161
run: |
62-
tar -xzvf workloads/container/yamls.tar.gz -C workloads/container/
62+
tar -xzvf workloads/container/vSwarm_deploy_metadata.tar.gz -C workloads/container/
6363
- name: Run vSwarm loader
6464
run: go run cmd/loader.go --config pkg/config/test_vswarm_config.json
6565

.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ pkg/workload/openwhisk/*.zip
1515
tools/*/__pycache__
1616
tools/trace_synthesizer/*.csv
1717
tools/trace_synthesizer/*/*.csv
18+
workloads/container/yamls/
1819

1920
### CMake ###
2021
CMakeLists.txt.user
@@ -209,4 +210,4 @@ tools/plotter/test-out
209210
*.swp
210211

211212
data/traces/azure_*
212-
data/traces/day*
213+
data/traces/day*

cmd/loader.go

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -176,14 +176,16 @@ func runTraceMode(cfg *config.LoaderConfiguration, readIATFromFile bool, writeIA
176176
durationToParse := determineDurationToParse(cfg.ExperimentDuration, cfg.WarmupDuration)
177177
yamlPath := parseYAMLSpecification(cfg)
178178
var functions []*common.Function
179+
var traceParser trace.Parser
180+
179181
// Azure trace parsing
180182
if !cfg.VSwarm {
181-
traceParser := trace.NewAzureParser(cfg.TracePath, durationToParse, yamlPath)
182-
functions = traceParser.Parse()
183+
traceParser = trace.NewAzureParser(cfg.TracePath, durationToParse, yamlPath)
183184
} else {
184-
traceParser := trace.NewMapperParser(cfg.TracePath, durationToParse)
185-
functions = traceParser.Parse()
185+
traceParser = trace.NewMapperParser(cfg.TracePath, durationToParse)
186186
}
187+
188+
functions = traceParser.Parse()
187189
// Dirigent metadata parsing
188190
dirigentMetadataParser := trace.NewDirigentMetadataParser(cfg.TracePath, functions, yamlPath, cfg.Platform)
189191
dirigentMetadataParser.Parse()
@@ -204,7 +206,7 @@ func runTraceMode(cfg *config.LoaderConfiguration, readIATFromFile bool, writeIA
204206
TraceGranularity: parseTraceGranularity(cfg),
205207
TraceDuration: durationToParse,
206208

207-
YAMLPath: yamlPath, //make a slice of YAMLPaths instead
209+
YAMLPath: yamlPath,
208210
TestMode: false,
209211

210212
Functions: functions,
Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
{
2-
"c13acdc7567b225971cef2416a3a2b03c8a4d8d154df48afe75834e2f5c59ddf": {
3-
"proxy-function": "video-processing-python-10"
2+
"c13acdc7567b225971cef2416a3a2b03c8a4d8d154df48afe75834e2f5c59ddfc455703077a17a9b8d0fc655d939fcc6d24d819fa9a1066b74f710c35a43cbc868baea05aa0c3619b6feb78c80a07e27e4e68f921d714b8125f916c3b3370bf2": {
3+
"proxy-function": "cartservice"
44
},
5-
"a2faad786b3c813b12ce57d349d5e62f6d0f22ceecfa86cd72a962853383b600": {
5+
"a2faad786b3c813b12ce57d349d5e62f6d0f22ceecfa86cd72a962853383b600d7028b01f2422ea9d4f695cae4085800eb34540674081a46bb4e4388e7995f7ac8b8b9160f181010f509ee0af3266cbcb5a5f4c7700ba8d4396f1147e1ad3bbd": {
66
"proxy-function": "image-rotate-go-11"
77
},
8-
"7dc5aeabc131669912e8c793c8925cc9928321f45f13a4af031592b4611630d7": {
8+
"7dc5aeabc131669912e8c793c8925cc9928321f45f13a4af031592b4611630d70728f480194febeddbc0d2a69a2cb38344e495f264d52c62102e7bd99505dbf22cc1c836a3b32265b4a36b6e6f56b2d7c0588a926df5d03a76b7b4388cbf2258": {
99
"proxy-function": "video-processing-python-100"
1010
},
11-
"ae8a1640fa932024f59b38a0b001808b5c64612bd60c6f3eb80ba9461ba2d091": {
11+
"ae8a1640fa932024f59b38a0b001808b5c64612bd60c6f3eb80ba9461ba2d09101d1a13d091b0cfc764a125ead054ad6a720cb29b0b1fc2c187e9be3311f09fe90ba6ab2b1b9f5b5925d1e9ee6242d0f840ebcfb3221567bc2b5f24cb864b016": {
1212
"proxy-function": "video-processing-python-10"
1313
}
1414
}

docs/configuration.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
| EnableDAGDataset | bool | true/false | true | Generate width and depth from dag_structure.csv in TracePath[^9] |
3939
| Width | int | > 0 | 2 | Default width of DAG |
4040
| Depth | int | > 0 | 2 | Default depth of DAG |
41+
| VSwarm | bool | true/false | false | Execute vSwarm functions from mapper_output.json
4142

4243
[^1]: To run RPS experiments replace the path with `RPS`.
4344

docs/generate_deploy_info_docs.md

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Deploy Info JSON
2+
3+
The `deploy_info.json` file is used to identify deployment information for services and their pre-dependencies. It is pre-generated, and stored inside the `vSwarm_deploy_metadata.tar.gz` to contain deployment information for vSwarm functions.
4+
5+
## Schema and Usage
6+
7+
The `deploy_info.json` file in the `vSwarm_deploy_metadata.tar.gz` is used to identify the relative file paths for the Knative YAML manifests for deploying vSwarm functions. It also contains the path of YAML files needed as part of the pre-deployment commands to run certain vSwarm benchmarks, for example the `online-shop-database` which requires to be deployed before running `cartservice` benchmark.
8+
9+
The `deploy_info.json` has the following schema:
10+
```console
11+
{
12+
vswarm-function-name:
13+
{
14+
YamlLocation: /path/to/yaml
15+
PredeploymentPath: [/path/to/predeployment-database/yaml]
16+
}
17+
}
18+
```
19+
20+
The `PredeploymentPath` is the path to the YAML file, which is applied via `kubectl apply -f`, before creating the service under `YamlLocation`.
21+
22+
## Deployment File Generation
23+
24+
While the `deploy_info.json` file ships with the `vSwarm_deploy_metadata.tar.gz`, In order to regenerate the `deploy_info.json` run from the root of this repository:
25+
```console
26+
tar -xzvf workloads/container/vSwarm_deploy_metadata.tar.gz -C workloads/container
27+
cd workloads/container/
28+
python3 generate_deploy_info.py
29+
```
30+
31+
## YAML Generation
32+
33+
The `workloads/container/generate_all_yamls.py` is a wrapper script that calls the `generate-yamls.py` script for each vSwarm benchmark in the `vSwarm_deploy_metadata.tar.gz`. The `generate-yamls.py` Python script generates YAML files for vSwarm benchmarks with different workload parameters to create a variety of durations and memory consumption.
34+
35+
While the YAMLs are pre-generated inside the `vSwarm_deploy_metadata.tar.gz` tarball, to regenerate the YAMLs, run this from the root of this repository:
36+
```console
37+
tar -xzvf workloads/container/vSwarm_deploy_metadata.tar.gz -C workloads/container
38+
cd workloads/container
39+
python3 generate_all_yamls.py
40+
```

docs/loader.md

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,20 +109,42 @@ value previously collected.
109109
To account for difference in CPU performance set `ITERATIONS_MULTIPLIER=102` if using
110110
Cloudlab `xl170` or `d430` machines. (Date of measurement: 18-Oct-2022)
111111

112+
## Executing vSwarm functions
113+
If you would like to use vSwarm benchmarks as profile functions to execute, first you need to generate a `mapper_output.json` using the `mapper` tool. Please refer to `mapper.md` docs for usage of the mapper tool to generate an output file. Once the `mapper_output.json` has been generated in the input trace directory, next run the following from the root of this repository:
114+
115+
```console
116+
# install pre-requisites
117+
sudo apt update
118+
sudo apt -y install git-lfs pip xz-utils
119+
git lfs install
120+
git lfs fetch
121+
git lfs checkout
122+
```
123+
This retrieves the `vSwarm_deploy_metadata.tar.gz` from Git LFS. Then, untar this tarball by running the following command from the root of this repository:
124+
125+
```bash
126+
$ tar -xzvf workloads/container/vSwarm_deploy_metadata.tar.gz -C workloads/container/
127+
```
128+
This untar the tarball into the `workloads/container/yamls` directory, which contains deployment information and deployment YAML files for all vSwarm benchmarks. If you would like to change some of these deployment files of regenerate them, refer to `generate_deploy_info_docs.md` for an outline of what these deployment files are, and how you can regenerate them.
129+
112130
## Single execution
113131

114132
To run load generator use the following command:
115133

116134
```bash
117135
$ go run cmd/loader.go --config cmd/config_knative_trace.json
118136
```
137+
To run load generator with vSwarm functions based on `mapper_output.json` run the following:
138+
139+
```bash
140+
$ go run cmd/loader.go --config cmd/config_vswarm_trace.json
141+
```
142+
To direct the loader to execute vSwarm functions, set the `VSwarm` flag in the loader configuration `true`. For information on how to configure the workload for load generator, please refer to `docs/configuration.md`.
119143

120144
Additionally, one can specify log verbosity argument as `--verbosity [info, debug, trace]`. The default value is `info`.
121145

122146
To execute in a dry run mode without generating any load, set the `--dry-run` flag to `true`. This is useful for testing and validating configurations without executing actual requests.
123147

124-
For to configure the workload for load generator, please refer to `docs/configuration.md`.
125-
126148
There are a couple of constants that should not be exposed to the users. They can be examined and changed
127149
in `pkg/common/constants.go`.
128150

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ require (
3333
github.com/campoy/embedmd v1.0.0 // indirect
3434
github.com/davecgh/go-spew v1.1.1 // indirect
3535
github.com/davidmz/go-pageant v1.0.2 // indirect
36+
github.com/go-cmd/cmd v1.4.3 // indirect
3637
github.com/go-fonts/liberation v0.3.3 // indirect
3738
github.com/go-latex/latex v0.0.0-20240709081214-31cef3c7570e // indirect
3839
github.com/go-logr/logr v1.4.2 // indirect

0 commit comments

Comments
 (0)