Remove benchmarker CSV input in favor of JSON awslabs#946

Signed-off-by: Scott Haddlesey <[email protected]>

Author: haddscot

benchmark/comparisonTest/main.go

@@ -34,7 +34,7 @@ func main() {
 
 	var (
 		numberOfTests int
-		configFile    string
+		jsonFile      string
 		showCom       bool
 		imageList     []benchmark.ImageDescriptor
 		err           error
@@ -43,7 +43,7 @@ func main() {
 
 	flag.BoolVar(&showCom, "show-commit", false, "tag the commit hash to the benchmark results")
 	flag.IntVar(&numberOfTests, "count", 5, "Describes the number of runs a benchmarker should run. Default: 5")
-	flag.StringVar(&configFile, "f", "default", "Path to a file describing image details as a json file or csv in this order ['Name','Image ref', 'Ready line', 'manifest ref'].")
+	flag.StringVar(&jsonFile, "f", "default", "Path to a json file describing image details in this order ['Name','Image ref', 'Ready line', 'manifest ref']")
 
 	flag.Parse()
 
@@ -53,12 +53,12 @@ func main() {
 		commit = "N/A"
 	}
 
-	if configFile == "default" {
+	if jsonFile == "default" {
 		imageList = benchmark.GetDefaultWorkloads()
 	} else {
-		imageList, err = benchmark.GetImageList(configFile)
+		imageList, err = benchmark.GetImageList(jsonFile)
 		if err != nil {
-			errMsg := fmt.Sprintf("Failed to read file %s with error:%v\n", configFile, err)
+			errMsg := fmt.Sprintf("Failed to read file %s with error:%v\n", jsonFile, err)
 			panic(errMsg)
 		}
 	}

benchmark/performanceTest/main.go

@@ -35,7 +35,7 @@ func main() {
 
 	var (
 		numberOfTests           int
-		configFile              string
+		jsonFile                string
 		showCom                 bool
 		parseFileAccessPatterns bool
 		commit                  string
@@ -46,7 +46,7 @@ func main() {
 	flag.BoolVar(&parseFileAccessPatterns, "parse-file-access", false, "Parse fuse file access patterns.")
 	flag.BoolVar(&showCom, "show-commit", false, "tag the commit hash to the benchmark results")
 	flag.IntVar(&numberOfTests, "count", 5, "Describes the number of runs a benchmarker should run. Default: 5")
-	flag.StringVar(&configFile, "f", "default", "Path to a file describing image details as a json file or csv in this order ['Name','Image ref', 'Ready line', 'manifest ref'].")
+	flag.StringVar(&jsonFile, "f", "default", "Path to a json file describing image details in this order ['Name','Image ref', 'Ready line', 'manifest ref']")
 
 	flag.Parse()
 
@@ -68,12 +68,12 @@ func main() {
 		}
 	}
 
-	if configFile == "default" {
+	if jsonFile == "default" {
 		imageList = benchmark.GetDefaultWorkloads()
 	} else {
-		imageList, err = benchmark.GetImageList(configFile)
+		imageList, err = benchmark.GetImageList(jsonFile)
 		if err != nil {
-			errMsg := fmt.Sprintf("Failed to read file %s with error:%v\n", configFile, err)
+			errMsg := fmt.Sprintf("Failed to read file %s with error:%v\n", jsonFile, err)
 			panic(errMsg)
 		}
 	}

benchmark/stargzTest/main.go

@@ -32,16 +32,16 @@ var (
 
 func main() {
 	commit := os.Args[1]
-	configFile := os.Args[2]
+	jsonFile := os.Args[2]
 	numberOfTests, err := strconv.Atoi(os.Args[3])
 	stargzBinary := os.Args[4]
 	if err != nil {
 		errMsg := fmt.Sprintf("Failed to parse number of test %s with error:%v\n", os.Args[3], err)
 		panic(errMsg)
 	}
-	imageList, err := benchmark.GetImageList(configFile)
+	imageList, err := benchmark.GetImageList(jsonFile)
 	if err != nil {
-		errMsg := fmt.Sprintf("Failed to read file %s with error:%v\n", configFile, err)
+		errMsg := fmt.Sprintf("Failed to read file %s with error:%v\n", jsonFile, err)
 		panic(errMsg)
 	}
 

benchmark/utils.go

@@ -17,9 +17,7 @@
 package benchmark
 
 import (
-	"encoding/csv"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -109,15 +107,7 @@ func GetImageList(file string) ([]ImageDescriptor, error) {
 		return nil, err
 	}
 	defer f.Close()
-	images, jsonErr := GetImageListFromJSON(f)
-	if jsonErr == nil {
-		return images, nil
-	}
-	images, csvErr := GetImageListFromCsv(f)
-	if csvErr == nil {
-		return images, nil
-	}
-	return nil, errors.Join(jsonErr, csvErr)
+	return GetImageListFromJSON(f)
 
 }
 
@@ -130,29 +120,6 @@ func GetImageListFromJSON(r io.Reader) ([]ImageDescriptor, error) {
 	return images, nil
 }
 
-func GetImageListFromCsv(r io.Reader) ([]ImageDescriptor, error) {
-	csv, err := csv.NewReader(r).ReadAll()
-	if err != nil {
-		return nil, err
-	}
-	var images []ImageDescriptor
-	for _, image := range csv {
-		if len(image) < 3 {
-			return nil, errors.New("image input is not sufficient")
-		}
-		var sociIndexManifestRef string
-		if len(image) == 4 {
-			sociIndexManifestRef = image[2]
-		}
-		images = append(images, ImageDescriptor{
-			ShortName:       image[0],
-			ImageRef:        image[1],
-			ReadyLine:       image[3],
-			SociIndexDigest: sociIndexManifestRef})
-	}
-	return images, nil
-}
-
 func GetCommitHash() (string, error) {
 	cmd := exec.Command("git", "rev-parse", "HEAD")
 	output, err := cmd.Output()

docs/benchmark.md

@@ -12,7 +12,7 @@ There are two types of benchmarks:
 - [Running benchmarks on default workloads](#running-benchmarks-on-default-workloads)
 - [Running benchmarks on custom workloads](#running-benchmarks-on-custom-workloads)
 - [Benchmark binaries cli flags](#benchmark-binaries-have-different-cli-flags)
-- [Csv file format for custom workloads](#csv-file-format-for-custom-workloads)
+- [JSON file format for custom workloads](#json-file-format-for-custom-workloads)
 - [Default workloads](#default-workloads)
 - [Benchmark results format](#benchmark-results)
 
@@ -36,31 +36,41 @@ Custom workloads can also be benchmarked with SOCI.
 
 In order to run the benchmarks on custom workloads the custom container image needs to have its soci indices generated and pushed to a contianer registry as described in the [Getting started docs](/docs/getting-started.md)
 
-Generate benchmark binaries: 
+Generate benchmark binaries:
 ``` make build-benchmarks``` will generate benchmark binaries for performance testing and comparison testing against overlayFS. The binaries will be available in the ```/benchmark/bin``` folder.
- 
-### Benchmark binaries have different cli flags: 
+
+### Benchmark binaries have different cli flags:
 
 | Flag | Description | Required / Optional |
 |----------|----------|----------|
-| ```-f``` | File path to a csv file containing details of multiple images to be tested | Optional
+| ```-f``` | File path to a json file containing details of multiple images to be tested | Optional
 | ```-count``` | Specify number of times the benchmarker needs to run  | Optional
 | ```-show-commit``` | Tag the latest commit hash to the benchmark results | Optional
 
-We can now run benchmarks on custom workloads using the ```-f``` flag to specify the file path to a csv file containing details of the workloads.
+We can now run benchmarks on custom workloads using the ```-f``` flag to specify the file path to a json file containing details of the workloads.
 
-### Csv file format for custom workloads
+### JSON file format for custom workloads
 
 Ensure that the file being used with the ```-f``` flag follows the following format
 
-```shell
-<Test_name>, <Container_Image_ref>, <Ready_line>, <Soci_Index_Manifest_Digest>
+```json
+{
+    "short_name": "<Test_name>",
+    "image_ref": "<Container_Image_ref>",
+    "ready_line": "<Ready_line>",
+    "soci_index_digest": "<Soci_Index_Manifest_Digest>"
+}
 ```
 
 Example :
 
-```shell
-ffmpeg, public.ecr.aws/soci-workshop-examples/ffmpeg:latest, "Hello World",ef63578971ebd8fc700c74c96f81dafab4f3875e9117ef3c5eb7446e169d91cb
+```json
+{
+    "short_name": "ffmpeg",
+    "image_ref": "public.ecr.aws/soci-workshop-examples/ffmpeg:latest",
+    "ready_line": "Hello World",
+    "soci_index_digest": "ef63578971ebd8fc700c74c96f81dafab4f3875e9117ef3c5eb7446e169d91cb"
+}
 ```
 
 ### Default workloads