Benchmark rally add flags (#1576)

* dd package-from-registry and use-corpus-at-path flags

* add extra info in docs

* minor rewording

* Update docs/howto/rally_benchmarking.md

Co-authored-by: Jaime Soriano Pastor <[email protected]>

* Update internal/benchrunner/runners/rally/runner.go

Co-authored-by: Jaime Soriano Pastor <[email protected]>

* cr fixes

* cr fixes

* cr fixes

* comment on countLine code

* refactor getPackageNameAndVersion

---------

Co-authored-by: Jaime Soriano Pastor <[email protected]>

Author: Andrea Spacca

cmd/benchmark.go

@@ -228,6 +228,8 @@ func getRallyCommand() *cobra.Command {
 	cmd.Flags().String(cobraext.VariantFlagName, "", cobraext.VariantFlagDescription)
 	cmd.Flags().StringP(cobraext.BenchCorpusRallyTrackOutputDirFlagName, "", "", cobraext.BenchCorpusRallyTrackOutputDirFlagDescription)
 	cmd.Flags().BoolP(cobraext.BenchCorpusRallyDryRunFlagName, "", false, cobraext.BenchCorpusRallyDryRunFlagDescription)
+	cmd.Flags().StringP(cobraext.BenchCorpusRallyUseCorpusAtPathFlagName, "", "", cobraext.BenchCorpusRallyUseCorpusAtPathFlagDescription)
+	cmd.Flags().StringP(cobraext.BenchCorpusRallyPackageFromRegistryFlagName, "", "", cobraext.BenchCorpusRallyPackageFromRegistryFlagDescription)
 
 	return cmd
 }
@@ -260,12 +262,31 @@ func rallyCommandAction(cmd *cobra.Command, args []string) error {
 		return cobraext.FlagParsingError(err, cobraext.BenchCorpusRallyDryRunFlagName)
 	}
 
-	packageRootPath, found, err := packages.FindPackageRoot()
-	if !found {
-		return errors.New("package root not found")
+	corpusAtPath, err := cmd.Flags().GetString(cobraext.BenchCorpusRallyUseCorpusAtPathFlagName)
+	if err != nil {
+		return cobraext.FlagParsingError(err, cobraext.BenchCorpusRallyUseCorpusAtPathFlagName)
 	}
+
+	packageFromRegistry, err := cmd.Flags().GetString(cobraext.BenchCorpusRallyPackageFromRegistryFlagName)
 	if err != nil {
-		return fmt.Errorf("locating package root failed: %w", err)
+		return cobraext.FlagParsingError(err, cobraext.BenchCorpusRallyPackageFromRegistryFlagName)
+	}
+
+	packageName, packageVersion, err := getPackageNameAndVersion(packageFromRegistry)
+	if err != nil {
+		return fmt.Errorf("getting package name and version failed, expected format: <package>-<version>: %w", err)
+	}
+
+	var packageRootPath string
+	var found bool
+	if len(packageName) == 0 {
+		packageRootPath, found, err = packages.FindPackageRoot()
+		if !found {
+			return errors.New("package root not found")
+		}
+		if err != nil {
+			return fmt.Errorf("locating package root failed: %w", err)
+		}
 	}
 
 	profile, err := cobraext.GetProfileFlag(cmd)
@@ -299,6 +320,8 @@ func rallyCommandAction(cmd *cobra.Command, args []string) error {
 		rally.WithProfile(profile),
 		rally.WithRallyTrackOutputDir(rallyTrackOutputDir),
 		rally.WithRallyDryRun(rallyDryRun),
+		rally.WithRallyPackageFromRegistry(packageName, packageVersion),
+		rally.WithRallyCorpusAtPath(corpusAtPath),
 	}
 
 	esMetricsClient, err := initializeESMetricsClient(cmd.Context())
@@ -345,6 +368,23 @@ func rallyCommandAction(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
+func getPackageNameAndVersion(packageFromRegistry string) (string, string, error) {
+	packageData := strings.SplitN(packageFromRegistry, "-", 2)
+
+	if len(packageData) != 2 {
+		return "", "", fmt.Errorf("package name and version from registry not valid (%s)", packageFromRegistry)
+	}
+
+	packageName := packageData[0]
+	packageVersion := packageData[1]
+
+	if len(packageName) > 0 && len(packageVersion) == 0 {
+		return "", "", fmt.Errorf("package name and version from registry not valid (%s)", packageFromRegistry)
+	}
+
+	return packageName, packageVersion, nil
+}
+
 func getSystemCommand() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "system",

docs/howto/rally_benchmarking.md

@@ -279,3 +279,15 @@ esrally --target-hosts='{"defauelt":["%es_cluster_host:es_cluster_port%"]}' --tr
 
 Please refer to [esrally CLI reference](https://esrally.readthedocs.io/en/stable/command_line_reference.html) for more details.
 
+## Replaying an existing corpus
+If the `use-corpus-at-path` flag is used, the corpus will not be generated but rather loaded from the file passed as value of the flag.
+If the `rally-track-output-dir` flag is used alongside the `use-corpus-at-path` flag, the persisted rally track will reference a copy of the existing corpus saved in the directory passed as value of the `rally-track-output-dir` flag.
+
+## Loading a package from registry
+If the `package-from-registry` flag is used, the package installed in Kibana and its assets referenced in the track will be loaded from registry. The format of the flag value is `%packageName%-%packageVersion%`.
+You still need to be in the root of the local package when running the command in order to read the benchmark scenario (`benchmark` flag).
+
+## Running benchmark with the same corpus against different versions of a package
+If you use both the `use-corpus-at-path` flag and the `package-from-registry` flag multiple times, you can run benchmark with the same corpus against different versions of the same package, just updating the package version in the relevant flag between executions.  
+If you add the `rally-track-output-dir` flag and the `dry-run` flag, you can persist multiple rally tracks with the same corpus for different package versions, using different values for the `rally-track-output-dir` flag. 
+

internal/benchrunner/runners/rally/options.go

@@ -26,6 +26,9 @@ type Options struct {
 	Profile             *profile.Profile
 	RallyTrackOutputDir string
 	DryRun              bool
+	PackageName         string
+	PackageVersion      string
+	CorpusAtPath        string
 }
 
 type ClientOptions struct {
@@ -102,3 +105,16 @@ func WithRallyDryRun(d bool) OptionFunc {
 		opts.DryRun = d
 	}
 }
+
+func WithRallyPackageFromRegistry(n, v string) OptionFunc {
+	return func(opts *Options) {
+		opts.PackageName = n
+		opts.PackageVersion = v
+	}
+}
+
+func WithRallyCorpusAtPath(c string) OptionFunc {
+	return func(opts *Options) {
+		opts.CorpusAtPath = c
+	}
+}