Undo cmd folder restructuring

Author: ethanyzhang

cmd/cmp.go

@@ -0,0 +1,22 @@
+//go:build experimental
+
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+	"pbench/cmd/cmp"
+)
+
+var cmpCmd = &cobra.Command{
+	Use:                   `cmp [flags] [directory 1] [directory 2]`,
+	DisableFlagsInUseLine: true,
+	Run:                   cmp.Run,
+	Args:                  cobra.ExactArgs(2),
+	Short:                 "Compare two query result directories",
+}
+
+func init() {
+	RootCmd.AddCommand(cmpCmd)
+	cmpCmd.Flags().StringVarP(&cmp.FileIdRegexStr, "file-id-regex", "r", `.*(query_\d{2}).*\.output`, "regex to extract file id from file names in two directories to find matching files to compare")
+	cmpCmd.Flags().StringVarP(&cmp.OutputPath, "output-path", "o", "./diff", "diff output path")
+}

cmd/cmp/cmp.go

@@ -1,28 +1,28 @@
-package genconfig
+package cmd
 
 import (
 	"github.com/spf13/cobra"
-	"pbench/cmd"
+	"pbench/cmd/genconfig"
 )
 
 var genConfigCmd = &cobra.Command{
 	Use:                   `genconfig [flags] [directory to search recursively for config.json]`,
 	DisableFlagsInUseLine: true,
-	Run:                   Run,
+	Run:                   genconfig.Run,
 	Args:                  cobra.ExactArgs(1),
 	Short:                 "Generate benchmark cluster configurations",
 }
 
 func init() {
-	cmd.RootCmd.AddCommand(genConfigCmd)
-	genConfigCmd.Flags().StringVarP(&TemplatePath, "template-dir",
+	RootCmd.AddCommand(genConfigCmd)
+	genConfigCmd.Flags().StringVarP(&genconfig.TemplatePath, "template-dir",
 		"t", "", "Specifies the template directory. Use built-in template if not specified.")
-	genConfigCmd.Flags().StringVarP(&ParameterPath, "parameter-file",
+	genConfigCmd.Flags().StringVarP(&genconfig.ParameterPath, "parameter-file",
 		"p", "", "Specifies the parameter file. Use built-in defaults if not specified.")
 	genConfigCmd.AddCommand(&cobra.Command{
 		Use:                   "default",
 		Short:                 "Print the built-in default generator parameter file.",
 		DisableFlagsInUseLine: true,
-		Run:                   PrintDefaultParams,
+		Run:                   genconfig.PrintDefaultParams,
 	})
 }

cmd/genconfig/genconfig.go renamed to cmd/genconfig.go

@@ -0,0 +1,43 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+	"os"
+	"pbench/cmd/loadjson"
+	"pbench/utils"
+	"runtime"
+	"time"
+)
+
+// loadJsonCmd represents the loadjson command
+var loadJsonCmd = &cobra.Command{
+	Use:                   `loadjson [flags] [list of files or directories to process]`,
+	Short:                 "Load query JSON files into event listener database and run recorders",
+	Long:                  `Load query JSON files into event listener database and run recorders`,
+	DisableFlagsInUseLine: true,
+	Args: func(cmd *cobra.Command, args []string) error {
+		if len(args) < 1 {
+			return fmt.Errorf("requires at least 1 arg, only received %d", len(args))
+		}
+		if loadjson.Parallelism < 1 || loadjson.Parallelism > runtime.NumCPU() {
+			return fmt.Errorf("invalid parallelism: %d, it should be >= 1 and <= %d", loadjson.Parallelism, runtime.NumCPU())
+		}
+		utils.ExpandHomeDirectory(&loadjson.OutputPath)
+		return nil
+	},
+	Run: loadjson.Run,
+}
+
+func init() {
+	RootCmd.AddCommand(loadJsonCmd)
+	wd, _ := os.Getwd()
+	loadJsonCmd.Flags().StringVarP(&loadjson.RunName, "name", "n", fmt.Sprintf("load_%s", time.Now().Format(utils.DirectoryNameTimeFormat)), `Assign a name to this run. (default: "load_<current time>")`)
+	loadJsonCmd.Flags().StringVarP(&loadjson.Comment, "comment", "c", "", `Add a comment to this run (optional)`)
+	loadJsonCmd.Flags().BoolVarP(&loadjson.RecordRun, "record-run", "r", false, "Record all the loaded JSON as a run")
+	loadJsonCmd.Flags().StringVarP(&loadjson.OutputPath, "output-path", "o", wd, "Output directory path")
+	loadJsonCmd.Flags().BoolVarP(&loadjson.ExtractPlanJson, "extract-plan", "x", false, "Extract the plan JSON from query JSON then save them to the output path")
+	loadJsonCmd.Flags().IntVarP(&loadjson.Parallelism, "parallel", "P", runtime.NumCPU(), "Number of parallel threads to load json files")
+	loadJsonCmd.Flags().StringVar(&loadjson.InfluxCfgPath, "influx", "", "InfluxDB connection config for run recorder (optional)")
+	loadJsonCmd.Flags().StringVar(&loadjson.MySQLCfgPath, "mysql", "", "MySQL connection config for event listener and run recorder (optional)")
+}

cmd/loadjson.go

@@ -1,10 +1,10 @@
-package replay
+package cmd
 
 import (
 	"fmt"
 	"github.com/spf13/cobra"
 	"os"
-	"pbench/cmd"
+	"pbench/cmd/replay"
 	"pbench/utils"
 	"time"
 )
@@ -22,17 +22,17 @@ We also expect the queries in this CSV file are sorted by "create_time" in ascen
 		if len(args) != 1 {
 			return fmt.Errorf("requires 1 arg, only received %d", len(args))
 		}
-		utils.ExpandHomeDirectory(&OutputPath)
+		utils.ExpandHomeDirectory(&replay.OutputPath)
 		utils.ExpandHomeDirectory(&args[0])
 		return nil
 	},
-	Run: Run,
+	Run: replay.Run,
 }
 
 func init() {
-	cmd.RootCmd.AddCommand(replayCmd)
+	RootCmd.AddCommand(replayCmd)
 	wd, _ := os.Getwd()
-	PrestoFlags.InstallPrestoFlags(replayCmd)
-	replayCmd.Flags().StringVarP(&OutputPath, "output-path", "o", wd, "Output directory path")
-	replayCmd.Flags().StringVarP(&RunName, "name", "n", fmt.Sprintf("replay_%s", time.Now().Format(utils.DirectoryNameTimeFormat)), `Assign a name to this run. (default: "replay_<current time>")`)
+	replay.PrestoFlags.Install(replayCmd)
+	replayCmd.Flags().StringVarP(&replay.OutputPath, "output-path", "o", wd, "Output directory path")
+	replayCmd.Flags().StringVarP(&replay.RunName, "name", "n", fmt.Sprintf("replay_%s", time.Now().Format(utils.DirectoryNameTimeFormat)), `Assign a name to this run. (default: "replay_<current time>")`)
 }

cmd/loadjson/loadjson.go

@@ -1,10 +1,10 @@
 //go:build experimental
 
-package round
+package cmd
 
 import (
 	"github.com/spf13/cobra"
-	"pbench/cmd"
+	"pbench/cmd/round"
 )
 
 // roundCmd represents the round command
@@ -16,19 +16,19 @@ After processing the first row, it will only look at the matched columns. So if
 A PR was opened to fix the native/Java decimal precision discrepancy but so far it does not work quite well:
 https://github.com/facebookincubator/velox/pull/7944`,
 	DisableFlagsInUseLine: true,
-	Args:                  Args,
-	Run:                   Run,
+	Args:                  round.Args,
+	Run:                   round.Run,
 }
 
 func init() {
-	cmd.RootCmd.AddCommand(roundCmd)
-	roundCmd.Flags().IntVarP(&DecimalPrecision, "precision", "p", 12, "Decimal precision to preserve.")
-	roundCmd.Flags().StringArrayVarP(&FileExtensions, "file-extension", "e", []string{".output"},
+	RootCmd.AddCommand(roundCmd)
+	roundCmd.Flags().IntVarP(&round.DecimalPrecision, "precision", "p", 12, "Decimal precision to preserve.")
+	roundCmd.Flags().StringArrayVarP(&round.FileExtensions, "file-extension", "e", []string{".output"},
 		"Specifies the file extensions ton include for processing (including the dot). You can specify multiple file extensions.")
-	roundCmd.Flags().StringVarP(&FileFormat, "format", "f", "json",
+	roundCmd.Flags().StringVarP(&round.FileFormat, "format", "f", "json",
 		`Specifies the format of the files. Accepted values are: "csv"" or "json" which is the output file from the "run"" command`)
-	roundCmd.Flags().BoolVarP(&InPlaceRewrite, "rewrite-in-place", "i", false,
+	roundCmd.Flags().BoolVarP(&round.InPlaceRewrite, "rewrite-in-place", "i", false,
 		"When turned on, we will rewrite the file in-place. Otherwise, we save the rewritten file separately.")
-	roundCmd.Flags().BoolVarP(&Recursive, "recursive", "r", false,
+	roundCmd.Flags().BoolVarP(&round.Recursive, "recursive", "r", false,
 		`Recursively walk a path if a directory is provided in the arguments.`)
 }

cmd/replay/replay.go renamed to cmd/replay.go

@@ -0,0 +1,32 @@
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+	"os"
+	"pbench/cmd/run"
+	"time"
+)
+
+// runCmd represents the run command
+var runCmd = &cobra.Command{
+	Use:                   `run [flags] [list of root-level benchmark stage JSON files]`,
+	Short:                 "Run a benchmark",
+	Long:                  `Run a benchmark that is defined by a sequence of JSON configuration files.`,
+	DisableFlagsInUseLine: true,
+	Args:                  cobra.MinimumNArgs(1),
+	Run:                   run.Run,
+}
+
+func init() {
+	RootCmd.AddCommand(runCmd)
+	runCmd.Flags().StringVarP(&run.Name, "name", "n", "", `Assign a name to this run. (default: "<main stage name>-<current time>")`)
+	runCmd.Flags().StringVarP(&run.Comment, "comment", "c", "", `Add a comment to this run (optional)`)
+	run.PrestoFlags.Install(runCmd)
+	wd, _ := os.Getwd()
+	runCmd.Flags().StringVarP(&run.OutputPath, "output-path", "o", wd, "Output directory path")
+	runCmd.Flags().Int64VarP(&run.RandSeed, "seed", "e", time.Now().UnixMicro(), "Random seed for randomized execution")
+	runCmd.Flags().IntVarP(&run.RandSkip, "rand-skip", "k", 0, "Skip the first N random selections from the sequence (optional)")
+	runCmd.Flags().StringVar(&run.InfluxCfgPath, "influx", "", "InfluxDB connection config for run recorder (optional)")
+	runCmd.Flags().StringVar(&run.MySQLCfgPath, "mysql", "", "MySQL connection config for run recorder (optional)")
+	runCmd.Flags().StringVar(&run.PulumiCfgPath, "pulumi", "", "(only works when a MySQL run recorder is specified) Pulumi API config for storing deployment details with MySQL (optional)")
+}

cmd/round/round.go renamed to cmd/round.go

@@ -0,0 +1,41 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+	"os"
+	"pbench/cmd/save"
+	"pbench/utils"
+	"runtime"
+)
+
+// saveCmd represents the save command
+var saveCmd = &cobra.Command{
+	Use:                   `save [flags] [list of table names]`,
+	Short:                 "Save table information for recreating the schema and data",
+	Long:                  `Save table information for recreating the schema and data`,
+	DisableFlagsInUseLine: true,
+	Args: func(cmd *cobra.Command, args []string) error {
+		utils.ExpandHomeDirectory(&save.OutputPath)
+		if save.InputFilePath != "" {
+			utils.ExpandHomeDirectory(&save.InputFilePath)
+		} else if len(args) < 1 {
+			return fmt.Errorf("requires at least 1 arg when -f is not used")
+		}
+		return nil
+	},
+	Run: save.Run,
+}
+
+func init() {
+	RootCmd.AddCommand(saveCmd)
+	save.PrestoFlags.Install(saveCmd)
+	wd, _ := os.Getwd()
+	saveCmd.Flags().StringVarP(&save.OutputPath, "output-path", "o", wd, "Output directory path")
+	saveCmd.Flags().StringVarP(&save.Catalog, "catalog", "", "", "Catalog name")
+	saveCmd.Flags().StringVarP(&save.Schema, "schema", "", "", "Schema name")
+	saveCmd.Flags().StringArrayVarP(&save.Session, "session", "", nil,
+		"Session property (property can be used multiple times; format is\nkey=value; use 'SHOW SESSION' in Presto CLI to see available properties)")
+	saveCmd.Flags().StringVarP(&save.InputFilePath, "file", "f", "", "CSV file to read catalog,schema,table.")
+	saveCmd.Flags().IntVarP(&save.Parallelism, "parallel", "P", runtime.NumCPU(), "Number of parallel threads to save table summaries.")
+}