Apps run config template parsing (#1152)

* initial commit for run template parsing

Signed-off-by: Pravin Pushkar <[email protected]>

* some code refactor.

Signed-off-by: Pravin Pushkar <[email protected]>

* add some UTs

Signed-off-by: Pravin Pushkar <[email protected]>

* call validate

Signed-off-by: Pravin Pushkar <[email protected]>

* add tests and correct err message

Signed-off-by: Pravin Pushkar <[email protected]>

* refactor Runconfig into 2 structs

Signed-off-by: Pravin Pushkar <[email protected]>

* Fix Arg parsing

Signed-off-by: Pravin Pushkar <[email protected]>

* fix UTs. make app-id be inferred from app-dir if empty

Signed-off-by: Pravin Pushkar <[email protected]>

* fix tests

Signed-off-by: Pravin Pushkar <[email protected]>

* put config reltd code into separate folder

Signed-off-by: Pravin Pushkar <[email protected]>

* merge logic for common and apps section of flags.

Signed-off-by: Pravin Pushkar <[email protected]>

* adding tests and few refactoring

Signed-off-by: Pravin Pushkar <[email protected]>

* fix tests

Signed-off-by: Pravin Pushkar <[email protected]>

* file close

Signed-off-by: Pravin Pushkar <[email protected]>

* review comments

Signed-off-by: Pravin Pushkar <[email protected]>

* fix review comments

Signed-off-by: Pravin Pushkar <[email protected]>

* refactor file names

Signed-off-by: Pravin Pushkar <[email protected]>

* resolve path to abs and add more e2e for path resolving

Signed-off-by: Pravin Pushkar <[email protected]>

* address review comments

Signed-off-by: Pravin Pushkar <[email protected]>

* remove getAppskeysmapping

Signed-off-by: Pravin Pushkar <[email protected]>

* add tests, and fix issue for setitng app id when empty.

Signed-off-by: Pravin Pushkar <[email protected]>

* fix tests

Signed-off-by: Pravin Pushkar <[email protected]>

Signed-off-by: Pravin Pushkar <[email protected]>
Co-authored-by: Mukundan Sundararajan <[email protected]>

Author: pravinpushkar

cmd/run.go

@@ -28,6 +28,7 @@ import (
 	"github.com/dapr/cli/pkg/metadata"
 	"github.com/dapr/cli/pkg/print"
 	"github.com/dapr/cli/pkg/standalone"
+	"github.com/dapr/cli/pkg/standalone/runfileconfig"
 	"github.com/dapr/cli/utils"
 )
 
@@ -57,6 +58,7 @@ var (
 	appHealthThreshold int
 	enableAPILogging   bool
 	apiListenAddresses string
+	runFilePath        string
 )
 
 const (
@@ -93,6 +95,10 @@ dapr run --app-id myapp --dapr-path /usr/local/dapr
 		viper.BindPFlag("placement-host-address", cmd.Flags().Lookup("placement-host-address"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
+		if len(runFilePath) > 0 {
+			executeRunWithAppsConfigFile(runFilePath)
+			return
+		}
 		if len(args) == 0 {
 			fmt.Println(print.WhiteBold("WARNING: no application command found."))
 		}
@@ -126,35 +132,38 @@ dapr run --app-id myapp --dapr-path /usr/local/dapr
 			}
 		}
 
-		output, err := standalone.Run(&standalone.RunConfig{
-			AppID:              appID,
-			AppPort:            appPort,
-			HTTPPort:           port,
-			GRPCPort:           grpcPort,
+		sharedRunConfig := &standalone.SharedRunConfig{
 			ConfigFile:         configFile,
-			Arguments:          args,
 			EnableProfiling:    enableProfiling,
-			ProfilePort:        profilePort,
 			LogLevel:           logLevel,
 			MaxConcurrency:     maxConcurrency,
-			Protocol:           protocol,
+			AppProtocol:        protocol,
 			PlacementHostAddr:  viper.GetString("placement-host-address"),
 			ComponentsPath:     componentsPath,
 			ResourcesPath:      resourcesPath,
 			AppSSL:             appSSL,
-			MetricsPort:        metricsPort,
 			MaxRequestBodySize: maxRequestBodySize,
 			HTTPReadBufferSize: readBufferSize,
-			UnixDomainSocket:   unixDomainSocket,
 			EnableAppHealth:    enableAppHealth,
 			AppHealthPath:      appHealthPath,
 			AppHealthInterval:  appHealthInterval,
 			AppHealthTimeout:   appHealthTimeout,
 			AppHealthThreshold: appHealthThreshold,
 			EnableAPILogging:   enableAPILogging,
-			InternalGRPCPort:   internalGRPCPort,
-			DaprPathCmdFlag:    daprPath,
 			APIListenAddresses: apiListenAddresses,
+		}
+		output, err := standalone.Run(&standalone.RunConfig{
+			AppID:            appID,
+			AppPort:          appPort,
+			HTTPPort:         port,
+			GRPCPort:         grpcPort,
+			ProfilePort:      profilePort,
+			Command:          args,
+			MetricsPort:      metricsPort,
+			UnixDomainSocket: unixDomainSocket,
+			InternalGRPCPort: internalGRPCPort,
+			DaprPathCmdFlag:  daprPath,
+			SharedRunConfig:  *sharedRunConfig,
 		})
 		if err != nil {
 			print.FailureStatusEvent(os.Stderr, err.Error())
@@ -416,5 +425,19 @@ func init() {
 	RunCmd.Flags().IntVar(&appHealthThreshold, "app-health-threshold", 0, "Number of consecutive failures for the app to be considered unhealthy")
 	RunCmd.Flags().BoolVar(&enableAPILogging, "enable-api-logging", false, "Log API calls at INFO verbosity. Valid values are: true or false")
 	RunCmd.Flags().StringVar(&apiListenAddresses, "dapr-listen-addresses", "", "Comma separated list of IP addresses that sidecar will listen to")
+	RunCmd.Flags().StringVarP(&runFilePath, "run-file", "f", "", "Path to the configuration file for the apps to run")
 	RootCmd.AddCommand(RunCmd)
 }
+
+func executeRunWithAppsConfigFile(runFilePath string) {
+	config := runfileconfig.RunFileConfig{}
+	apps, err := config.GetApps(runFilePath)
+	if err != nil {
+		print.FailureStatusEvent(os.Stdout, fmt.Sprintf("Error getting apps from config file: %s", err))
+		os.Exit(1)
+	}
+	if len(apps) == 0 {
+		print.FailureStatusEvent(os.Stdout, "No apps to run")
+		os.Exit(1)
+	}
+}

pkg/standalone/run.go

@@ -35,34 +35,39 @@ const sentryDefaultAddress = "localhost:50001"
 
 // RunConfig represents the application configuration parameters.
 type RunConfig struct {
-	AppID              string `env:"APP_ID" arg:"app-id"`
-	AppPort            int    `env:"APP_PORT" arg:"app-port"`
-	HTTPPort           int    `env:"DAPR_HTTP_PORT" arg:"dapr-http-port"`
-	GRPCPort           int    `env:"DAPR_GRPC_PORT" arg:"dapr-grpc-port"`
-	ConfigFile         string `arg:"config"`
-	Protocol           string `arg:"app-protocol"`
-	Arguments          []string
-	APIListenAddresses string `arg:"dapr-listen-addresses"`
-	EnableProfiling    bool   `arg:"enable-profiling"`
-	ProfilePort        int    `arg:"profile-port"`
-	LogLevel           string `arg:"log-level"`
-	MaxConcurrency     int    `arg:"app-max-concurrency"`
-	PlacementHostAddr  string `arg:"placement-host-address"`
+	AppID            string   `env:"APP_ID" arg:"app-id" yaml:"app_id"`
+	AppPort          int      `env:"APP_PORT" arg:"app-port" yaml:"app_port"`
+	HTTPPort         int      `env:"DAPR_HTTP_PORT" arg:"dapr-http-port" yaml:"dapr_http_port"`
+	GRPCPort         int      `env:"DAPR_GRPC_PORT" arg:"dapr-grpc-port" yaml:"dapr_grpc_port"`
+	ProfilePort      int      `arg:"profile-port" yaml:"profile_port"`
+	Command          []string `yaml:"command"`
+	MetricsPort      int      `env:"DAPR_METRICS_PORT" arg:"metrics-port" yaml:"metrics_port"`
+	UnixDomainSocket string   `arg:"unix-domain-socket" yaml:"unix_domain_socket"`
+	InternalGRPCPort int      `arg:"dapr-internal-grpc-port" yaml:"dapr_internal_grpc_port"`
+	DaprPathCmdFlag  string   `yaml:"dapr_path_cmd_flag"`
+	SharedRunConfig  `yaml:",inline"`
+}
+
+// SharedRunConfig represents the application configuration parameters, which can be shared across many apps.
+type SharedRunConfig struct {
+	ConfigFile         string `arg:"config" yaml:"config_file"`
+	AppProtocol        string `arg:"app-protocol" yaml:"app_protocol"`
+	APIListenAddresses string `arg:"dapr-listen-addresses" yaml:"api_listen_addresses"`
+	EnableProfiling    bool   `arg:"enable-profiling" yaml:"enable_profiling"`
+	LogLevel           string `arg:"log-level" yaml:"log_level"`
+	MaxConcurrency     int    `arg:"app-max-concurrency" yaml:"_appmax_concurrency"`
+	PlacementHostAddr  string `arg:"placement-host-address" yaml:"placement_host_address"`
 	ComponentsPath     string `arg:"components-path"`
-	ResourcesPath      string `arg:"resources-path"`
-	AppSSL             bool   `arg:"app-ssl"`
-	MetricsPort        int    `env:"DAPR_METRICS_PORT" arg:"metrics-port"`
-	MaxRequestBodySize int    `arg:"dapr-http-max-request-size"`
-	HTTPReadBufferSize int    `arg:"dapr-http-read-buffer-size"`
-	UnixDomainSocket   string `arg:"unix-domain-socket"`
-	InternalGRPCPort   int    `arg:"dapr-internal-grpc-port"`
-	EnableAppHealth    bool   `arg:"enable-app-health-check"`
-	AppHealthPath      string `arg:"app-health-check-path"`
-	AppHealthInterval  int    `arg:"app-health-probe-interval" ifneq:"0"`
-	AppHealthTimeout   int    `arg:"app-health-probe-timeout" ifneq:"0"`
-	AppHealthThreshold int    `arg:"app-health-threshold" ifneq:"0"`
-	EnableAPILogging   bool   `arg:"enable-api-logging"`
-	DaprPathCmdFlag    string
+	ResourcesPath      string `arg:"resources-path" yaml:"resources_path"`
+	AppSSL             bool   `arg:"app-ssl" yaml:"app_ssl"`
+	MaxRequestBodySize int    `arg:"dapr-http-max-request-size" yaml:"dapr_http_max_request_size"`
+	HTTPReadBufferSize int    `arg:"dapr-http-read-buffer-size" yaml:"dapr_http_read_buffer_size"`
+	EnableAppHealth    bool   `arg:"enable-app-health-check" yaml:"enable_app_health_check"`
+	AppHealthPath      string `arg:"app-health-check-path" yaml:"app_health_check_path"`
+	AppHealthInterval  int    `arg:"app-health-probe-interval" ifneq:"0" yaml:"app_health_probe_interval"`
+	AppHealthTimeout   int    `arg:"app-health-probe-timeout" ifneq:"0" yaml:"app_health_probe_timeout"`
+	AppHealthThreshold int    `arg:"app-health-threshold" ifneq:"0" yaml:"app_health_threshold"`
+	EnableAPILogging   bool   `arg:"enable-api-logging" yaml:"enable_api_logging"`
 }
 
 func (meta *DaprMeta) newAppID() string {
@@ -239,10 +244,33 @@ func (config *RunConfig) getArgs() []string {
 	args := []string{}
 
 	schema := reflect.ValueOf(*config)
+	args = getArgsFromSchema(schema, args)
+
+	if config.ConfigFile != "" {
+		sentryAddress := mtlsEndpoint(config.ConfigFile)
+		if sentryAddress != "" {
+			// mTLS is enabled locally, set it up.
+			args = append(args, "--enable-mtls", "--sentry-address", sentryAddress)
+		}
+	}
+
+	if print.IsJSONLogEnabled() {
+		args = append(args, "--log-as-json")
+	}
+	return args
+}
+
+// Recursive function to get all the args from the config struct.
+// This is needed because the config struct has embedded struct.
+func getArgsFromSchema(schema reflect.Value, args []string) []string {
 	for i := 0; i < schema.NumField(); i++ {
 		valueField := schema.Field(i).Interface()
 		typeField := schema.Type().Field(i)
 		key := typeField.Tag.Get("arg")
+		if typeField.Type.Kind() == reflect.Struct {
+			args = getArgsFromSchema(schema.Field(i), args)
+			continue
+		}
 		if len(key) == 0 {
 			continue
 		}
@@ -262,18 +290,6 @@ func (config *RunConfig) getArgs() []string {
 			}
 		}
 	}
-
-	if config.ConfigFile != "" {
-		sentryAddress := mtlsEndpoint(config.ConfigFile)
-		if sentryAddress != "" {
-			// mTLS is enabled locally, set it up.
-			args = append(args, "--enable-mtls", "--sentry-address", sentryAddress)
-		}
-	}
-
-	if print.IsJSONLogEnabled() {
-		args = append(args, "--log-as-json")
-	}
 	return args
 }
 
@@ -343,16 +359,16 @@ func mtlsEndpoint(configFile string) string {
 }
 
 func getAppCommand(config *RunConfig) *exec.Cmd {
-	argCount := len(config.Arguments)
+	argCount := len(config.Command)
 
 	if argCount == 0 {
 		return nil
 	}
-	command := config.Arguments[0]
+	command := config.Command[0]
 
 	args := []string{}
 	if argCount > 1 {
-		args = config.Arguments[1:]
+		args = config.Command[1:]
 	}
 
 	cmd := exec.Command(command, args...)

pkg/standalone/run_test.go

@@ -172,25 +172,28 @@ func TestRun(t *testing.T) {
 	componentsDir := GetDaprComponentsPath(myDaprPath)
 	configFile := GetDaprConfigPath(myDaprPath)
 
-	basicConfig := &RunConfig{
-		AppID:              "MyID",
-		AppPort:            3000,
-		HTTPPort:           8000,
-		GRPCPort:           50001,
+	sharedRunConfig := &SharedRunConfig{
 		LogLevel:           "WARN",
-		Arguments:          []string{"MyCommand", "--my-arg"},
 		EnableProfiling:    false,
-		ProfilePort:        9090,
-		Protocol:           "http",
+		AppProtocol:        "http",
 		ComponentsPath:     componentsDir,
 		AppSSL:             true,
-		MetricsPort:        9001,
 		MaxRequestBodySize: -1,
-		InternalGRPCPort:   5050,
 		HTTPReadBufferSize: -1,
 		EnableAPILogging:   true,
 		APIListenAddresses: "127.0.0.1",
 	}
+	basicConfig := &RunConfig{
+		AppID:            "MyID",
+		AppPort:          3000,
+		HTTPPort:         8000,
+		GRPCPort:         50001,
+		Command:          []string{"MyCommand", "--my-arg"},
+		ProfilePort:      9090,
+		MetricsPort:      9001,
+		InternalGRPCPort: 5050,
+		SharedRunConfig:  *sharedRunConfig,
+	}
 
 	t.Run("run happy http", func(t *testing.T) {
 		output, err := Run(basicConfig)
@@ -203,7 +206,7 @@ func TestRun(t *testing.T) {
 	})
 
 	t.Run("run without app command", func(t *testing.T) {
-		basicConfig.Arguments = nil
+		basicConfig.Command = nil
 		basicConfig.LogLevel = "INFO"
 		basicConfig.EnableAPILogging = true
 		basicConfig.ConfigFile = configFile

pkg/standalone/runfileconfig/run_file_config.go

@@ -0,0 +1,43 @@
+/*
+Copyright 2022 The Dapr Authors
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package runfileconfig
+
+import "github.com/dapr/cli/pkg/standalone"
+
+// RunFileConfig represents the complete configuration options for the run file.
+// It is meant to be used with - "dapr run --run-file <path-to-run-file>" command.
+type RunFileConfig struct {
+	Common  Common `yaml:"common"`
+	Apps    []Apps `yaml:"apps"`
+	Version int    `yaml:"version"`
+}
+
+// Apps represents the configuration options for the apps in the run file.
+type Apps struct {
+	standalone.RunConfig `yaml:",inline"`
+	AppDirPath           string     `yaml:"app_dir_path"`
+	Env                  []EnvItems `yaml:"env"`
+}
+
+// Common represents the configuration options for the common section in the run file.
+type Common struct {
+	Env                        []EnvItems `yaml:"env"`
+	standalone.SharedRunConfig `yaml:",inline"`
+}
+
+// EnvItems represents the env configuration options that are present in commmon and/or individual app's section.
+type EnvItems struct {
+	Name  string `yaml:"name"`
+	Value string `yaml:"value"`
+}