--http-probe-cmd-upload http probe flag for uploads and the related --http-probe-cmd-file data format enhancements as well as api spec request data generation enhancements

Signed-off-by: Kyle Quest <[email protected]>

Author: kcq

README.md

@@ -430,8 +430,9 @@ In the interactive CLI prompt mode you must specify the target image using the `
 - `--poststart-compose-svc` - placeholder for now
 - `--http-probe` - Enables/disables HTTP probing (ENABLED by default; you have to disable the probe if you don't need it by setting the flag to `false`: `--http-probe=false`)
 - `--http-probe-off` - Alternative way to disable HTTP probing
-- `--http-probe-cmd` - Additional HTTP probe command [can use this flag multiple times]
+- `--http-probe-cmd` - User defined HTTP probe(s) as `[[[[\"crawl\":]PROTO:]METHOD:]PATH]` [can use this flag multiple times]
 - `--http-probe-cmd-file` - File with user defined HTTP probe commands
+- `--http-probe-cmd-upload` - User defined HTTP probe(s) to submit form data as `[[[[[PROTO:]FORM_FILE_NAME:]FORM_FIELD_NAME:]FILE_OR_GENERATE_TYPE:]PATH]` [can use this flag multiple times
 - `--http-probe-start-wait` - Number of seconds to wait before starting HTTP probing
 - `--http-probe-retry-count` - Number of retries for each HTTP probe (default value: 5)
 - `--http-probe-retry-wait` - Number of seconds to wait before retrying HTTP probe (doubles when target is not ready; default value: 8)
@@ -958,12 +959,14 @@ To configure Docker Desktop to create the default Unix socket open its UI and go
 
 ## HTTP PROBE COMMANDS
 
-If the HTTP probe is enabled (note: it is enabled by default) it will default to running `GET /` with HTTP and then HTTPS on every exposed port. You can add additional commands using the `--http-probe-cmd` and `--http-probe-cmd-file` options.
+If the HTTP probe is enabled (note: it is enabled by default) it will default to running `GET /` with HTTP and then HTTPS on every exposed port. You can add additional commands using the `--http-probe-cmd`, `--http-probe-cmd-upload` and `--http-probe-cmd-file` options.
 
 If you want to disable HTTP probing set the `--http-probe` flag to false (e.g., `--http-probe=false`). You can also use the `--http-probe-off` flag to do the same (simply use the flag without any parameters).
 
 The `--http-probe-cmd` option is good when you want to specify a small number of simple commands where you select some or all of these HTTP command options: crawling (defaults to false), protocol, method (defaults to GET), resource (path and query string).
 
+The `--http-probe-cmd-upload` option is good when you want to upload a file. The file upload is done using a multipart form data request (the most common way to upload files). The flag value has the following format: `[[[[[PROTO:]FORM_FILE_NAME:]FORM_FIELD_NAME:]FILE_OR_GENERATE_TYPE:]PATH]`. The most basic flag value is a `PATH` (e.g., `--http-probe-cmd-upload /uploadpath`). When no file to upload or data generate instruction is provided a simple text file is generated. The `FILE_OR_GENERATE_TYPE` section can either point to a local file path for the file to upload or it can be one of the data generate instructions: `generate.text`, `generate.text.json`, `generate.image`, `generate.image.png`, `generate.image.jpeg`, `generate.image.gif`. You can specify a custom form field name to use for the request using the `FORM_FIELD_NAME` section (defaults to `file`). You can also specify the file name for the upload using the `FORM_FILE_NAME` section (defaults to `file.data` or the base path of the file name when `FILE_OR_GENERATE_TYPE` points to a local file). The `PROTO` section is used to scope the request to a specific protocol (`http`, `https`, etc). Note that you can also specify an upload command using a JSON record provided with the `--http-probe-cmd-file` flag (see below).
+
 If you only want to use custom HTTP probe command and you don't want the default `GET /` command added to the command list you explicitly provided you'll need to set `--http-probe` to false when you specify your custom HTTP probe command. Note that this inconsistency will be addressed in the future releases to make it less confusing.
 
 Possible field combinations:
@@ -992,9 +995,14 @@ Available HTTP command options:
 * `headers` - array of strings with column delimited key/value pairs (e.g., "Content-Type: application/json")
 * `body` - request body as a string
 * `body_file` - request body loaded from the provided file
+* `body_generate` - auto-generate request body using one of the instructions: `generate.text`, `generate.text.json`, `generate.image`, `generate.image.png`, `generate.image.jpeg`, `generate.image.gif`
+* `body_is_form` - request body is a multipart form (request data from `body`, `body_file` or `body_generate` will be form encoded)
+* `form_field_name` - form field name to use (for form submissions)
+* `form_file_name` - form file name to use (for form submissions)
 * `username` - username to use for basic auth
 * `password` - password to use for basic auth
 * `crawl` - boolean to indicate if you want to crawl the target (to visit all referenced resources)
+* `fastcgi` - Fast CGI config params (todo: document sub-fields, see code for details)
 
 Here's a probe command file example:
 

pkg/app/master/command/build/prompt.go

@@ -50,6 +50,7 @@ var CommandFlagSuggestions = &command.FlagSuggestions{
 		{Text: command.FullFlagName(command.FlagHTTPProbeOff), Description: command.FlagHTTPProbeOffUsage},
 		{Text: command.FullFlagName(command.FlagHTTPProbe), Description: command.FlagHTTPProbeUsage},
 		{Text: command.FullFlagName(command.FlagHTTPProbeCmd), Description: command.FlagHTTPProbeCmdUsage},
+		{Text: command.FullFlagName(command.FlagHTTPProbeCmdUpload), Description: command.FlagHTTPProbeCmdUploadUsage},
 		{Text: command.FullFlagName(command.FlagHTTPProbeCmdFile), Description: command.FlagHTTPProbeCmdFileUsage},
 		{Text: command.FullFlagName(command.FlagHTTPProbeStartWait), Description: command.FlagHTTPProbeStartWaitUsage},
 		{Text: command.FullFlagName(command.FlagHTTPProbeRetryCount), Description: command.FlagHTTPProbeRetryCountUsage},

pkg/app/master/command/cliflags.go

@@ -111,6 +111,7 @@ const (
 	FlagHTTPProbeOff              = "http-probe-off" //alternative way to disable http probing
 	FlagHTTPProbeCmd              = "http-probe-cmd"
 	FlagHTTPProbeCmdFile          = "http-probe-cmd-file"
+	FlagHTTPProbeCmdUpload        = "http-probe-cmd-upload"
 	FlagHTTPProbeStartWait        = "http-probe-start-wait"
 	FlagHTTPProbeRetryCount       = "http-probe-retry-count"
 	FlagHTTPProbeRetryWait        = "http-probe-retry-wait"
@@ -223,6 +224,7 @@ const (
 	FlagHTTPProbeUsage                 = "Enable or disable HTTP probing"
 	FlagHTTPProbeOffUsage              = "Alternative way to disable HTTP probing"
 	FlagHTTPProbeCmdUsage              = "User defined HTTP probe(s) as [[[[\"crawl\":]PROTO:]METHOD:]PATH]"
+	FlagHTTPProbeCmdUploadUsage        = "User defined HTTP probe(s) to submit form data as [[[[[PROTO:]FORM_FILE_NAME:]FORM_FIELD_NAME:]FILE_OR_GENERATE_TYPE:]PATH]"
 	FlagHTTPProbeCmdFileUsage          = "File with user defined HTTP probes"
 	FlagHTTPProbeStartWaitUsage        = "Number of seconds to wait before starting HTTP probing"
 	FlagHTTPProbeRetryCountUsage       = "Number of retries for each HTTP probe"
@@ -610,6 +612,12 @@ var CommonFlags = map[string]cli.Flag{
 		Usage:   FlagHTTPProbeCmdUsage,
 		EnvVars: []string{"DSLIM_HTTP_PROBE_CMD"},
 	},
+	FlagHTTPProbeCmdUpload: &cli.StringSliceFlag{
+		Name:    FlagHTTPProbeCmdUpload,
+		Value:   cli.NewStringSlice(),
+		Usage:   FlagHTTPProbeCmdUploadUsage,
+		EnvVars: []string{"DSLIM_HTTP_PROBE_CMD_UPLOAD"},
+	},
 	FlagHTTPProbeCmdFile: &cli.StringFlag{
 		Name:    FlagHTTPProbeCmdFile,
 		Value:   "",
@@ -946,6 +954,7 @@ func HTTPProbeFlags() []cli.Flag {
 func HTTPProbeFlagsBasic() []cli.Flag {
 	return []cli.Flag{
 		Cflag(FlagHTTPProbeCmd),
+		Cflag(FlagHTTPProbeCmdUpload),
 		Cflag(FlagHTTPProbeCmdFile),
 		Cflag(FlagHTTPProbeStartWait),
 		Cflag(FlagHTTPProbeRetryCount),

pkg/app/master/command/clifvgetter.go

@@ -158,6 +158,15 @@ func GetHTTPProbes(ctx *cli.Context) ([]config.HTTPProbeCmd, error) {
 		return nil, err
 	}
 
+	httpProbeCmdUploads, err := ParseHTTPProbeUploads(ctx.StringSlice(FlagHTTPProbeCmdUpload))
+	if err != nil {
+		return nil, err
+	}
+
+	if httpProbeCmdUploads != nil {
+		httpProbeCmds = append(httpProbeCmds, httpProbeCmdUploads...)
+	}
+
 	moreHTTPProbeCmds, err := ParseHTTPProbesFile(ctx.String(FlagHTTPProbeCmdFile))
 	if err != nil {
 		return nil, err

pkg/app/master/command/clifvparser.go

@@ -647,6 +647,133 @@ func ParseHTTPProbes(values []string) ([]config.HTTPProbeCmd, error) {
 	return probes, nil
 }
 
+func ParseHTTPProbeUploads(values []string) ([]config.HTTPProbeCmd, error) {
+	// FORMAT:
+	// [[[[[PROTO:]FORM_FILE_NAME:]FORM_FIELD_NAME:]FILE_OR_GENERATE_TYPE:]PATH]
+	probes := []config.HTTPProbeCmd{}
+
+	method := "POST"
+	for _, raw := range values {
+		parts := strings.Split(raw, ":")
+
+		proto := "http"
+		resource := "/"
+		var formFieldName string //use the default from the form processor
+		var formFileName string  //use the default from the form processor
+		var bodyFile string
+		var bodyGenerate string
+
+		switch len(parts) {
+		case 1:
+			// parts[0] - PATH
+			if parts[0] == "" || !isResource(parts[0]) {
+				return nil, fmt.Errorf("invalid HTTP probe command resource: %+v", raw)
+			}
+
+			resource = parts[0]
+		case 2:
+			// parts[0] - FILE_OR_GENERATE_TYPE
+			// parts[1] - PATH
+			if parts[0] != "" {
+				if strings.HasPrefix(parts[0], "generate.") {
+					bodyGenerate = parts[0]
+				} else {
+					bodyFile = parts[0]
+				}
+			}
+
+			if parts[1] == "" || !isResource(parts[1]) {
+				return nil, fmt.Errorf("invalid HTTP probe command resource: %+v", raw)
+			}
+
+			resource = parts[1]
+		case 3:
+			// parts[0] - FORM_FIELD_NAME
+			// parts[1] - FILE_OR_GENERATE_TYPE
+			// parts[2] - PATH
+			formFieldName = parts[0]
+
+			if parts[1] != "" {
+				if strings.HasPrefix(parts[1], "generate.") {
+					bodyGenerate = parts[1]
+				} else {
+					bodyFile = parts[1]
+				}
+			}
+
+			if parts[2] == "" || !isResource(parts[2]) {
+				return nil, fmt.Errorf("invalid HTTP probe command resource: %+v", raw)
+			}
+
+			resource = parts[2]
+		case 4:
+			// parts[0] - FORM_FILE_NAME
+			// parts[1] - FORM_FIELD_NAME
+			// parts[2] - FILE_OR_GENERATE_TYPE
+			// parts[3] - PATH
+			formFileName = parts[0]
+			formFieldName = parts[1]
+
+			if parts[2] != "" {
+				if strings.HasPrefix(parts[2], "generate.") {
+					bodyGenerate = parts[2]
+				} else {
+					bodyFile = parts[2]
+				}
+			}
+
+			if parts[3] == "" || !isResource(parts[3]) {
+				return nil, fmt.Errorf("invalid HTTP probe command resource: %+v", raw)
+			}
+
+			resource = parts[3]
+		case 5:
+			// parts[0] - PROTO
+			// parts[1] - FORM_FILE_NAME
+			// parts[2] - FORM_FIELD_NAME
+			// parts[3] - FILE_OR_GENERATE_TYPE
+			// parts[4] - PATH
+			if parts[0] != "" && !isMethod(parts[0]) {
+				return nil, fmt.Errorf("invalid HTTP probe command method: %+v", raw)
+			}
+
+			formFileName = parts[1]
+			formFieldName = parts[2]
+
+			if parts[3] != "" {
+				if strings.HasPrefix(parts[3], "generate.") {
+					bodyGenerate = parts[3]
+				} else {
+					bodyFile = parts[3]
+				}
+			}
+
+			if parts[4] == "" || !isResource(parts[4]) {
+				return nil, fmt.Errorf("invalid HTTP probe command resource: %+v", raw)
+			}
+
+			resource = parts[4]
+		default:
+			return nil, fmt.Errorf("invalid HTTP probe upload command: %s", raw)
+		}
+
+		cmd := config.HTTPProbeCmd{
+			BodyIsForm:    true,
+			Protocol:      proto,
+			Method:        method,
+			Resource:      resource,
+			BodyFile:      bodyFile,
+			BodyGenerate:  bodyGenerate,
+			FormFieldName: formFieldName,
+			FormFileName:  formFileName,
+		}
+
+		probes = append(probes, cmd)
+	}
+
+	return probes, nil
+}
+
 func ParseHTTPProbesFile(filePath string) ([]config.HTTPProbeCmd, error) {
 	probes := []config.HTTPProbeCmd{}
 

pkg/app/master/command/profile/prompt.go

@@ -25,6 +25,7 @@ var CommandFlagSuggestions = &command.FlagSuggestions{
 		{Text: command.FullFlagName(command.FlagHTTPProbeOff), Description: command.FlagHTTPProbeOffUsage},
 		{Text: command.FullFlagName(command.FlagHTTPProbe), Description: command.FlagHTTPProbeUsage},
 		{Text: command.FullFlagName(command.FlagHTTPProbeCmd), Description: command.FlagHTTPProbeCmdUsage},
+		{Text: command.FullFlagName(command.FlagHTTPProbeCmdUpload), Description: command.FlagHTTPProbeCmdUploadUsage},
 		{Text: command.FullFlagName(command.FlagHTTPProbeCmdFile), Description: command.FlagHTTPProbeCmdFileUsage},
 		{Text: command.FullFlagName(command.FlagHTTPProbeStartWait), Description: command.FlagHTTPProbeStartWaitUsage},
 		{Text: command.FullFlagName(command.FlagHTTPProbeRetryCount), Description: command.FlagHTTPProbeRetryCountUsage},

pkg/app/master/config/config.go

@@ -135,75 +135,6 @@ type VolumeMount struct {
 	Options     string
 }
 
-const (
-	ProtoHTTP   = "http"
-	ProtoHTTPS  = "https"
-	ProtoHTTP2  = "http2"
-	ProtoHTTP2C = "http2c"
-	ProtoWS     = "ws"
-	ProtoWSS    = "wss"
-)
-
-func IsProto(value string) bool {
-	switch strings.ToLower(value) {
-	case ProtoHTTP,
-		ProtoHTTPS,
-		ProtoHTTP2,
-		ProtoHTTP2C,
-		ProtoWS,
-		ProtoWSS:
-		return true
-	default:
-		return false
-	}
-}
-
-// HTTPProbeCmd provides the HTTP probe parameters
-type HTTPProbeCmd struct {
-	Method   string   `json:"method"`
-	Resource string   `json:"resource"`
-	Port     int      `json:"port"`
-	Protocol string   `json:"protocol"`
-	Headers  []string `json:"headers"`
-	Body     string   `json:"body"`
-	BodyFile string   `json:"body_file"`
-	Username string   `json:"username"`
-	Password string   `json:"password"`
-	Crawl    bool     `json:"crawl"`
-
-	FastCGI *FastCGIProbeWrapperConfig `json:"fastcgi,omitempty"`
-}
-
-// FastCGI permits fine-grained configuration of the fastcgi RoundTripper.
-type FastCGIProbeWrapperConfig struct {
-	// Root is the fastcgi root directory.
-	// Defaults to the root directory of the container.
-	Root string `json:"root,omitempty"`
-
-	// The path in the URL will be split into two, with the first piece ending
-	// with the value of SplitPath. The first piece will be assumed as the
-	// actual resource (CGI script) name, and the second piece will be set to
-	// PATH_INFO for the CGI script to use.
-	SplitPath []string `json:"split_path,omitempty"`
-
-	// Extra environment variables.
-	EnvVars map[string]string `json:"env,omitempty"`
-
-	// The duration used to set a deadline when connecting to an upstream.
-	DialTimeout time.Duration `json:"dial_timeout,omitempty"`
-
-	// The duration used to set a deadline when reading from the FastCGI server.
-	ReadTimeout time.Duration `json:"read_timeout,omitempty"`
-
-	// The duration used to set a deadline when sending to the FastCGI server.
-	WriteTimeout time.Duration `json:"write_timeout,omitempty"`
-}
-
-// HTTPProbeCmds is a list of HTTPProbeCmd instances
-type HTTPProbeCmds struct {
-	Commands []HTTPProbeCmd `json:"commands"`
-}
-
 // DockerClient provides Docker client parameters
 type DockerClient struct {
 	UseTLS      bool
@@ -232,30 +163,6 @@ type ContinueAfter struct {
 	ContinueChan <-chan struct{}
 }
 
-type HTTPProbeOptions struct {
-	Do            bool
-	Full          bool
-	ExitOnFailure bool
-
-	Cmds  []HTTPProbeCmd
-	Ports []uint16
-
-	StartWait  int
-	RetryCount int
-	RetryWait  int
-
-	CrawlMaxDepth       int
-	CrawlMaxPageCount   int
-	CrawlConcurrency    int
-	CrawlConcurrencyMax int
-
-	APISpecs     []string
-	APISpecFiles []string
-
-	ProxyEndpoint string
-	ProxyPort     int
-}
-
 type AppNodejsInspectOptions struct {
 	IncludePackages []string
 	NextOpts        NodejsWebFrameworkInspectOptions