--http-probe-client-timeout and --http-probe-client-timeout-crawl slim/profile command flags

Signed-off-by: Kyle Quest <[email protected]>

Author: kcq

README.md

@@ -430,6 +430,8 @@ In the interactive CLI prompt mode you must specify the target image using the `
 - `--poststart-compose-svc` - placeholder for now
 - `--http-probe` - Enables/disables HTTP probing (ENABLED by default; you have to disable the probe if you don't need it by setting the flag to `false`: `--http-probe=false`)
 - `--http-probe-off` - Alternative way to disable HTTP probing
+- `--http-probe-client-timeout` - Probe network client timeout in seconds (defaults to 30 seconds)
+- `--http-probe-client-timeout-crawl` - Crawl probe network client timeout in seconds (defaults to probe network client timeout if set or internal default if not, 10 seconds)
 - `--http-probe-cmd` - User defined HTTP probe(s) as `[[[[\"crawl\":]PROTO:]METHOD:]PATH]` [can use this flag multiple times]
 - `--http-probe-cmd-file` - File with user defined HTTP probe commands
 - `--http-probe-cmd-upload` - User defined HTTP probe(s) to submit form data as `[[[[[PROTO:]FORM_FILE_NAME:]FORM_FIELD_NAME:]FILE_OR_GENERATE_TYPE:]PATH]` [can use this flag multiple times

pkg/app/master/command/build/prompt.go

@@ -50,6 +50,8 @@ var CommandFlagSuggestions = &command.FlagSuggestions{
 		{Text: command.FullFlagName(command.FlagCROShmSize), Description: command.FlagCROShmSizeUsage},
 		{Text: command.FullFlagName(command.FlagHTTPProbeOff), Description: command.FlagHTTPProbeOffUsage},
 		{Text: command.FullFlagName(command.FlagHTTPProbe), Description: command.FlagHTTPProbeUsage},
+		{Text: command.FullFlagName(command.FlagHTTPProbeClientTimeout), Description: command.FlagHTTPProbeClientTimeoutUsage},
+		{Text: command.FullFlagName(command.FlagHTTPProbeClientTimeoutCrawl), Description: command.FlagHTTPProbeClientTimeoutCrawlUsage},
 		{Text: command.FullFlagName(command.FlagHTTPProbeCmd), Description: command.FlagHTTPProbeCmdUsage},
 		{Text: command.FullFlagName(command.FlagHTTPProbeCmdUpload), Description: command.FlagHTTPProbeCmdUploadUsage},
 		{Text: command.FullFlagName(command.FlagHTTPProbeCmdFile), Description: command.FlagHTTPProbeCmdFileUsage},

pkg/app/master/command/cliflags.go

@@ -112,6 +112,8 @@ const (
 
 	FlagHTTPProbe                   = "http-probe"
 	FlagHTTPProbeOff                = "http-probe-off" //alternative way to disable http probing
+	FlagHTTPProbeClientTimeout      = "http-probe-client-timeout"
+	FlagHTTPProbeClientTimeoutCrawl = "http-probe-client-timeout-crawl"
 	FlagHTTPProbeCmd                = "http-probe-cmd"
 	FlagHTTPProbeCmdFile            = "http-probe-cmd-file"
 	FlagHTTPProbeCmdUpload          = "http-probe-cmd-upload"
@@ -228,6 +230,8 @@ const (
 
 	FlagHTTPProbeUsage                   = "Enable or disable HTTP probing"
 	FlagHTTPProbeOffUsage                = "Alternative way to disable HTTP probing"
+	FlagHTTPProbeClientTimeoutUsage      = "Probe network client timeout in seconds (defaults to 30 seconds)"
+	FlagHTTPProbeClientTimeoutCrawlUsage = "Crawl probe network client timeout in seconds (defaults to probe network client timeout if set or internal default if not, 10 seconds)"
 	FlagHTTPProbeCmdUsage                = "User defined HTTP probe(s) as [[[[\"crawl\":]PROTO:]METHOD:]PATH]"
 	FlagHTTPProbeCmdUploadUsage          = "User defined HTTP probe(s) to submit form data as [[[[[PROTO:]FORM_FILE_NAME:]FORM_FIELD_NAME:]FILE_OR_GENERATE_TYPE:]PATH]"
 	FlagHTTPProbeCmdFileUsage            = "File with user defined HTTP probes"
@@ -619,6 +623,18 @@ var CommonFlags = map[string]cli.Flag{
 		Usage:   FlagHTTPProbeOffUsage,
 		EnvVars: []string{"DSLIM_HTTP_PROBE_OFF"},
 	},
+	FlagHTTPProbeClientTimeout: &cli.IntFlag{
+		Name:    FlagHTTPProbeClientTimeout,
+		Value:   0, //0 means use defaults in the app code
+		Usage:   FlagHTTPProbeClientTimeoutUsage,
+		EnvVars: []string{"DSLIM_HTTP_PROBE_CLIENT_TIMEOUT"},
+	},
+	FlagHTTPProbeClientTimeoutCrawl: &cli.IntFlag{
+		Name:    FlagHTTPProbeClientTimeoutCrawl,
+		Value:   0, //0 means use defaults in the app code
+		Usage:   FlagHTTPProbeClientTimeoutCrawlUsage,
+		EnvVars: []string{"DSLIM_HTTP_PROBE_CLIENT_TIMEOUT_CRAWL"},
+	},
 	FlagHTTPProbeCmd: &cli.StringSliceFlag{
 		Name:    FlagHTTPProbeCmd,
 		Value:   cli.NewStringSlice(),
@@ -985,6 +1001,8 @@ func HTTPProbeFlags() []cli.Flag {
 
 func HTTPProbeFlagsBasic() []cli.Flag {
 	return []cli.Flag{
+		Cflag(FlagHTTPProbeClientTimeout),
+		Cflag(FlagHTTPProbeClientTimeoutCrawl),
 		Cflag(FlagHTTPProbeCmd),
 		Cflag(FlagHTTPProbeCmdUpload),
 		Cflag(FlagHTTPProbeCmdFile),

pkg/app/master/command/clifvgetter.go

@@ -58,6 +58,9 @@ func GetHTTPProbeOptions(xc *app.ExecutionContext, ctx *cli.Context, doProbe boo
 	opts := config.HTTPProbeOptions{
 		Full: ctx.Bool(FlagHTTPProbeFull),
 
+		ClientTimeout:      ctx.Int(FlagHTTPProbeClientTimeout),
+		CrawlClientTimeout: ctx.Int(FlagHTTPProbeClientTimeoutCrawl),
+
 		StartWait:  ctx.Int(FlagHTTPProbeStartWait),
 		RetryOff:   ctx.Bool(FlagHTTPProbeRetryOff),
 		RetryCount: ctx.Int(FlagHTTPProbeRetryCount),

pkg/app/master/command/profile/prompt.go

@@ -24,6 +24,8 @@ var CommandFlagSuggestions = &command.FlagSuggestions{
 		{Text: command.FullFlagName(command.FlagCROShmSize), Description: command.FlagCROShmSizeUsage},
 		{Text: command.FullFlagName(command.FlagHTTPProbeOff), Description: command.FlagHTTPProbeOffUsage},
 		{Text: command.FullFlagName(command.FlagHTTPProbe), Description: command.FlagHTTPProbeUsage},
+		{Text: command.FullFlagName(command.FlagHTTPProbeClientTimeout), Description: command.FlagHTTPProbeClientTimeoutUsage},
+		{Text: command.FullFlagName(command.FlagHTTPProbeClientTimeoutCrawl), Description: command.FlagHTTPProbeClientTimeoutCrawlUsage},
 		{Text: command.FullFlagName(command.FlagHTTPProbeCmd), Description: command.FlagHTTPProbeCmdUsage},
 		{Text: command.FullFlagName(command.FlagHTTPProbeCmdUpload), Description: command.FlagHTTPProbeCmdUploadUsage},
 		{Text: command.FullFlagName(command.FlagHTTPProbeCmdFile), Description: command.FlagHTTPProbeCmdFileUsage},

pkg/app/master/config/http_probe.go

@@ -84,6 +84,10 @@ type HTTPProbeOptions struct {
 	ExitOnFailure      bool
 	ExitOnFailureCount int
 
+	ClientTimeout      int
+	CrawlClientTimeout int
+	WsClientTimeout    int //todo
+
 	Cmds  []HTTPProbeCmd
 	Ports []uint16
 

pkg/app/master/probe/http/crawler.go

@@ -24,12 +24,15 @@ func (p *CustomProbe) crawl(proto, domain, addr string) {
 	var httpClient *http.Client
 	if strings.HasPrefix(proto, config.ProtoHTTP2) {
 		var err error
-		if httpClient, err = getHTTPClient(proto); err != nil {
+		if httpClient, err = getHTTPClient(proto, p.opts.ClientTimeout); err != nil {
 			p.xc.Out.Error("HTTP probe - construct client error - %v", err.Error())
 			return
 		}
 
-		httpClient.Timeout = 10 * time.Second //matches the timeout used by Colly
+		if p.opts.ClientTimeout == 0 {
+			httpClient.Timeout = 10 * time.Second //matches the timeout used by Colly
+		}
+
 		jar, _ := cookiejar.New(nil)
 		httpClient.Jar = jar
 	}
@@ -62,6 +65,14 @@ func (p *CustomProbe) crawl(proto, domain, addr string) {
 			c.SetClient(httpClient)
 		}
 
+		if p.opts.CrawlClientTimeout > 0 {
+			c.SetRequestTimeout(time.Duration(p.opts.CrawlClientTimeout) * time.Second)
+			log.Debugf("http.CustomProbe.crawl - set request timeout - p.opts.CrawlClientTimeout (%v)", p.opts.CrawlClientTimeout)
+		} else if p.opts.ClientTimeout > 0 {
+			c.SetRequestTimeout(time.Duration(p.opts.ClientTimeout) * time.Second)
+			log.Debugf("http.CustomProbe.crawl - set request timeout - p.opts.ClientTimeout (%v)", p.opts.ClientTimeout)
+		}
+
 		if p.opts.CrawlMaxDepth > 0 {
 			c.MaxDepth = p.opts.CrawlMaxDepth
 		}

pkg/app/master/probe/http/custom_probe.go

@@ -730,10 +730,10 @@ func (p *CustomProbe) Start() {
 					switch {
 					case cmd.FastCGI != nil:
 						log.Debug("HTTP probe - FastCGI embedded proxy configured")
-						client = getFastCGIClient(cmd.FastCGI)
+						client = getFastCGIClient(p.opts.ClientTimeout, cmd.FastCGI)
 					default:
 						var err error
-						if client, err = getHTTPClient(proto); err != nil {
+						if client, err = getHTTPClient(proto, p.opts.ClientTimeout); err != nil {
 							p.xc.Out.Error("HTTP probe - construct client error - %v", err.Error())
 							continue
 						}

pkg/app/master/probe/http/httpclient.go

@@ -13,12 +13,18 @@ import (
 	"github.com/mintoolkit/mint/pkg/app/master/probe/http/internal"
 )
 
-func getHTTP1Client() *http.Client {
+const defaultClientTimeout = 30
+
+func getHTTP1Client(clientTimeout int) *http.Client {
+	if clientTimeout <= 0 {
+		clientTimeout = defaultClientTimeout
+	}
+
 	client := &http.Client{
-		Timeout: time.Second * 30,
+		Timeout: time.Second * time.Duration(clientTimeout),
 		Transport: &http.Transport{
 			MaxIdleConns:    10,
-			IdleConnTimeout: 30 * time.Second,
+			IdleConnTimeout: time.Duration(clientTimeout) * time.Second,
 			TLSClientConfig: &tls.Config{
 				InsecureSkipVerify: true,
 			},
@@ -28,15 +34,19 @@ func getHTTP1Client() *http.Client {
 	return client
 }
 
-func getHTTP2Client(h2c bool) *http.Client {
+func getHTTP2Client(clientTimeout int, h2c bool) *http.Client {
+	if clientTimeout <= 0 {
+		clientTimeout = defaultClientTimeout
+	}
+
 	transport := &http2.Transport{
 		TLSClientConfig: &tls.Config{
 			InsecureSkipVerify: true,
 		},
 	}
 
 	client := &http.Client{
-		Timeout:   time.Second * 30,
+		Timeout:   time.Second * time.Duration(clientTimeout),
 		Transport: transport,
 	}
 
@@ -50,14 +60,14 @@ func getHTTP2Client(h2c bool) *http.Client {
 	return client
 }
 
-func getHTTPClient(proto string) (*http.Client, error) {
+func getHTTPClient(proto string, clientTimeout int) (*http.Client, error) {
 	switch proto {
 	case config.ProtoHTTP2:
-		return getHTTP2Client(false), nil
+		return getHTTP2Client(clientTimeout, false), nil
 	case config.ProtoHTTP2C:
-		return getHTTP2Client(true), nil
+		return getHTTP2Client(clientTimeout, true), nil
 	default:
-		return getHTTP1Client(), nil
+		return getHTTP1Client(clientTimeout), nil
 	}
 
 	return nil, fmt.Errorf("unsupported HTTP-family protocol %s", proto)
@@ -84,9 +94,12 @@ func getHTTPScheme(proto string) string {
 	return scheme
 }
 
-func getFastCGIClient(cfg *config.FastCGIProbeWrapperConfig) *http.Client {
+func getFastCGIClient(clientTimeout int, cfg *config.FastCGIProbeWrapperConfig) *http.Client {
+	if clientTimeout <= 0 {
+		clientTimeout = defaultClientTimeout
+	}
 
-	genericTimeout := time.Second * 30
+	genericTimeout := time.Second * time.Duration(clientTimeout)
 	var dialTimeout, readTimeout, writeTimeout time.Duration
 	if dialTimeout = cfg.DialTimeout; dialTimeout == 0 {
 		dialTimeout = genericTimeout

pkg/app/master/probe/http/swagger.go

@@ -205,7 +205,7 @@ func apiSpecPrefix(spec *openapi3.T) (string, error) {
 func (p *CustomProbe) loadAPISpecs(proto, targetHost, port string) {
 
 	baseAddr := getHTTPAddr(proto, targetHost, port)
-	client, err := getHTTPClient(proto)
+	client, err := getHTTPClient(proto, p.opts.ClientTimeout)
 	if err != nil {
 		p.xc.Out.Error("HTTP probe - construct client error - %v", err.Error())
 		return
@@ -386,7 +386,7 @@ func (p *CustomProbe) probeAPISpecEndpoints(proto, targetHost, port, prefix stri
 			})
 	}
 
-	httpClient, err := getHTTPClient(proto)
+	httpClient, err := getHTTPClient(proto, p.opts.ClientTimeout)
 	if err != nil {
 		p.xc.Out.Error("HTTP probe - construct client error - %v", err.Error())
 		return