From 42c7924c16bd5e2e1032a655c6ab1a91207b96e7 Mon Sep 17 00:00:00 2001 From: Chris Bowen Date: Fri, 25 Aug 2017 12:02:21 -0400 Subject: [PATCH 01/16] split \r\n to replace them properly in output error string --- cmd/site.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/site.go b/cmd/site.go index 02236c5..d7f0586 100644 --- a/cmd/site.go +++ b/cmd/site.go @@ -51,8 +51,10 @@ func queryPage(u string) { if err != nil { log.Fatal(err) } - // Convert rd to string and remove line breaks to output on single line - rs := s.Replace(string(rd), "\r\n", "", -1) + // Convert rd to string and remove carriage returns from output + rs := s.Replace(string(rd), "\r", "", -1) + // Remove line feeds from output + rs = s.Replace(rs, "\n", "", -1) log.Fatalf("Status Code: %v Body: %s", resp.StatusCode, rs) } else if debugFlag { log.Println("Status Code:", resp.StatusCode) From 06fbb45f383d4efa6ee3aa9fcde0e4ef036de79d Mon Sep 17 00:00:00 2001 From: Chris Bowen Date: Tue, 29 Aug 2017 11:29:25 -0400 Subject: [PATCH 02/16] adding 1 second sleep timer before site.go exits on failure --- cmd/site.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/site.go b/cmd/site.go index d7f0586..fc0bf2e 100644 --- a/cmd/site.go +++ b/cmd/site.go @@ -55,7 +55,11 @@ func queryPage(u string) { rs := s.Replace(string(rd), "\r", "", -1) // Remove line feeds from output rs = s.Replace(rs, "\n", "", -1) - log.Fatalf("Status Code: %v Body: %s", resp.StatusCode, rs) + log.Printf("Status Code: %v Body: %s", resp.StatusCode, rs) + // Sleep for 1 second allowing output error to Stdout to be picked up + // by logging container. + time.Sleep(1000 * time.Millisecond) + os.Exit(1) } else if debugFlag { log.Println("Status Code:", resp.StatusCode) } From db3628a1f8b3dcf4aebb10f3992ba587eaed459c Mon Sep 17 00:00:00 2001 From: Chris Bowen Date: Tue, 29 Aug 2017 14:46:52 -0400 Subject: [PATCH 03/16] updated timeout to 10 seconds for testing --- cmd/site.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/site.go b/cmd/site.go index fc0bf2e..5a05162 100644 --- a/cmd/site.go +++ b/cmd/site.go @@ -56,9 +56,9 @@ func queryPage(u string) { // Remove line feeds from output rs = s.Replace(rs, "\n", "", -1) log.Printf("Status Code: %v Body: %s", resp.StatusCode, rs) - // Sleep for 1 second allowing output error to Stdout to be picked up + // Sleep for 10 seconds allowing output error to Stdout to be picked up // by logging container. - time.Sleep(1000 * time.Millisecond) + time.Sleep(10000 * time.Millisecond) os.Exit(1) } else if debugFlag { log.Println("Status Code:", resp.StatusCode) From fc8406782431e9d30151c79fed2548846b68cf39 Mon Sep 17 00:00:00 2001 From: Chris Bowen Date: Wed, 30 Aug 2017 13:48:37 -0400 Subject: [PATCH 04/16] updated sleep time to 15 seconds for debugging --- cmd/site.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/site.go b/cmd/site.go index 5a05162..cfc76db 100644 --- a/cmd/site.go +++ b/cmd/site.go @@ -55,10 +55,10 @@ func queryPage(u string) { rs := s.Replace(string(rd), "\r", "", -1) // Remove line feeds from output rs = s.Replace(rs, "\n", "", -1) - log.Printf("Status Code: %v Body: %s", resp.StatusCode, rs) - // Sleep for 10 seconds allowing output error to Stdout to be picked up + fmt.Printf("Status Code: %v Body: %s", resp.StatusCode, rs) + // Sleep for 10 seconds allowing output error to Stdout to be picked up // by logging container. - time.Sleep(10000 * time.Millisecond) + time.Sleep(15000 * time.Millisecond) os.Exit(1) } else if debugFlag { log.Println("Status Code:", resp.StatusCode) From 3f259ff64268f044cfb50888a0124e831adf594c Mon Sep 17 00:00:00 2001 From: Chris Bowen Date: Thu, 31 Aug 2017 11:03:47 -0400 Subject: [PATCH 05/16] added counter and time to exit values to site.go --- README.md | 4 ++-- cmd/site.go | 66 ++++++++++++++++++++++++++++++++++++++--------------- 2 files changed, 49 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index fe5ccf2..5fb8cd2 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Service Usage: spinner service [name] [flags] Site Usage: - spinner site [url] [flags] + spinner site [url] [counter limit] [time to exit] [flags] ``` @@ -51,4 +51,4 @@ Spinner. spinner.exe service W3SVC -t c:\\iislog\\W3SVC\\u_extend1.log ``` -The Linux build is experimental and does not include the `service` command. \ No newline at end of file +The Linux build is experimental and does not include the `service` command. diff --git a/cmd/site.go b/cmd/site.go index cfc76db..4719992 100644 --- a/cmd/site.go +++ b/cmd/site.go @@ -19,6 +19,7 @@ import ( "log" "net/http" "os" + "strconv" s "strings" "time" @@ -27,8 +28,8 @@ import ( var urlFlag string -func queryPage(u string) { - +func queryPage(u, cl, tte string) { + var c int64 var fullURL string if s.HasPrefix(u, "http://") || s.HasPrefix(u, "https://") { @@ -38,7 +39,17 @@ func queryPage(u string) { } fmt.Println("Full URL being monitored:", fullURL) + fmt.Println("Counter limit:", cl) for { + cl, err := strconv.ParseInt(cl, 10, 8) + if err != nil { + log.Fatal(err) + } + + tte, err := strconv.ParseInt(tte, 10, 8) + if err != nil { + log.Fatal(err) + } resp, err := http.Get(fullURL) @@ -47,38 +58,51 @@ func queryPage(u string) { } if resp.StatusCode >= 300 { - rd, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Fatal(err) + c++ + if c >= cl { + rd, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Fatal(err) + } + // Convert rd to string and remove carriage returns from output + rs := s.Replace(string(rd), "\r", "", -1) + // Remove line feeds from output + rs = s.Replace(rs, "\n", "", -1) + fmt.Printf("Spinner exiting... Status Code: %v Body: %s", resp.StatusCode, rs) + // Sleep for 10 seconds allowing output error to Stdout to be picked up + // by logging container. + time.Sleep(time.Duration(tte) * time.Second) + os.Exit(1) + } else { + log.Println("Status Code:", resp.StatusCode, "Counter count:", c) } - // Convert rd to string and remove carriage returns from output - rs := s.Replace(string(rd), "\r", "", -1) - // Remove line feeds from output - rs = s.Replace(rs, "\n", "", -1) - fmt.Printf("Status Code: %v Body: %s", resp.StatusCode, rs) - // Sleep for 10 seconds allowing output error to Stdout to be picked up - // by logging container. - time.Sleep(15000 * time.Millisecond) - os.Exit(1) } else if debugFlag { + c = 0 log.Println("Status Code:", resp.StatusCode) } resp.Body.Close() - time.Sleep(1000 * time.Millisecond) + time.Sleep(1 * time.Second) } } // siteCmd represents the site command var siteCmd = &cobra.Command{ - Use: "site [url]", + Use: "site [url] [counter limit] [time to exit]", Short: "Watch a Site", Aliases: []string{"url", "address"}, Example: "spinner.exe site http://localhost -t c:\\iislog\\W3SVC\\u_extend1.log", Long: `Poll Web Site by Get request and terminate this process if the a >300 status code is returned. +Counter Limit (default 1) is the number of times the site being monitored can +be down before spinner exits. + +Time to Exit (default 1) is the time (in seconds) after the response body is +logged to stdout before spinner will shutdown. This can be useful +if the monitoring software does not catch the error quick enough. + Use this as the entrypoint for a container to stop the container if the given service stops.`, Run: func(cmd *cobra.Command, args []string) { @@ -92,9 +116,13 @@ the given service stops.`, if tailFile != "" { go TailLog() } - - queryPage(args[0]) - + if len(args) == 1 { + queryPage(args[0], "1", "1") + } else if len(args) == 2 { + queryPage(args[0], args[1], "1") + } else { + queryPage(args[0], args[1], args[2]) + } }, } From d53652e103220d980dbba18145b672968e1ef1f4 Mon Sep 17 00:00:00 2001 From: Chris Bowen Date: Thu, 31 Aug 2017 11:23:35 -0400 Subject: [PATCH 06/16] updated comments to be accurate --- cmd/site.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/site.go b/cmd/site.go index 4719992..3fb65b8 100644 --- a/cmd/site.go +++ b/cmd/site.go @@ -69,8 +69,8 @@ func queryPage(u, cl, tte string) { // Remove line feeds from output rs = s.Replace(rs, "\n", "", -1) fmt.Printf("Spinner exiting... Status Code: %v Body: %s", resp.StatusCode, rs) - // Sleep for 10 seconds allowing output error to Stdout to be picked up - // by logging container. + // Sleep for n second(s) allowing output error to stdout to be picked up + // by monitoring software/container (if needed) time.Sleep(time.Duration(tte) * time.Second) os.Exit(1) } else { From 22aeed433aa3f951599655ae93e3ffa2001f7fa1 Mon Sep 17 00:00:00 2001 From: Chris Bowen Date: Thu, 31 Aug 2017 11:42:47 -0400 Subject: [PATCH 07/16] changed log output --- cmd/site.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/site.go b/cmd/site.go index 3fb65b8..0a9ddfc 100644 --- a/cmd/site.go +++ b/cmd/site.go @@ -68,7 +68,7 @@ func queryPage(u, cl, tte string) { rs := s.Replace(string(rd), "\r", "", -1) // Remove line feeds from output rs = s.Replace(rs, "\n", "", -1) - fmt.Printf("Spinner exiting... Status Code: %v Body: %s", resp.StatusCode, rs) + log.Println("Spinner exiting... Status Code:", resp.StatusCode, " Body:", rs) // Sleep for n second(s) allowing output error to stdout to be picked up // by monitoring software/container (if needed) time.Sleep(time.Duration(tte) * time.Second) From 42e62067ca35d78c1ef689d2952d85cd150f58c9 Mon Sep 17 00:00:00 2001 From: Chris Bowen Date: Sat, 2 Sep 2017 11:35:16 -0400 Subject: [PATCH 08/16] WIP: outfile for site.go throwing access denied error --- README.md | 3 ++- cmd/root.go | 2 ++ cmd/site.go | 41 +++++++++++++++++++++++++++++++++-------- main.go | 2 +- 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 5fb8cd2..9b4953a 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,8 @@ Flags: --config string config file (default is $HOME/.spinner.yaml) -d, --debug Print debug logging -h, --help help for spinner - -t, --tail string Path to file to tail and pipe to STDOUT. + -t, --tail string Path to file to tail and pipe to STDOUT + -o, --out Path to file to write large errors to Service Usage: spinner service [name] [flags] diff --git a/cmd/root.go b/cmd/root.go index 33a4440..f0724c3 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -25,6 +25,7 @@ import ( var cfgFile string var tailFile string +var outFile string var debugFlag bool //func Kill(err error) { @@ -72,6 +73,7 @@ func init() { RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.spinner.yaml)") RootCmd.PersistentFlags().StringVarP(&tailFile, "tail", "t", "", "Path to file to tail and pipe to STDOUT.") + RootCmd.PersistentFlags().StringVarP(&outFile, "out", "o", "", "Path to file to write errors out to.") RootCmd.PersistentFlags().BoolVarP(&debugFlag, "debug", "d", false, "Print debug logging") } diff --git a/cmd/site.go b/cmd/site.go index 0a9ddfc..fd19edd 100644 --- a/cmd/site.go +++ b/cmd/site.go @@ -14,11 +14,13 @@ package cmd import ( + "bufio" "fmt" "io/ioutil" "log" "net/http" "os" + "path/filepath" "strconv" s "strings" "time" @@ -38,8 +40,7 @@ func queryPage(u, cl, tte string) { fullURL = "http://" + u } - fmt.Println("Full URL being monitored:", fullURL) - fmt.Println("Counter limit:", cl) + log.Println("Full URL being monitored:", fullURL, "Counter limit:", cl) for { cl, err := strconv.ParseInt(cl, 10, 8) if err != nil { @@ -52,7 +53,6 @@ func queryPage(u, cl, tte string) { } resp, err := http.Get(fullURL) - if err != nil { log.Fatal("An error occurred during the request:", err) } @@ -68,11 +68,36 @@ func queryPage(u, cl, tte string) { rs := s.Replace(string(rd), "\r", "", -1) // Remove line feeds from output rs = s.Replace(rs, "\n", "", -1) - log.Println("Spinner exiting... Status Code:", resp.StatusCode, " Body:", rs) + + // Output to file if path is set + if outFile != "" { + d, _ := filepath.Split(outFile) + err := os.MkdirAll(d, os.ModePerm) + if err != nil { + panic(err) + } + + f, err := os.OpenFile(outFile, os.O_RDWR|os.O_APPEND, 0660) + if err != nil { + panic(err) + } + defer f.Close() + + w := bufio.NewWriter(f) + _, err = fmt.Fprintf(w, "%v %s\n", time.Now().Format("2006/01/02 15:04:05"), rs) + if err != nil { + panic(err) + } + + w.Flush() + } + + // Output to stdout + log.Println("Status Code:", resp.StatusCode, " Body:", rs) // Sleep for n second(s) allowing output error to stdout to be picked up // by monitoring software/container (if needed) time.Sleep(time.Duration(tte) * time.Second) - os.Exit(1) + log.Fatalln("Spinner shutting down, status code was:", resp.StatusCode) } else { log.Println("Status Code:", resp.StatusCode, "Counter count:", c) } @@ -92,9 +117,9 @@ var siteCmd = &cobra.Command{ Use: "site [url] [counter limit] [time to exit]", Short: "Watch a Site", Aliases: []string{"url", "address"}, - Example: "spinner.exe site http://localhost -t c:\\iislog\\W3SVC\\u_extend1.log", - Long: `Poll Web Site by Get request and terminate this process if -the a >300 status code is returned. + Example: "spinner.exe site http://localhost 5 5 -t c:\\iislog\\W3SVC\\u_extend1.log -o c:\\logs\\spinner.log", + Long: `Poll Web Site by Get request and terminate this process if the a >300 +status code is returned. Counter Limit (default 1) is the number of times the site being monitored can be down before spinner exits. diff --git a/main.go b/main.go index c8e85b7..4f958d5 100644 --- a/main.go +++ b/main.go @@ -13,7 +13,7 @@ package main -import "github.com/Ticketmaster/spinner/cmd" +import "github.com/chrisjbowen/spinner/cmd" func main() { cmd.Execute() From c81c48b32bc9709288220f3c68285c09a16a61e5 Mon Sep 17 00:00:00 2001 From: Chris Bowen Date: Thu, 14 Sep 2017 09:28:12 -0400 Subject: [PATCH 09/16] updated output print --- cmd/site.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/site.go b/cmd/site.go index fd19edd..aed1272 100644 --- a/cmd/site.go +++ b/cmd/site.go @@ -41,6 +41,7 @@ func queryPage(u, cl, tte string) { } log.Println("Full URL being monitored:", fullURL, "Counter limit:", cl) + for { cl, err := strconv.ParseInt(cl, 10, 8) if err != nil { @@ -77,14 +78,14 @@ func queryPage(u, cl, tte string) { panic(err) } - f, err := os.OpenFile(outFile, os.O_RDWR|os.O_APPEND, 0660) + f, err := os.OpenFile(outFile, os.O_APPEND|os.O_CREATE, 0666) if err != nil { panic(err) } defer f.Close() w := bufio.NewWriter(f) - _, err = fmt.Fprintf(w, "%v %s\n", time.Now().Format("2006/01/02 15:04:05"), rs) + _, err = fmt.Fprintln(w, time.Now().Format("2006/01/02 15:04:05")+rs) if err != nil { panic(err) } From c482a6d9638a9738d9174c31c7993a91d64ff1cc Mon Sep 17 00:00:00 2001 From: Chris Bowen Date: Thu, 14 Sep 2017 09:33:15 -0400 Subject: [PATCH 10/16] updated formatting on text output --- cmd/site.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/site.go b/cmd/site.go index aed1272..26b5b22 100644 --- a/cmd/site.go +++ b/cmd/site.go @@ -85,7 +85,7 @@ func queryPage(u, cl, tte string) { defer f.Close() w := bufio.NewWriter(f) - _, err = fmt.Fprintln(w, time.Now().Format("2006/01/02 15:04:05")+rs) + _, err = fmt.Fprintln(w, time.Now().Format("2006/01/02 15:04:05 ")+rs) if err != nil { panic(err) } From 235f898cf999c40a63646782abcefb0db53d645c Mon Sep 17 00:00:00 2001 From: Chris Bowen Date: Thu, 14 Sep 2017 13:36:58 -0400 Subject: [PATCH 11/16] updated response body output to file --- README.md | 2 +- cmd/site.go | 12 ++++++------ main.go | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 9b4953a..9f05a23 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Flags: -d, --debug Print debug logging -h, --help help for spinner -t, --tail string Path to file to tail and pipe to STDOUT - -o, --out Path to file to write large errors to + -o, --out Path to file to write response body if spinner exits Service Usage: spinner service [name] [flags] diff --git a/cmd/site.go b/cmd/site.go index 26b5b22..140f85e 100644 --- a/cmd/site.go +++ b/cmd/site.go @@ -78,19 +78,19 @@ func queryPage(u, cl, tte string) { panic(err) } - f, err := os.OpenFile(outFile, os.O_APPEND|os.O_CREATE, 0666) + f, err := os.OpenFile(outFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) if err != nil { panic(err) } defer f.Close() - w := bufio.NewWriter(f) - _, err = fmt.Fprintln(w, time.Now().Format("2006/01/02 15:04:05 ")+rs) + ls := time.Now().Format("2006/01/02 15:04:05") + " [spinner-app] " + rs + "\n" + + b := bufio.NewWriterSize(f, 4000) + _, err = b.WriteString(ls) if err != nil { - panic(err) + log.Println(err) } - - w.Flush() } // Output to stdout diff --git a/main.go b/main.go index 4f958d5..c8e85b7 100644 --- a/main.go +++ b/main.go @@ -13,7 +13,7 @@ package main -import "github.com/chrisjbowen/spinner/cmd" +import "github.com/Ticketmaster/spinner/cmd" func main() { cmd.Execute() From d8f31ab70f9ff35615eeebce913d0a076cd47dae Mon Sep 17 00:00:00 2001 From: Chris Bowen Date: Thu, 14 Sep 2017 13:55:32 -0400 Subject: [PATCH 12/16] updated help and readme information --- README.md | 2 +- cmd/site.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9f05a23..1ff0a1b 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Flags: -d, --debug Print debug logging -h, --help help for spinner -t, --tail string Path to file to tail and pipe to STDOUT - -o, --out Path to file to write response body if spinner exits + -o, --out Path to file to write response body if site response status >= 300 Service Usage: spinner service [name] [flags] diff --git a/cmd/site.go b/cmd/site.go index 140f85e..3eb4e5d 100644 --- a/cmd/site.go +++ b/cmd/site.go @@ -129,6 +129,10 @@ Time to Exit (default 1) is the time (in seconds) after the response body is logged to stdout before spinner will shutdown. This can be useful if the monitoring software does not catch the error quick enough. +OutFile is the location path to a file to write out the response body if +the response status is >= 300. This can be useful in troubleshooting why +the application is exiting. + Use this as the entrypoint for a container to stop the container if the given service stops.`, Run: func(cmd *cobra.Command, args []string) { From 104472815e85941684e92945d6a35ab6a91498fc Mon Sep 17 00:00:00 2001 From: Chris Bowen Date: Fri, 15 Sep 2017 12:41:26 -0400 Subject: [PATCH 13/16] updated formatting for date --- cmd/site.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/site.go b/cmd/site.go index 3eb4e5d..6738835 100644 --- a/cmd/site.go +++ b/cmd/site.go @@ -84,9 +84,9 @@ func queryPage(u, cl, tte string) { } defer f.Close() - ls := time.Now().Format("2006/01/02 15:04:05") + " [spinner-app] " + rs + "\n" + ls := time.Now().Format("2006-01-02 15:04:05") + " [spinner-app] " + rs + "\n" - b := bufio.NewWriterSize(f, 4000) + b := bufio.NewWriterSize(f, 10000) _, err = b.WriteString(ls) if err != nil { log.Println(err) From 2bf2bc4365d8659eaabf4017da0c33577a16d15b Mon Sep 17 00:00:00 2001 From: Chris Bowen Date: Mon, 18 Sep 2017 12:00:37 -0400 Subject: [PATCH 14/16] added buffer flush --- cmd/site.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/site.go b/cmd/site.go index 6738835..7c44407 100644 --- a/cmd/site.go +++ b/cmd/site.go @@ -86,11 +86,16 @@ func queryPage(u, cl, tte string) { ls := time.Now().Format("2006-01-02 15:04:05") + " [spinner-app] " + rs + "\n" - b := bufio.NewWriterSize(f, 10000) + b := bufio.NewWriterSize(f, 4000) _, err = b.WriteString(ls) if err != nil { log.Println(err) } + + err = b.Flush() + if err != nil { + log.Println(err) + } } // Output to stdout From 499f3f8163ab44367e48c6b1085e9987bb0f5836 Mon Sep 17 00:00:00 2001 From: Chris Bowen Date: Mon, 18 Sep 2017 12:05:08 -0400 Subject: [PATCH 15/16] updated buffer size to 10k --- cmd/site.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/site.go b/cmd/site.go index 7c44407..4da3fd7 100644 --- a/cmd/site.go +++ b/cmd/site.go @@ -86,7 +86,7 @@ func queryPage(u, cl, tte string) { ls := time.Now().Format("2006-01-02 15:04:05") + " [spinner-app] " + rs + "\n" - b := bufio.NewWriterSize(f, 4000) + b := bufio.NewWriterSize(f, 10000) _, err = b.WriteString(ls) if err != nil { log.Println(err) From 6ebfb65cb58168fc4c8193276ab270378c98d4b6 Mon Sep 17 00:00:00 2001 From: Chris Bowen Date: Tue, 26 Sep 2017 11:28:28 -0400 Subject: [PATCH 16/16] updated get request to include 30 second timeout --- cmd/site.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmd/site.go b/cmd/site.go index 4da3fd7..756a48c 100644 --- a/cmd/site.go +++ b/cmd/site.go @@ -53,7 +53,14 @@ func queryPage(u, cl, tte string) { log.Fatal(err) } - resp, err := http.Get(fullURL) + // Set 30 second timeout for page GET + tr := &http.Transport{ + IdleConnTimeout: 30 * time.Second, + DisableCompression: true, + } + + client := &http.Client{Transport: tr} + resp, err := client.Get(fullURL) if err != nil { log.Fatal("An error occurred during the request:", err) }