Replace RetryHTTP util with more generic Retry util; improve RefreshToken

Author: jpahm

scrapers/coursebook.go

@@ -76,14 +76,16 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string) {
 			queryStr := fmt.Sprintf("action=search&s%%5B%%5D=term_%s&s%%5B%%5D=%s&s%%5B%%5D=%s", term, coursePrefix, clevel)
 
 			// Try HTTP request, retrying if necessary
-			res, err := utils.RetryHTTP(func() *http.Request {
+			var res *http.Response
+			err := utils.Retry(func() error {
 				req, err := http.NewRequest("POST", "https://coursebook.utdallas.edu/clips/clip-cb11-hat.zog", strings.NewReader(queryStr))
 				if err != nil {
 					panic(err)
 				}
 				req.Header = coursebookHeaders
-				return req
-			}, cli, func(res *http.Response, numRetries int) {
+				res, err = cli.Do(req)
+				return err
+			}, 10, func(numRetries int) {
 				log.Printf("ERROR: Section find for course prefix %s failed! Response code was: %s", coursePrefix, res.Status)
 				// Wait longer if 3 retries fail; we've probably been IP ratelimited...
 				if numRetries >= 3 {
@@ -97,6 +99,7 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string) {
 				// Give coursebook some time to recognize the new token
 				time.Sleep(500 * time.Millisecond)
 			})
+
 			if err != nil {
 				panic(err)
 			}
@@ -128,14 +131,16 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string) {
 			queryStr := fmt.Sprintf("id=%s&req=0bd73666091d3d1da057c5eeb6ef20a7df3CTp0iTMYFuu9paDeUptMzLYUiW4BIk9i8LIFcBahX2E2b18WWXkUUJ1Y7Xq6j3WZAKPbREfGX7lZY96lI7btfpVS95YAprdJHX9dc5wM=&action=section&div=r-62childcontent", id)
 
 			// Try HTTP request, retrying if necessary
-			res, err := utils.RetryHTTP(func() *http.Request {
+			var res *http.Response
+			err := utils.Retry(func() error {
 				req, err := http.NewRequest("POST", "https://coursebook.utdallas.edu/clips/clip-cb11-hat.zog", strings.NewReader(queryStr))
 				if err != nil {
 					panic(err)
 				}
 				req.Header = coursebookHeaders
-				return req
-			}, cli, func(res *http.Response, numRetries int) {
+				res, err = cli.Do(req)
+				return err
+			}, 10, func(numRetries int) {
 				log.Printf("ERROR: Section id lookup for id %s failed! Response code was: %s", id, res.Status)
 				// Wait longer if 3 retries fail; we've probably been IP ratelimited...
 				if numRetries >= 3 {
@@ -149,6 +154,7 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string) {
 				// Give coursebook some time to recognize the new token
 				time.Sleep(500 * time.Millisecond)
 			})
+
 			if err != nil {
 				panic(err)
 			}

utils/methods.go

@@ -11,7 +11,6 @@ import (
 	"fmt"
 	"io/fs"
 	"log"
-	"net/http"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -59,46 +58,59 @@ func RefreshToken(chromedpCtx context.Context) map[string][]string {
 		panic(err)
 	}
 
+	delayedRetryCallback := func(numRetries int) {
+		time.Sleep(250 * time.Millisecond * time.Duration(numRetries))
+	}
+
 	VPrintf("Getting new token...")
-	_, err = chromedp.RunResponse(chromedpCtx,
-		chromedp.ActionFunc(func(ctx context.Context) error {
-			err := network.ClearBrowserCookies().Do(ctx)
-			return err
-		}),
-		chromedp.Navigate(`https://wat.utdallas.edu/login`),
-		chromedp.WaitVisible(`form#login-form`),
-		chromedp.SendKeys(`input#netid`, netID),
-		chromedp.SendKeys(`input#password`, password),
-		chromedp.WaitVisible(`button#login-button`),
-		chromedp.Click(`button#login-button`),
-		chromedp.WaitVisible(`body`),
-	)
+	err = Retry(func() error {
+		_, err = chromedp.RunResponse(chromedpCtx,
+			chromedp.ActionFunc(func(ctx context.Context) error {
+				err := network.ClearBrowserCookies().Do(ctx)
+				return err
+			}),
+			chromedp.Navigate(`https://wat.utdallas.edu/login`),
+			chromedp.WaitVisible(`form#login-form`),
+			chromedp.SendKeys(`input#netid`, netID),
+			chromedp.SendKeys(`input#password`, password),
+			chromedp.WaitVisible(`button#login-button`),
+			chromedp.Click(`button#login-button`),
+			chromedp.WaitVisible(`body`),
+		)
+		return err
+	}, 3, delayedRetryCallback)
+
 	if err != nil {
 		panic(err)
 	}
 
 	time.Sleep(250 * time.Millisecond)
 
 	var cookieStrs []string
-	_, err = chromedp.RunResponse(chromedpCtx,
-		chromedp.Navigate(`https://coursebook.utdallas.edu/`),
-		chromedp.ActionFunc(func(ctx context.Context) error {
-			cookies, err := network.GetCookies().Do(ctx)
-			cookieStrs = make([]string, len(cookies))
-			gotToken := false
-			for i, cookie := range cookies {
-				cookieStrs[i] = fmt.Sprintf("%s=%s", cookie.Name, cookie.Value)
-				if cookie.Name == "PTGSESSID" {
-					VPrintf("Got new token: PTGSESSID = %s", cookie.Value)
-					gotToken = true
+
+	err = Retry(func() error {
+		_, err = chromedp.RunResponse(chromedpCtx,
+			chromedp.Navigate(`https://coursebook.utdallas.edu/`),
+			chromedp.ActionFunc(func(ctx context.Context) error {
+				cookies, err := network.GetCookies().Do(ctx)
+				cookieStrs = make([]string, len(cookies))
+				gotToken := false
+				for i, cookie := range cookies {
+					cookieStrs[i] = fmt.Sprintf("%s=%s", cookie.Name, cookie.Value)
+					if cookie.Name == "PTGSESSID" {
+						VPrintf("Got new token: PTGSESSID = %s", cookie.Value)
+						gotToken = true
+					}
 				}
-			}
-			if !gotToken {
-				return errors.New("failed to get a new token")
-			}
-			return err
-		}),
-	)
+				if !gotToken {
+					return errors.New("failed to get a new token")
+				}
+				return err
+			}),
+		)
+		return err
+	}, 3, delayedRetryCallback)
+
 	if err != nil {
 		panic(err)
 	}
@@ -249,22 +261,16 @@ func Regexpf(format string, vars ...interface{}) *regexp.Regexp {
 	return regexp.MustCompile(fmt.Sprintf(format, vars...))
 }
 
-// Attempts to run the given HTTP request with the given HTTP client, wrapping the request with a retry callback
-func RetryHTTP(requestCreator func() *http.Request, client *http.Client, retryCallback func(res *http.Response, numRetries int)) (res *http.Response, err error) {
-	// Retry loop for requests
-	numRetries := 0
-	for {
-		// Perform HTTP request, retrying if we get a non-200 response code
-		res, err = client.Do(requestCreator())
-		// Retry handling
-		if res.StatusCode != 200 {
-			retryCallback(res, numRetries)
-			numRetries++
-			continue
+// Attempts to retry running the given error-returning function up to a maximum number of retries, at which point the last error is returned. A callback is called between each retry.
+func Retry(action func() error, maxRetries int, retryCallback func(numRetries int)) error {
+	for retries := 1; ; retries++ {
+		// Perform the action
+		err := action()
+		if err == nil || retries > maxRetries {
+			return err
 		}
-		break
+		retryCallback(retries)
 	}
-	return res, err
 }
 
 func GetCoursePrefixes(chromedpCtx context.Context) []string {