Improve coursebook scraping hang prevention

Author: jpahm

scrapers/coursebook.go

@@ -6,6 +6,7 @@ package scrapers
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"log"
 	"net/http"
@@ -84,20 +85,15 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string) {
 				}
 				req.Header = coursebookHeaders
 				res, err = cli.Do(req)
+				if res.StatusCode != 200 {
+					return errors.New("Non-200 response status code")
+				}
 				return err
 			}, 10, func(numRetries int) {
-				log.Printf("ERROR: Section find for course prefix %s failed! Response code was: %s", coursePrefix, res.Status)
-				// Wait longer if 3 retries fail; we've probably been IP ratelimited...
-				if numRetries >= 3 {
-					log.Printf("WARNING: More than 3 retries have failed. Waiting for 5 minutes before attempting further retries.")
-					time.Sleep(5 * time.Minute)
-				} else {
-					log.Printf("Getting new token and retrying in 3 seconds...")
-					time.Sleep(3 * time.Second)
-				}
+				log.Printf("WARN: Section find for course prefix %s failed! Response code was: %s", coursePrefix, res.Status)
 				coursebookHeaders = utils.RefreshToken(chromedpCtx)
-				// Give coursebook some time to recognize the new token
-				time.Sleep(500 * time.Millisecond)
+				// Wait proportionally long to how many times we've retried; generally works pretty well
+				time.Sleep(500 * time.Millisecond * time.Duration(numRetries))
 			})
 
 			if err != nil {
@@ -128,7 +124,7 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string) {
 
 			// Get section info
 			// Worth noting that the "req" and "div" params in the request below don't actually seem to matter... consider them filler to make sure the request goes through
-			queryStr := fmt.Sprintf("id=%s&req=0bd73666091d3d1da057c5eeb6ef20a7df3CTp0iTMYFuu9paDeUptMzLYUiW4BIk9i8LIFcBahX2E2b18WWXkUUJ1Y7Xq6j3WZAKPbREfGX7lZY96lI7btfpVS95YAprdJHX9dc5wM=&action=section&div=r-62childcontent", id)
+			queryStr := fmt.Sprintf("id=%s&req=b30da8ab21637dbef35fd7682f48e1c1W0ypMhaj%%2FdsnYn3Wa03BrxSNgCeyvLfvucSTobcSXRf38SWaUaNfMjJQn%%2BdcabF%%2F7ZuG%%2BdKqHAqmrxEKyg8AdB0FqVGcz4rkff3%%2B3SIUIt8%%3D&action=info", id)
 
 			// Try HTTP request, retrying if necessary
 			var res *http.Response
@@ -139,20 +135,15 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string) {
 				}
 				req.Header = coursebookHeaders
 				res, err = cli.Do(req)
+				if res.StatusCode != 200 {
+					return errors.New("Non-200 response status code")
+				}
 				return err
 			}, 10, func(numRetries int) {
-				log.Printf("ERROR: Section id lookup for id %s failed! Response code was: %s", id, res.Status)
-				// Wait longer if 3 retries fail; we've probably been IP ratelimited...
-				if numRetries >= 3 {
-					log.Printf("WARNING: More than 3 retries have failed. Waiting for 5 minutes before attempting further retries.")
-					time.Sleep(5 * time.Minute)
-				} else {
-					log.Printf("Getting new token and retrying in 3 seconds...")
-					time.Sleep(3 * time.Second)
-				}
+				log.Printf("WARN: Section id lookup for id %s failed! Response code was: %s", id, res.Status)
 				coursebookHeaders = utils.RefreshToken(chromedpCtx)
-				// Give coursebook some time to recognize the new token
-				time.Sleep(500 * time.Millisecond)
+				// Wait proportionally long to how many times we've retried; generally works pretty well
+				time.Sleep(500 * time.Millisecond * time.Duration(numRetries))
 			})
 
 			if err != nil {
@@ -181,6 +172,10 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string) {
 			sectionsInCoursePrefix++
 		}
 		log.Printf("\nFinished scraping course prefix %s. Got %d sections.", coursePrefix, sectionsInCoursePrefix)
+		// Panic if we got fewer sections than we should've
+		if sectionsInCoursePrefix != len(sectionIDs) {
+			log.Panicf("Section count mismatch! Coursebook has %d sections for course prefix %s but we only got %d", sectionsInCoursePrefix, coursePrefix, sectionsInCoursePrefix)
+		}
 		totalSections += sectionsInCoursePrefix
 	}
 	log.Printf("\nDone scraping term! Scraped a total of %d sections.", totalSections)

utils/methods.go

@@ -64,19 +64,19 @@ func RefreshToken(chromedpCtx context.Context) map[string][]string {
 
 	VPrintf("Getting new token...")
 	err = Retry(func() error {
-		_, err = chromedp.RunResponse(chromedpCtx,
+		r, err := chromedp.RunResponse(chromedpCtx,
 			chromedp.ActionFunc(func(ctx context.Context) error {
 				err := network.ClearBrowserCookies().Do(ctx)
 				return err
 			}),
 			chromedp.Navigate(`https://wat.utdallas.edu/login`),
-			chromedp.WaitVisible(`form#login-form`),
 			chromedp.SendKeys(`input#netid`, netID),
 			chromedp.SendKeys(`input#password`, password),
-			chromedp.WaitVisible(`button#login-button`),
 			chromedp.Click(`button#login-button`),
-			chromedp.WaitVisible(`body`),
 		)
+		if r != nil && r.Status != 200 {
+			return errors.New("Non-200 response status code")
+		}
 		return err
 	}, 3, delayedRetryCallback)
 
@@ -89,7 +89,7 @@ func RefreshToken(chromedpCtx context.Context) map[string][]string {
 	var cookieStrs []string
 
 	err = Retry(func() error {
-		_, err = chromedp.RunResponse(chromedpCtx,
+		r, err := chromedp.RunResponse(chromedpCtx,
 			chromedp.Navigate(`https://coursebook.utdallas.edu/`),
 			chromedp.ActionFunc(func(ctx context.Context) error {
 				cookies, err := network.GetCookies().Do(ctx)
@@ -108,6 +108,9 @@ func RefreshToken(chromedpCtx context.Context) map[string][]string {
 				return err
 			}),
 		)
+		if r != nil && r.Status != 200 {
+			return errors.New("Non-200 response status code")
+		}
 		return err
 	}, 3, delayedRetryCallback)