Minor coursebook scraper improvements

Author: jpahm

parser/courseParser_test.go

@@ -1,9 +1,10 @@
 package parser
 
 import (
+	"testing"
+
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
-	"testing"
 
 	"github.com/UTDNebula/nebula-api/api/schema"
 )

scrapers/coursebook.go

@@ -38,12 +38,7 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string) {
 	}
 
 	// Init http client
-	tr := &http.Transport{
-		MaxIdleConns:       10,
-		IdleConnTimeout:    30 * time.Second,
-		DisableCompression: true,
-	}
-	cli := &http.Client{Transport: tr}
+	cli := &http.Client{}
 
 	// Make the output directory for this term
 	termDir := fmt.Sprintf("%s/%s", outDir, term)
@@ -119,6 +114,11 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string) {
 		}
 		log.Printf("Found %d sections for course prefix %s", len(sectionIDs), coursePrefix)
 
+		// Get a new token before starting the section lookup
+		coursebookHeaders = utils.RefreshToken(chromedpCtx)
+		// Give coursebook some time to recognize the new token
+		time.Sleep(500 * time.Millisecond)
+
 		// Get HTML data for all section IDs
 		sectionsInCoursePrefix := 0
 		for sectionIndex, id := range sectionIDs {

utils/methods.go

@@ -15,6 +15,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"time"
 
 	"strings"
 
@@ -76,6 +77,8 @@ func RefreshToken(chromedpCtx context.Context) map[string][]string {
 		panic(err)
 	}
 
+	time.Sleep(250 * time.Millisecond)
+
 	var cookieStrs []string
 	_, err = chromedp.RunResponse(chromedpCtx,
 		chromedp.Navigate(`https://coursebook.utdallas.edu/`),
@@ -142,6 +145,8 @@ func RefreshAstraToken(chromedpCtx context.Context) map[string][]string {
 		panic(err)
 	}
 
+	time.Sleep(250 * time.Millisecond)
+
 	// Save all cookies to string
 	cookieStr := ""
 	_, err = chromedp.RunResponse(chromedpCtx,