Switch to using external tool pdftotext

Author: TyHil

Dockerfile

@@ -23,6 +23,9 @@ RUN apt-get update && apt-get install -y chromium
 ENV CHROMIUM_BIN=/usr/bin/chromium
 ENV GOOGLE_CHROME_BIN=/usr/bin/chromium
 
+# Install poppler-utils for pdftotext for academic calendar parsing
+RUN apt-get install -y poppler-utils
+
 # Copy build file from builder
 COPY --from=builder /app/api-tools /app/api-tools
 

go.mod

@@ -9,7 +9,6 @@ require (
 	github.com/chromedp/chromedp v0.12.1
 	github.com/google/go-cmp v0.7.0
 	github.com/joho/godotenv v1.5.1
-	github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80
 	github.com/valyala/fastjson v1.6.4
 	go.mongodb.org/mongo-driver v1.17.3
 	golang.org/x/net v0.43.0

parser/academicCalendars.go

@@ -1,3 +1,10 @@
+/*
+Code requires having pdftotext installed: https://www.xpdfreader.com/pdftotext-man.html
+apt-get install -y poppler-utils
+I found all the Go programs for PDF text extraction were all either paid, had a
+complicated installation process, or errored on one of the PDFs.
+*/
+
 package parser
 
 import (
@@ -12,13 +19,14 @@ import (
 	"log"
 	"net/http"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/UTDNebula/api-tools/utils"
 	"github.com/UTDNebula/nebula-api/api/schema"
-	"github.com/ledongthuc/pdf"
 	"google.golang.org/genai"
 )
 
@@ -93,7 +101,21 @@ func ParseAcademicCalendars(inDir string, outDir string) {
 
 				academicCalendar, err := parsePdf(path)
 				if err != nil {
-					panic(err)
+					if strings.Contains(err.Error(), "429") {
+						// Exponential-ish backoff up to 60s for 429 rate limiting
+						backoffs := []time.Duration{20 * time.Second, 40 * time.Second, 60 * time.Second}
+						for _, delay := range backoffs {
+							time.Sleep(delay)
+							academicCalendar, err = parsePdf(path)
+							if err == nil || !strings.Contains(err.Error(), "429") {
+								break
+							}
+						}
+					}
+
+					if err != nil {
+						panic(err)
+					}
 				}
 
 				mu.Lock()
@@ -200,33 +222,20 @@ func parsePdf(path string) (schema.AcademicCalendar, error) {
 }
 
 // Read the text from the first page of a PDF
+// Using external program pdftotext
 func readPdf(path string) (string, error) {
-	// Open the PDF
-	f, r, err := pdf.Open(path)
-	if err != nil {
-		return "", err
-	}
-	defer f.Close()
+	cmd := exec.Command("pdftotext", "-l", "1", "-raw", path, "-")
 
-	// Make sure at least one page exists
-	if r.NumPage() < 1 {
-		return "", fmt.Errorf("no pages in PDF")
-	}
-
-	// Get the first page
-	page := r.Page(1) // pages are 1-indexed
-	if page.V.IsNull() {
-		return "", fmt.Errorf("failed to read page 1")
-	}
+	var out bytes.Buffer
+	var stderr bytes.Buffer
+	cmd.Stdout = &out
+	cmd.Stderr = &stderr
 
-	// Read text
-	var buf bytes.Buffer
-	text := page.Content().Text
-	for _, t := range text {
-		buf.WriteString(t.S) // S is the actual string
+	if err := cmd.Run(); err != nil {
+		return "", fmt.Errorf("failed to run pdftotext: %v (%s)", err, stderr.String())
 	}
 
-	return buf.String(), nil
+	return out.String(), nil
 }
 
 // Check cache for a response to the same prompt