log the crawl error

dennis-tra · dennis-tra · commit 2317f90c8fbd · 2023-06-16T16:27:45.000+02:00
diff --git a/pkg/crawl/scheduler.go b/pkg/crawl/scheduler.go
@@ -410,7 +410,10 @@ func (s *Scheduler) handleResult(ctx context.Context, cr Result) {
 		}
 	}
 
-	if cr.ConnectError == nil && cr.CrawlError != nil {
+	if cr.CrawlError != nil {
+		// Don't count this against the "dialability" errors - for now.
+		// s.connErrs[cr.CrawlErrorStr] += 1
+
 		// Log and count crawl errors
 		if cr.CrawlErrorStr == models.NetErrorUnknown {
 			logEntry = logEntry.WithError(cr.CrawlError)

Original file line number	Diff line number	Diff line change
`@@ -410,7 +410,10 @@ func (s *Scheduler) handleResult(ctx context.Context, cr Result) {`
`410`	`410`	`}`
`411`	`411`	`}`
`412`	`412`
`413`		`- if cr.ConnectError == nil && cr.CrawlError != nil {`
	`413`	`+ if cr.CrawlError != nil {`
	`414`	`+ // Don't count this against the "dialability" errors - for now.`
	`415`	`+ // s.connErrs[cr.CrawlErrorStr] += 1`
	`416`	`+`
`414`	`417`	`// Log and count crawl errors`
`415`	`418`	`if cr.CrawlErrorStr == models.NetErrorUnknown {`
`416`	`419`	`logEntry = logEntry.WithError(cr.CrawlError)`