Merge pull request #2309 from GDATTACKER-RESEARCHER/dev

Mzack9999 · web-flow · commit c6948ed01ac0 · 2025-11-18T16:55:40.000+04:00
Improve error handling in htmlToText function
diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go
@@ -1,6 +1,7 @@
 package httpx
 
 import (
+	"context"
 	"crypto/tls"
 	"fmt"
 	"io"
@@ -25,7 +26,6 @@ import (
 	pdhttputil "github.com/projectdiscovery/utils/http"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 	urlutil "github.com/projectdiscovery/utils/url"
-	"golang.org/x/net/context"
 	"golang.org/x/net/http2"
 )
 
diff --git a/common/pagetypeclassifier/pagetypeclassifier.go b/common/pagetypeclassifier/pagetypeclassifier.go
@@ -14,26 +14,23 @@ type PageTypeClassifier struct {
 	classifier *naive_bayes.NaiveBayesClassifier
 }
 
-func New() *PageTypeClassifier {
+func New() (*PageTypeClassifier, error) {
 	classifier, err := naive_bayes.NewClassifierFromFileData(classifierData)
 	if err != nil {
-		panic(err)
+		return nil, err
 	}
-	return &PageTypeClassifier{classifier: classifier}
+	return &PageTypeClassifier{classifier: classifier}, nil
 }
 
 func (n *PageTypeClassifier) Classify(html string) string {
-	text := htmlToText(html)
-	if text == "" {
+	text, err := htmlToText(html)
+	if err != nil || text == "" {
 		return "other"
 	}
 	return n.classifier.Classify(text)
 }
 
-func htmlToText(html string) string {
-	text, err := htmltomarkdown.ConvertString(html)
-	if err != nil {
-		panic(err)
-	}
-	return text
+// htmlToText safely converts HTML to text and protects against panics from Go's HTML parser.
+func htmlToText(html string) (string, error) {
+	return htmltomarkdown.ConvertString(html)
 }
diff --git a/common/pagetypeclassifier/pagetypeclassifier_test.go b/common/pagetypeclassifier/pagetypeclassifier_test.go
@@ -3,19 +3,22 @@ package pagetypeclassifier
 import (
 	"testing"
 
-	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestPageTypeClassifier(t *testing.T) {
 
 	t.Run("test creation of new PageTypeClassifier", func(t *testing.T) {
-		epc := New()
-		assert.NotNil(t, epc)
+		epc, err := New()
+		require.NoError(t, err)
+		require.NotNil(t, epc)
 	})
 
 	t.Run("test classification non error page text", func(t *testing.T) {
-		epc := New()
-		assert.Equal(t, "nonerror", epc.Classify(`<!DOCTYPE html>
+		epc, err := New()
+		require.NoError(t, err)
+		require.NotNil(t, epc)
+		require.Equal(t, "nonerror", epc.Classify(`<!DOCTYPE html>
 		<html lang="en">
 		<head>
 			<meta charset="UTF-8">
@@ -30,8 +33,10 @@ func TestPageTypeClassifier(t *testing.T) {
 	})
 
 	t.Run("test classification on error page text", func(t *testing.T) {
-		epc := New()
-		assert.Equal(t, "error", epc.Classify(`<!DOCTYPE html>
+		epc, err := New()
+		require.NoError(t, err)
+		require.NotNil(t, epc)
+		require.Equal(t, "error", epc.Classify(`<!DOCTYPE html>
 		<html>
 		<head>
 			<title>Error 403: Forbidden</title>
diff --git a/runner/runner.go b/runner/runner.go
@@ -385,7 +385,11 @@ func New(options *Options) (*Runner, error) {
 	}
 
 	runner.simHashes = gcache.New[uint64, struct{}](1000).ARC().Build()
-	runner.pageTypeClassifier = pagetypeclassifier.New()
+	pageTypeClassifier, err := pagetypeclassifier.New()
+	if err != nil {
+		return nil, err
+	}
+	runner.pageTypeClassifier = pageTypeClassifier
 
 	if options.HttpApiEndpoint != "" {
 		apiServer := NewServer(options.HttpApiEndpoint, options)

Original file line number	Diff line number	Diff line change
`@@ -14,26 +14,23 @@ type PageTypeClassifier struct {`
`14`	`14`	`classifier *naive_bayes.NaiveBayesClassifier`
`15`	`15`	`}`
`16`	`16`
`17`		`-func New() *PageTypeClassifier {`
	`17`	`+func New() (*PageTypeClassifier, error) {`
`18`	`18`	`classifier, err := naive_bayes.NewClassifierFromFileData(classifierData)`
`19`	`19`	`if err != nil {`
`20`		`- panic(err)`
	`20`	`+ return nil, err`
`21`	`21`	`}`
`22`		`- return &PageTypeClassifier{classifier: classifier}`
	`22`	`+ return &PageTypeClassifier{classifier: classifier}, nil`
`23`	`23`	`}`
`24`	`24`
`25`	`25`	`func (n *PageTypeClassifier) Classify(html string) string {`
`26`		`- text := htmlToText(html)`
`27`		`- if text == "" {`
	`26`	`+ text, err := htmlToText(html)`
	`27`	`+ if err != nil \|\| text == "" {`
`28`	`28`	`return "other"`
`29`	`29`	`}`
`30`	`30`	`return n.classifier.Classify(text)`
`31`	`31`	`}`
`32`	`32`
`33`		`-func htmlToText(html string) string {`
`34`		`- text, err := htmltomarkdown.ConvertString(html)`
`35`		`- if err != nil {`
`36`		`- panic(err)`
`37`		`- }`
`38`		`- return text`
	`33`	`+// htmlToText safely converts HTML to text and protects against panics from Go's HTML parser.`
	`34`	`+func htmlToText(html string) (string, error) {`
	`35`	`+ return htmltomarkdown.ConvertString(html)`
`39`	`36`	`}`