feat: add entities, improve logging

Author: ndabAP

go.mod

@@ -2,7 +2,7 @@ module github.com/ndabAP/entityscrape
 
 go 1.18
 
-require github.com/ndabAP/assocentity/v12 v12.0.0
+require github.com/ndabAP/assocentity/v12 v12.2.0
 
 require (
 	cloud.google.com/go/compute v1.12.1 // indirect

go.sum

@@ -52,6 +52,8 @@ github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57Q
 github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
 github.com/ndabAP/assocentity/v12 v12.0.0 h1:fCwj0S+jKlp2l4pYWAjl8dTey1j3RInx22HHGXlZ7FA=
 github.com/ndabAP/assocentity/v12 v12.0.0/go.mod h1:4TGqBbxSnNKX7odkQGGSYzdtIeJSKkz5ZSyI/Zf0QmI=
+github.com/ndabAP/assocentity/v12 v12.2.0 h1:SDATlGnfKvWR1Ei6WRxYOPk+mv1h0lIDb9xKXAGw9yA=
+github.com/ndabAP/assocentity/v12 v12.2.0/go.mod h1:4TGqBbxSnNKX7odkQGGSYzdtIeJSKkz5ZSyI/Zf0QmI=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

main.go

@@ -84,6 +84,8 @@ func scrape(texts, entities []string, tokenizer tokenize.Tokenizer) error {
 	entity := entities[0]
 	log.Printf("entity=%s", entity)
 
+	l := log.New(os.Stderr, entity+":", 0)
+
 	// Ignore articles without entity
 	temp := texts[:0]
 	for _, text := range texts {
@@ -92,9 +94,9 @@ func scrape(texts, entities []string, tokenizer tokenize.Tokenizer) error {
 		}
 	}
 	texts = temp
-	log.Printf("len(texts)=%d", len(texts))
+	l.Printf("len(texts)=%d", len(texts))
 
-	poS := tokenize.ADJ | tokenize.ADP | tokenize.ADV | tokenize.CONJ | tokenize.DET | tokenize.NOUN | tokenize.NUM | tokenize.PRON | tokenize.VERB
+	poS := tokenize.ADJ | tokenize.ADP | tokenize.ADV | tokenize.CONJ | tokenize.DET | tokenize.NOUN | tokenize.NUM | tokenize.PRON | tokenize.PRT | tokenize.VERB
 	meanN, err := assocentity.MeanN(
 		context.Background(),
 		tokenizer,
@@ -103,18 +105,18 @@ func scrape(texts, entities []string, tokenizer tokenize.Tokenizer) error {
 		entities,
 	)
 	if err != nil {
-		log.Fatal(err)
+		l.Fatal(err)
 	}
 
-	log.Printf("len(meanN)=%d", len(meanN))
+	l.Printf("len(meanN)=%d", len(meanN))
 
 	if len(meanN) == 0 {
-		log.Print("no meanN found, exiting")
+		l.Print("no meanN found, exiting")
 		os.Exit(0)
 	}
 
 	// Convert to slice to make it sortable
-	log.Println("convert to slice")
+	l.Println("convert to slice")
 	type meanNVal struct {
 		dist float64
 		tok  tokenize.Token
@@ -129,7 +131,7 @@ func scrape(texts, entities []string, tokenizer tokenize.Tokenizer) error {
 	}
 
 	// Sort by closest distance
-	log.Println("sort by pos and distance")
+	l.Println("sort by pos and distance")
 	sort.Slice(meanNVals, func(i, j int) bool {
 		if meanNVals[i].tok.PoS != meanNVals[j].tok.PoS {
 			return meanNVals[i].tok.PoS < meanNVals[j].tok.PoS
@@ -138,7 +140,7 @@ func scrape(texts, entities []string, tokenizer tokenize.Tokenizer) error {
 	})
 
 	// Top 10 per pos
-	log.Println("limit top 10")
+	l.Println("limit top 10")
 	type topMeanNVal struct {
 		Dist float64 `json:"distance"`
 		Pos  string  `json:"pos"`
@@ -160,18 +162,18 @@ func scrape(texts, entities []string, tokenizer tokenize.Tokenizer) error {
 
 		poSCounter[meanNVal.tok.PoS] += 1
 	}
-	log.Printf("len(topMeanNVals)=%d", len(topMeanNVals))
+	l.Printf("len(topMeanNVals)=%d", len(topMeanNVals))
 
 	// Write top 10 to disk
-	log.Println("write to disk")
+	l.Println("write to disk")
 	file, err := json.MarshalIndent(&topMeanNVals, "", " ")
 	if err != nil {
-		log.Fatal(err)
+		l.Fatal(err)
 	}
 	name := url.QueryEscape(strings.ToLower(entity))
 	path := filepath.Join("web/public", name+".json")
 	if err := os.WriteFile(path, file, 0600); err != nil {
-		log.Fatal(err)
+		l.Fatal(err)
 	}
 
 	return nil

source/entities.txt

@@ -1,6 +1,7 @@
+Barack Obama,Obama
+Brexit
 Donald Trump,Trump
 ISIS
-Barack Obama,Obama
+North Korea,North Korean,North Korea
 Vladimir Putin,Putin
-Xi Jinping,Xi,Jinping
-North Korea,North Korean,North Korea
+Xi Jinping,Xi,Jinping