v1: dependency parser

.gitignore

@@ -1,3 +1,3 @@
 .env
-node_modules
-source/*.zip
+.vscode
+__debug_bin*

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023 Julian Claus
+Copyright (c) 2025 Julian Claus
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -0,0 +1,21 @@
+GOLANGCILINT_VERSION := v2.5.0
+GO_PATH := $(shell go env GOPATH)/bin
+GOLANGCILINT_BIN := $(GO_PATH)/golangci-lint
+
+.PHONY: lint fmt test
+
+all: lint fmt test
+
+$(GOLANGCILINT_BIN):
+	@if ! test -x $(GOLANGCILINT_BIN) || ! $(GOLANGCILINT_BIN) --version | grep -q $(GOLANGCILINT_VERSION); then \
+		curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GO_PATH) $(GOLANGCILINT_VERSION); \
+	fi
+
+fmt: $(GOLANGCILINT_BIN)
+	$(GOLANGCILINT_BIN) fmt ./... -v
+
+lint: $(GOLANGCILINT_BIN)
+	@$(GOLANGCILINT_BIN) run $(LINT_FLAGS) ./... --fix -v
+
+test:
+	go test -v ./... -short

README.md

@@ -1,13 +1,2 @@
 # entityscrape
 
-This a social experiment which shows the mean distance between part of speeches
-(e. g. adjectives or nouns) in news articles (like from NBC or CNN) and randomly
-selected entities (like Xi Jingping or ISIS). Go straight to the
-[website](https://ndabap.github.io/entityscrape/index.html)!
-
-The Go package [assocentity](https://github.com/ndabAP/assocentity) was
-used for creating this experiment. You can create new ones by updating the
-`source/entities.txt` file and run the CLI with the provided Visual Studio Code
-debug configuration.
-
-**Data source**: [dai, tianru, 2017, "News Articles", Harvard Dataverse, V1](https://dataverse.harvard.edu/dataset.xhtml?persistentId=doi:10.7910/DVN/GMFCTR).

cases/conduct.go

@@ -0,0 +1,192 @@
+package cases
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"log/slog"
+	"math/rand/v2"
+	"os"
+	"strings"
+	"unicode"
+
+	"github.com/ndabAP/assocentity"
+	"github.com/ndabAP/assocentity/tokenize"
+	"github.com/ndabAP/assocentity/tokenize/nlp"
+	"github.com/ndabAP/entityscrape/translator"
+	"golang.org/x/text/language"
+)
+
+func (study study[samples, aggregated]) Conduct(ctx context.Context) error {
+	slog.Debug("processing subjects", "n", len(study.Subjects))
+
+	translator := translator.NewGoogle(ctx, GoogleCloudSvcAccountKey)
+	for subject, analyses := range study.Subjects {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+		}
+
+		slog.Debug("processing analyses", "subject", subject)
+		var (
+			entity    = analyses.Entity
+			ext       = analyses.Ext
+			feats     = analyses.Feats
+			filenames = analyses.Filenames
+			reduct    = analyses.Reduct
+			lang      = analyses.Language
+			parser    = analyses.Parser
+		)
+		tokenizer := nlp.New(GoogleCloudSvcAccountKey, lang.String())
+		analyses, err := study.analysis(
+			ctx,
+			entity,
+			filenames,
+			parser,
+			reduct,
+			tokenizer,
+			feats,
+		)
+		if err != nil {
+			return err
+		}
+		slog.Debug("analysis done")
+
+		slog.Debug("collecting samples")
+		samples := study.collect(analyses)
+		slog.Debug("sample collection done")
+
+		slog.Debug("aggregating samples")
+		aggregated := study.aggregate(samples)
+		slog.Debug("aggregation done")
+
+		slog.Debug("reporting aggregation")
+		translator := func(w []string) ([]string, error) {
+			switch lang {
+			case language.English:
+				slog.Debug("skipping translation for English")
+				return w, nil
+			default:
+			}
+
+			return translator.Translate(w, lang, language.English)
+		}
+		if err := func() error {
+			pref := strings.Map(func(r rune) rune {
+				switch {
+				case r >= 'a' && r <= 'z':
+					return r
+				case r >= 'A' && r <= 'Z':
+					return unicode.ToLower(r)
+				case r == ' ', r == '-':
+					return '_'
+				default:
+					return -1
+				}
+			}, subject)
+			writer, err := study.store.NewWriter(pref, ext)
+			if err != nil {
+				return err
+			}
+			//nolint:errcheck
+			defer writer.Close()
+
+			if err := study.report(aggregated, translator, writer); err != nil {
+				return err
+			}
+
+			return nil
+		}(); err != nil {
+			return err
+		}
+		slog.Debug("reporting done", "subject", subject)
+	}
+
+	return nil
+}
+
+func (study study[samples, aggregated]) analysis(
+	ctx context.Context,
+	entity,
+	filenames []string,
+	parser Parser,
+	reduct bool,
+	tokenizer tokenize.Tokenizer,
+	feats tokenize.Features,
+) (
+	assocentity.Analyses,
+	error,
+) {
+	slog.Debug("parsing files", "n", len(filenames))
+	if reduct {
+		slog.Debug("entity reduct enabled")
+	}
+
+	var (
+		texts = make([]string, 0, len(filenames))
+
+		textChan = make(chan []byte, 50)
+		errChan  = make(chan error, 1)
+	)
+
+	// Consumer
+	go func() {
+		defer close(errChan)
+
+		for text := range textChan {
+			n := rand.Uint64N(100)
+			if n >= SampleRate {
+				continue
+			}
+
+			var err error
+			if reduct {
+				text, err = study.reduct(text, entity)
+				if errors.Is(err, errEntityNotFound) {
+					continue
+				}
+			}
+			if err != nil {
+				errChan <- err
+				return
+			}
+			texts = append(texts, string(text))
+		}
+	}()
+	// Producer
+	go func() {
+		defer close(textChan)
+
+		for _, filename := range filenames {
+			file, err := os.Open(filename)
+			if err != nil {
+				errChan <- err
+				return
+			}
+			for err := range parser(file, textChan) {
+				if errors.Is(err, bufio.ErrTooLong) {
+					continue
+				}
+				errChan <- err
+			}
+			//nolint:errcheck
+			_ = file.Close()
+		}
+	}()
+
+	select {
+	case <-ctx.Done():
+		return assocentity.Analyses{}, ctx.Err()
+
+	case err := <-errChan:
+		if err != nil {
+			return assocentity.Analyses{}, err
+		}
+	}
+	slog.Debug("texts sampled and parsed", "n", len(texts))
+
+	slog.Debug("creating analyses")
+	src := assocentity.NewSource(entity, texts)
+	return src.Analyses(ctx, tokenizer, feats, assocentity.NFKC)
+}

cases/corpus.go

@@ -0,0 +1,35 @@
+package cases
+
+import (
+	"io/fs"
+	"path/filepath"
+)
+
+// corpusDir is the absolute directory to the corpus.
+var corpusDir string
+
+func SetCorpusRootDir(base string) {
+	corpusDir = filepath.Join(base, "corpus")
+}
+
+func GetCorpusRootDir() string {
+	return corpusDir
+}
+
+func WalkCorpus(corpus string, fn func(filename string) error) error {
+	root := filepath.Join(corpusDir, corpus)
+	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			return nil
+		}
+		switch filepath.Ext(path) {
+		case ".gitignore", ".gitkeep":
+			return nil
+		}
+		return fn(path)
+	})
+	return err
+}