KATI/scraper.py at master · kannera/KATI · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
from korpreader import *
import json


def scrape(regex, cqp_query):

    output_cats = ("lemma", "word", "pos", "msd", "lemmacomp")
    filename = "vai_all/context_word_corpus/"+regex

    initiate_korp_query(regex, ["klk_fi_"+str(i) for i in range(1980, 2000)], filename, cqp_query, output_cats, sample_size=10000, rewrite=True)

with open("vai_all/word_frequencies_by_synpattern.json", "r", encoding="utf-8") as f:
    data = json.load(f)


skips = ("nurmi", "mato", "ilma", "lampi", "jajalka", "kär", "selkää", "vatsaä", "sydänä", "niveli", "olvi")
start_replacements = ("hoito", "huolto", "pöllö", "koti", "poika", "akka")
lemma_comps = ("pikku", "käsi", "selkä", "jalka", "vatsa", "tyrä", "silmä", "munuainen", "nivunen", "kone")

for word in data:
    cqp_query = "lemma"
    if word not in skips:

        if "COMPOUND_ATTRIBUTE" in data[word]:
            cqp_query = "lemmacomp"
            regex = "ATT_"+word
            scrape(regex, cqp_query)

        if  "_POST_COMP_ATTRIBUTE" in data[word]:
            if word in lemma_comps: cqp_query = "lemmacomp"
            regex = "HEAD_"+word
            scrape(regex, cqp_query)