projects/pipelines/edit_tree_lemmatizer/project.yml at v3 · explosion/projects · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
title: "Demo the trainable edit-tree lemmatizer"

vars:
  lang: "nl"
  treebank: "UD_Dutch-Alpino"
  train_name: "nl_alpino-ud-train"
  dev_name: "nl_alpino-ud-dev"
  test_name: "nl_alpino-ud-test"
  gpu: -1

directories: ["configs", "assets", "corpus", "training"]

assets:
  - dest: "assets/${vars.treebank}"
    git:
      repo: "https://github.com/UniversalDependencies/${vars.treebank}"
      branch: "master"
      path: ""

workflows:
  all:
    - preprocess
    - create-config
    - train
    - evaluate

commands:
  - name: "preprocess"
    help: "Convert data to spaCy format"
    script:
      - "mkdir -p corpus/${vars.treebank}"
      - "python -m spacy convert assets/${vars.treebank}/${vars.train_name}.conllu corpus/${vars.treebank}/ --n-sents 10"
      - "mv corpus/${vars.treebank}/${vars.train_name}.spacy corpus/${vars.treebank}/train.spacy"
      - "python -m spacy convert assets/${vars.treebank}/${vars.dev_name}.conllu corpus/${vars.treebank}/ --n-sents 10"
      - "mv corpus/${vars.treebank}/${vars.dev_name}.spacy corpus/${vars.treebank}/dev.spacy"
      - "python -m spacy convert assets/${vars.treebank}/${vars.test_name}.conllu corpus/${vars.treebank}/ --n-sents 10"
      - "mv corpus/${vars.treebank}/${vars.test_name}.spacy corpus/${vars.treebank}/test.spacy"
    deps:
      - "assets/${vars.treebank}/"
    outputs:
      - "corpus/${vars.treebank}/train.spacy"
      - "corpus/${vars.treebank}/dev.spacy"
      - "corpus/${vars.treebank}/test.spacy"

  - name: "create-config"
    help: "Create a config"
    script:
      - "python -m spacy init config -l ${vars.lang} -p trainable_lemmatizer configs/${vars.treebank}/config.cfg -F"

  - name: "train"
    help: "Train the lemmatizer"
    script:
      - "python -m spacy train configs/${vars.treebank}/config.cfg --output training/${vars.treebank} --gpu-id ${vars.gpu} --paths.train corpus/${vars.treebank}/train.spacy --paths.dev corpus/${vars.treebank}/dev.spacy"
    deps:
      - "configs/${vars.treebank}/config.cfg"
      - "corpus/${vars.treebank}/train.spacy"
      - "corpus/${vars.treebank}/dev.spacy"
    outputs:
      - "training/${vars.treebank}/model-best"

  - name: "evaluate"
    help: "Evaluate the lemmatization model evaluate on the test corpus."
    script:
      - "python -m spacy evaluate training/${vars.treebank}/model-best corpus/${vars.treebank}/test.spacy"
    deps:
      - "training/${vars.treebank}/model-best"