project.yml

title: "Comparing embedding layers in spaCy"
description: |
  This project contains the code to reproduce the results of the
  [Multi hash embeddings in spaCy](https://arxiv.org/abs/2212.09255) technical report by Explosion.
  The `project.yml` provides commands to download and preprocess the data sets as well as to
  run the training and evaluation procedures. Different configuration of `vars` correspond
  to different experiments in the report.

  There are a few scripts included that were used during the technical report writing process
  to run experiments in bulk and summarize the results. 
  The `scripts/run_experiments.py` runs multiple experiments one after the other
  by constructing and running `spacy project run` commands. The module
  `scipts/collate_results.py` summarizes the results of the same trials with multiple seeds.
  Finally, `scripts/plot_results.py` was used to produce the visualizations in the report.
  These are all small command line apps and you can learn more about the usage as usual with the
  `--help` flag.

  The `rows` argument for the `train-adjusted-rows` command is provided as a list and 
  this may lead to errors on Windows machines. Unfortunately, this might lead not being able to
  reproduce the `MultiHashEmbed (adjusted)` experiments from the paper on Windows using `run_experiment.py`.
  This is due to known issue with handling quotes on Windows and is something we are looking into.
  The config files can be edited by manually or in some other way to adjust the number of rows for 
  the hash embedding layers. We apologize for the inconvenience.

vars:
  ner_config: "multihashembed"
  dataset: "archaeo"
  language: "nl"
  vectors: "nl_core_news_lg"
  tables_path: "tables"
  metrics_dir: "metrics"
  attrs: null
  rows: [5000, 2500, 2500, 2500]
  include_static_vectors: true
  min_freq: 10
  gpu_id: -1
  seed: 42
  batch_size: 1000
  num_hashes: 4

directories:
  - "assets"
  - "configs"
  - "corpus"
  - "scripts"
  - "training"
  - "metrics"
  - "vectors"
  - "tables"
  - "debug"

workflows:
  setup:
    - "download-models"
    - "init-fasttext"
    - "prepare-datasets"
    - "make-tables"

  trial:
    - "init-labels"
    - "train"
    - "evaluate"
    - "evaluate-seen-only"
    - "evaluate-unseen-only"


assets:
  - dest: "assets/fasttext.en.gz"
    description: "English fastText vectors."
    url: https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.en.300.vec.gz
  - dest: "assets/fasttext.es.gz"
    description: "Spanish fastText vectors."
    url: https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.es.300.vec.gz
  - dest: "assets/fasttext.nl.gz"
    description: "Dutch fastText vectors."
    url: https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.nl.300.vec.gz
  - dest: "span-labeling-datasets"
    git:
      repo: "https://github.com/explosion/projects"
      branch: "v3"
      path: "benchmarks/span-labeling-datasets"

commands:
  
  - name: "prepare-datasets"
    help: "Download and preprocess all available data sets using the span-labeling-datasets project."
    script:
      - "python -m spacy project assets span-labeling-datasets"
      - "python -m spacy project run all span-labeling-datasets"
      - "python -m spacy project run clean-archaeo span-labeling-datasets"
      - "python -m spacy project run generate-unseen span-labeling-datasets"
      - "python -m scripts.cli_util copy-contents span-labeling-datasets/corpus/ner corpus"
      - "python -m scripts.cli_util copy-contents span-labeling-datasets/unseen corpus"
      - "python -m scripts.cli_util remove span-labeling-datasets"
    deps:
      - span-labeling-datasets/
    outputs: 
      - corpus/anem-dev-seen.spacy
      - corpus/anem-dev-unseen.spacy
      - corpus/anem-dev.spacy
      - corpus/anem-test-seen.spacy
      - corpus/anem-test-unseen.spacy
      - corpus/anem-test.spacy
      - corpus/anem-train.spacy
      - corpus/archaeo-dev-seen.spacy
      - corpus/archaeo-dev-unseen.spacy
      - corpus/archaeo-dev.spacy
      - corpus/archaeo-test-seen.spacy
      - corpus/archaeo-test-unseen.spacy
      - corpus/archaeo-test.spacy
      - corpus/archaeo-train.spacy
      - corpus/es-conll-dev-seen.spacy
      - corpus/es-conll-dev-unseen.spacy
      - corpus/es-conll-dev.spacy
      - corpus/es-conll-test-seen.spacy
      - corpus/es-conll-test-unseen.spacy
      - corpus/es-conll-test.spacy
      - corpus/es-conll-train.spacy
      - corpus/labels
      - corpus/nl-conll-dev-seen.spacy
      - corpus/nl-conll-dev-unseen.spacy
      - corpus/nl-conll-dev.spacy
      - corpus/nl-conll-test-seen.spacy
      - corpus/nl-conll-test-unseen.spacy
      - corpus/nl-conll-test.spacy
      - corpus/nl-conll-train.spacy
      - corpus/restaurant-dev-seen.spacy
      - corpus/restaurant-dev-unseen.spacy
      - corpus/restaurant-dev.spacy
      - corpus/restaurant-test-seen.spacy
      - corpus/restaurant-test-unseen.spacy
      - corpus/restaurant-test.spacy
      - corpus/restaurant-train.spacy
      - corpus/wnut17-dev-seen.spacy
      - corpus/wnut17-dev-unseen.spacy
      - corpus/wnut17-dev.spacy
      - corpus/wnut17-test-seen.spacy
      - corpus/wnut17-test-unseen.spacy
      - corpus/wnut17-test.spacy
      - corpus/wnut17-train.spacy


  - name: "download-models"
    help: "Download spaCy models for their word-embeddings."
    script:
      - "python -m spacy download en_core_web_lg"
      - "python -m spacy download es_core_news_lg"
      - "python -m spacy download nl_core_news_lg"
      - "python -m spacy download fi_core_news_lg"


  - name: "init-fasttext"
    help: "Initialize the FastText vectors."
    script:
      - "python -m scripts.cli_util unzip assets/fasttext.en.gz"
      - "python -m scripts.cli_util unzip assets/fasttext.es.gz"
      - "python -m scripts.cli_util unzip assets/fasttext.nl.gz"
      - "python -m spacy init vectors en assets/fasttext.en vectors/fasttext-en"
      - "python -m spacy init vectors es assets/fasttext.es vectors/fasttext-es"
      - "python -m spacy init vectors nl assets/fasttext.nl vectors/fasttext-nl"
    deps:
      - assets/fasttext.en.gz
      - assets/fasttext.es.gz
      - assets/fasttext.nl.gz
    outputs:
      - vectors/fasttext-en
      - vectors/fasttext-es
      - vectors/fasttext-nl

  - name: "make-tables"
    help: "Pre-compute token-to-id tables for MultiEmbed."
    script:
      - >-
        python -m scripts.token_map
        configs/multiembed.cfg
        ${vars.tables_path}/${vars.dataset}
        --min-freq ${vars.min_freq}
        --paths.train corpus/${vars.dataset}-train.spacy
        --paths.dev corpus/${vars.dataset}-dev.spacy
        --code scripts/custom_components.py
        --paths.tables ${vars.tables_path}
    deps:
      - corpus/${vars.dataset}-train.spacy
      - corpus/${vars.dataset}-dev.spacy
    outputs:
      - ${vars.tables_path}/${vars.dataset}


  - name: "init-labels"
    help: "Initialize labels first before training"
    script:
      - >-
        python -m spacy init labels
        configs/${vars.ner_config}.cfg
        corpus/labels/${vars.dataset}.labels
        --verbose 
        --paths.train corpus/${vars.dataset}-train.spacy
        --paths.dev corpus/${vars.dataset}-dev.spacy
        --paths.tables tables/${vars.dataset}/${vars.dataset}-train.tables
        --gpu-id ${vars.gpu_id}
        --code scripts/custom_components.py
    deps:
      - corpus/${vars.dataset}-train.spacy
      - corpus/${vars.dataset}-dev.spacy
    outputs:
      - corpus/labels/${vars.dataset}.labels

  - name: "train"
    help: "Train NER model."
    script:
      - >-
        python -m spacy train
        configs/${vars.ner_config}.cfg
        --nlp.lang ${vars.language}
        --output training/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/
        --paths.train corpus/${vars.dataset}-train.spacy
        --paths.dev corpus/${vars.dataset}-dev.spacy
        --paths.vectors ${vars.vectors}
        --paths.tables ${vars.tables_path}/${vars.dataset}/${vars.dataset}-train.tables
        --components.tok2vec.model.embed.include_static_vectors ${vars.include_static_vectors}
        --nlp.batch_size ${vars.batch_size}
        --system.seed ${vars.seed}
        --gpu-id ${vars.gpu_id}
        --code scripts/custom_components.py
    deps:
      - corpus/${vars.dataset}-train.spacy
      - corpus/${vars.dataset}-dev.spacy
    outputs:
      - training/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/model-last
      - training/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/model-best

  - name: "train-adjust-rows"
    help: "Train NER model with adjustable number of rows."
    script:
      - >-
        python -m spacy train
        configs/${vars.ner_config}.cfg
        --nlp.lang ${vars.language}
        --output training/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/
        --paths.train corpus/${vars.dataset}-train.spacy
        --paths.dev corpus/${vars.dataset}-dev.spacy
        --paths.vectors ${vars.vectors}
        --paths.tables ${vars.tables_path}/${vars.dataset}/${vars.dataset}-train.tables
        --components.tok2vec.model.embed.include_static_vectors ${vars.include_static_vectors}
        --components.tok2vec.model.embed.rows '${vars.rows}'
        --nlp.batch_size ${vars.batch_size}
        --system.seed ${vars.seed}
        --gpu-id ${vars.gpu_id}
        --code scripts/custom_components.py
    deps:
      - corpus/${vars.dataset}-train.spacy
      - corpus/${vars.dataset}-dev.spacy
    outputs:
      - training/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/model-last
      - training/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/model-best

  - name: "train-hash"
    help: "Train NER model with different number of hash functions. (only works with the multifewerhashembed.cfg)"
    script:
      - python -m scripts.cli_util expected-arg config multifewerhashembed ${vars.ner_config}
      - >-
        python -m spacy train
        configs/${vars.ner_config}.cfg
        --nlp.lang ${vars.language}
        --output training/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/
        --paths.train corpus/${vars.dataset}-train.spacy
        --paths.dev corpus/${vars.dataset}-dev.spacy
        --paths.vectors ${vars.vectors}
        --paths.tables ${vars.tables_path}/${vars.dataset}/${vars.dataset}-train.tables
        --components.tok2vec.model.embed.include_static_vectors ${vars.include_static_vectors}
        --components.tok2vec.model.embed.num_hashes ${vars.num_hashes}
        --nlp.batch_size ${vars.batch_size}
        --system.seed ${vars.seed}
        --gpu-id ${vars.gpu_id}
        --code scripts/custom_components.py
    deps:
      - corpus/${vars.dataset}-train.spacy
      - corpus/${vars.dataset}-dev.spacy
    outputs:
      - training/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/model-last
      - training/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/model-best

  - name: "evaluate"
    help: "Evaluate NER model."
    script:
      - mkdir -p ${vars.metrics_dir}/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/
      - >-
        python -m spacy evaluate
        training/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/model-best
        corpus/${vars.dataset}-test.spacy
        --output ${vars.metrics_dir}/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/scores-test.json
        --gpu-id ${vars.gpu_id}
        --code scripts/custom_components.py
      - >-
        python -m spacy evaluate
        training/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/model-best
        corpus/${vars.dataset}-dev.spacy
        --output ${vars.metrics_dir}/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/scores-dev.json
        --gpu-id ${vars.gpu_id}
        --code scripts/custom_components.py
    deps:
      - training/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/model-best
      - corpus/${vars.dataset}-dev.spacy
    outputs:
      - ${vars.metrics_dir}/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/scores-test.json
      - ${vars.metrics_dir}/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/scores-dev.json
  
  - name: "evaluate-seen-only"
    help: "Evaluate NER model on the dev and tests sets only considering entities that appear in the training set."
    script:
      - >-
        python -m spacy evaluate
        training/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/model-best
        corpus/${vars.dataset}-dev-seen.spacy
        --output ${vars.metrics_dir}-dev-seen/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/scores.json
        --gpu-id ${vars.gpu_id}
        --code scripts/custom_components.py
      - mkdir -p ${vars.metrics_dir}-test-seen/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/
      - >-
        python -m spacy evaluate
        training/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/model-best
        corpus/${vars.dataset}-test-seen.spacy
        --output ${vars.metrics_dir}-test-seen/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/scores.json
        --gpu-id ${vars.gpu_id}
        --code scripts/custom_components.py
    deps:
      - training/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/model-best
      - corpus/${vars.dataset}-dev-seen.spacy
      - corpus/${vars.dataset}-test-seen.spacy
    outputs:
      - ${vars.metrics_dir}-dev-seen/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/scores.json
      - ${vars.metrics_dir}-test-seen/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/scores.json


  - name: "evaluate-unseen-only"
    help: "Evaluate NER model on the dev and tests sets only considering entities that did not appear in the training set."
    script:
      - mkdir -p ${vars.metrics_dir}-dev-unseen/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/
      - >-
        python -m spacy evaluate
        training/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/model-best
        corpus/${vars.dataset}-dev-unseen.spacy
        --output ${vars.metrics_dir}-dev-unseen/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/scores.json
        --gpu-id ${vars.gpu_id}
        --code scripts/custom_components.py
      - mkdir -p ${vars.metrics_dir}-test-unseen/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/
      - >-
        python -m spacy evaluate
        training/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/model-best
        corpus/${vars.dataset}-test-unseen.spacy
        --output ${vars.metrics_dir}-test-unseen/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/scores.json
        --gpu-id ${vars.gpu_id}
        --code scripts/custom_components.py
      - mkdir -p ${vars.metrics_dir}-dev-seen/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/
    deps:
      - training/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/model-best
      - corpus/${vars.dataset}-dev-unseen.spacy
      - corpus/${vars.dataset}-test-unseen.spacy
    outputs:
      - ${vars.metrics_dir}-dev-unseen/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/scores.json
      - ${vars.metrics_dir}-test-unseen/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/scores.json