constrained-fever/code/bert-concat/run_eval.sh at main · adithya7/constrained-fever · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
#!/bin/bash

# for the reasoning architecture `bert-concat`
#   for each of [`baseline`, `cwa`, `skip-fact`] encoder versions
#     for each of [`Std.`, `Symm.`, `Anon.`] evaluation sets

# ------------------------------------------------- #

mkdir -p eval_predictions

CUDA_VISIBLE_DEVICES=0 python -W ignore train.py eval \
  -input ../../eval/dev_retrievedsents_leetal.json \
  -model ../../checkpoints/fever_models/bert/baseline \
  -task fever \
  -bs 16 \
  -out eval_predictions/baseline_std_dev.jsonl

CUDA_VISIBLE_DEVICES=0 python -W ignore train.py eval \
  -input ../../eval/fever_symmetric_dev_kgat.jsonl \
  -model ../../checkpoints/fever_models/bert/baseline \
  -task fever \
  -bs 16 \
  -out eval_predictions/baseline_symm_dev.jsonl

CUDA_VISIBLE_DEVICES=0 python -W ignore train.py eval \
  -input ../../eval/anon_bert_dev.json \
  -model ../../checkpoints/fever_models/bert/baseline \
  -task fever \
  -bs 16 \
  -out eval_predictions/baseline_anon_dev.jsonl

## ------------------------------------------------- #

CUDA_VISIBLE_DEVICES=0 python -W ignore train.py eval \
  -input ../../eval/dev_retrievedsents_leetal.json \
  -model ../../checkpoints/fever_models/bert/cwa \
  -task fever \
  -bs 16 \
  -out eval_predictions/cwa_std_dev.jsonl

CUDA_VISIBLE_DEVICES=0 python -W ignore train.py eval \
  -input ../../eval/fever_symmetric_dev_kgat.jsonl \
  -model ../../checkpoints/fever_models/bert/cwa \
  -task fever \
  -bs 16 \
  -out eval_predictions/cwa_symm_dev.jsonl

CUDA_VISIBLE_DEVICES=0 python -W ignore train.py eval \
  -input ../../eval/anon_bert_dev.json \
  -model ../../checkpoints/fever_models/bert/cwa \
  -task fever \
  -bs 16 \
  -out eval_predictions/cwa_anon_dev.jsonl

## ------------------------------------------------- #

CUDA_VISIBLE_DEVICES=0 python -W ignore train.py eval \
  -input ../../eval/dev_retrievedsents_leetal.json \
  -model ../../checkpoints/fever_models/bert/skip-fact \
  -task fever \
  -bs 16 \
  -out eval_predictions/skip-fact_std_dev.jsonl

CUDA_VISIBLE_DEVICES=0 python -W ignore train.py eval \
  -input ../../eval/fever_symmetric_dev_kgat.jsonl \
  -model ../../checkpoints/fever_models/bert/skip-fact \
  -task fever \
  -bs 16 \
  -out eval_predictions/skip-fact_symm_dev.jsonl

CUDA_VISIBLE_DEVICES=0 python -W ignore train.py eval \
  -input ../../eval/anon_bert_dev.json \
  -model ../../checkpoints/fever_models/bert/skip-fact \
  -task fever \
  -bs 16 \
  -out eval_predictions/skip-fact_anon_dev.jsonl

# ------------------------------------------------- #