Skip to content

Commit 86b41a5

Browse files
pengyuniepengyunie
authored
Merge pull request #6 from EngineeringSoftware/updating-interface
Major changes towards v1.1.0+8.10.0, including interface changes etc.
2 parents dc9a529 + 9db22bb commit 86b41a5

29 files changed

+1241
-1492
lines changed

‎.gitignore‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
# Temp file
2+
*~
3+
\#*\#
4+
.DS_Store
5+
16
# Log file
27
experiments.log
38

@@ -112,7 +117,7 @@ MANIFEST
112117
# Usually these files are written by a python script from a template
113118
# before PyInstaller builds the exe, so as to inject date/other infos into it.
114119
*.manifest
115-
*.spec
120+
#*.spec
116121

117122
# Installer logs
118123
pip-log.txt

‎CHANGELOG.md‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# 1.1.0 for Coq v8.10.0
2+
3+
- Add a new command line interface.
4+
5+
- Add a new VSCode interface with a LSP server.
6+
7+
- In the aforementioned two interfaces, most arguments can be supplied
8+
via global or project-local .roosterizerc files.
9+
10+
- Drop the redundant OpenNMTInterfaceForNaming model, and simplify
11+
model selecting after only having the multi-source model.
12+
13+
- Add the script to package as binary distributions using PyInstaller.
14+
15+
16+
# dev for Coq v8.10.0 [dev]
17+
18+
- Initial public release.

‎Makefile‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
INSTALL_DIR = $(HOME)/opt
2+
3+
PY_SRCS = preprocess.py train.py translate.py $(wildcard roosterize/*.py) $(wildcard onmt/*.py)
4+
5+
6+
all: package
7+
8+
.PHONY: package
9+
package: dist/roosterize/roosterize dist/roosterize.tgz
10+
11+
dist/roosterize/roosterize: roosterize.spec $(PY_SRCS)
12+
pyinstaller roosterize.spec -y --log-level WARN
13+
14+
dist/roosterize.tgz: dist/roosterize/roosterize
15+
cd dist && tar czf roosterize.tgz roosterize/
16+
17+
.PHONY: install
18+
install: package
19+
rm -rf $(INSTALL_DIR)/roosterize
20+
mkdir -p $(INSTALL_DIR)/roosterize
21+
cp -r dist/roosterize $(INSTALL_DIR)/roosterize/bin
22+
23+
.PHONY: clean
24+
clean:
25+
-rm -rf dist/ build/

‎bin/roosterize‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
3+
_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
4+
5+
6+
( cd $_DIR/../
7+
python -m roosterize.main $@
8+
)

‎dist-bin/roosterize‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
3+
_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
4+
5+
6+
( cd $_DIR/../
7+
./roosterize $@
8+
)

‎onmt/utils/logging.py‎

Lines changed: 4 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,6 @@
1-
# -*- coding: utf-8 -*-
2-
from __future__ import absolute_import
1+
from seutil import LoggingUtils
32

4-
import logging
3+
logger = LoggingUtils.get_logger(__name__)
54

6-
logger = logging.getLogger()
7-
8-
9-
def init_logger(log_file=None, log_file_level=logging.NOTSET):
10-
log_format = logging.Formatter("[%(asctime)s %(levelname)s] %(message)s")
11-
logger = logging.getLogger()
12-
logger.setLevel(logging.INFO)
13-
14-
console_handler = logging.StreamHandler()
15-
console_handler.setFormatter(log_format)
16-
logger.handlers = [console_handler]
17-
18-
if log_file and log_file != '':
19-
file_handler = logging.FileHandler(log_file)
20-
file_handler.setLevel(log_file_level)
21-
file_handler.setFormatter(log_format)
22-
logger.addHandler(file_handler)
23-
24-
return logger
5+
def init_logger(log_file=None, log_file_level=None):
6+
return logger

‎pretrain.sh‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
#!/bin/bash
2+
3+
_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
4+
5+
6+
# This script is for pre-training the models
7+
8+
9+
function process_data() {
10+
local ds=$1; shift
11+
12+
( cd $_DIR/python/
13+
rm -rf $_DIR/output/$ds/data
14+
python -m roosterize.main extract_data_from_corpus\
15+
--corpus=$_DIR/../math-comp-corpus\
16+
--output=$_DIR/output/$ds/data\
17+
--groups=$ds
18+
)
19+
}
20+
21+
function train_model() {
22+
local ds=$1; shift
23+
24+
( cd $_DIR/python/
25+
rm -rf $_DIR/output/$ds/model
26+
python -m roosterize.main train_model\
27+
--train=$_DIR/output/$ds/data/$ds-train\
28+
--val=$_DIR/output/$ds/data/$ds-val\
29+
--model-dir=$_DIR/output/$ds/model\
30+
--output=$_DIR/output/$ds/data\
31+
--config-file=$_DIR/configs/Stmt+ChopKnlTree+attn+copy.json
32+
)
33+
}
34+
35+
function package_model() {
36+
local ds=$1; shift
37+
38+
( cd $_DIR/output/$ds/
39+
tar czf roosterize-model-$ds.tgz model/
40+
)
41+
}
42+
43+
function eval_model() {
44+
local ds=$1; shift
45+
46+
( cd $_DIR/python/
47+
rm -rf $_DIR/output/$ds/results
48+
python -m roosterize.main eval_model\
49+
--data=$_DIR/output/$ds/data/$ds-test\
50+
--model-dir=$_DIR/output/$ds/model\
51+
--output=$_DIR/output/$ds/results
52+
)
53+
}
54+
55+
function retrain_all_models() {
56+
for ds in t1 ta; do
57+
process_data $ds
58+
train_model $ds
59+
package_model $ds
60+
done
61+
}
62+
63+
64+
# ==========
65+
# Main function -- program entry point
66+
# This script can be executed as ./run.sh the_function_to_run
67+
68+
function main() {
69+
local action=${1:?Need Argument}; shift
70+
71+
( cd ${_DIR}
72+
$action "$@"
73+
)
74+
}
75+
76+
main "$@"
77+

‎requirements.txt‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
1-
configargparse
2-
nltk
1+
configargparse~=1.2.3
32
future
4-
seutil==0.4.12
3+
nltk~=3.5
4+
numpy~=1.19.2
5+
pygls~=0.9.1
6+
seutil>=0.5.4
57
six
6-
torchtext==0.4.0
7-
tqdm==4.30.*
8+
torch==1.1.0 -f https://download.pytorch.org/whl/cpu/torch_stable.html
9+
torchtext==0.4.0 -f https://download.pytorch.org/whl/cpu/torch_stable.html
10+
tqdm~=4.30.0

‎roosterize.spec‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# -*- mode: python ; coding: utf-8 -*-
2+
3+
block_cipher = None
4+
5+
6+
a = Analysis(['roosterize/main.py'],
7+
pathex=['/home/pynie/projects/roosterize-vscode/python'],
8+
binaries=[],
9+
datas=[('./dist-bin/roosterize', 'bin')],
10+
hiddenimports=[],
11+
hookspath=[],
12+
runtime_hooks=[],
13+
excludes=[],
14+
win_no_prefer_redirects=False,
15+
win_private_assemblies=False,
16+
cipher=block_cipher,
17+
noarchive=False)
18+
pyz = PYZ(a.pure, a.zipped_data,
19+
cipher=block_cipher)
20+
exe = EXE(pyz,
21+
a.scripts,
22+
[],
23+
exclude_binaries=True,
24+
name='roosterize',
25+
debug=False,
26+
bootloader_ignore_signals=False,
27+
strip=False,
28+
upx=True,
29+
console=True )
30+
coll = COLLECT(exe,
31+
a.binaries,
32+
a.zipfiles,
33+
a.datas,
34+
strip=False,
35+
upx=True,
36+
upx_exclude=[],
37+
name='roosterize')

0 commit comments

Comments
 (0)