Skip to content

Commit 9c19009

Browse files
folivoramanhanand-nv
folivoramanh
and
anand-nv
authored
Fix Jenkinsfile for CI (#325) (#327)
* Fix Jenkinsfile for CI * Fix requirements for test * Update paths and docker * Fix docker name * Fix click version * Change path of grammars for sparrowhawk tests * Update paths in sh_test.sh * Update paths * Revert paths --------- Signed-off-by: Anand Joseph <[email protected]> Signed-off-by: folivoramanh <[email protected]> Co-authored-by: anand-nv <[email protected]>
1 parent b63d929 commit 9c19009

File tree

3 files changed

+72
-61
lines changed

3 files changed

+72
-61
lines changed

‎.pre-commit-config.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,4 +50,4 @@ repos:
5050
- id: black
5151
name: Format code
5252
args: [--skip-string-normalization, --line-length=119]
53-
additional_dependencies: ['click==8.0.2']
53+
additional_dependencies: ['click>=8.0.2']

‎Jenkinsfile‎

Lines changed: 68 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
pipeline {
22
agent {
33
docker {
4-
image 'tnitn_ci:py310'
5-
args '--user 0:128 -v /home/jenkinsci:/home/jenkinsci -v $HOME/.cache:/root/.cache --shm-size=4g --entrypoint=""'
4+
image 'tnitn_ci_py310:24.07'
5+
args '-v /mnt/jenkins/jenkinsci:/home/jenkins -v $HOME/.cache:/root/.cache --shm-size=4g --entrypoint=""'
66
}
77
}
88
options {
@@ -11,33 +11,27 @@ pipeline {
1111
}
1212
environment {
1313

14-
AR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-24-24-0'
15-
DE_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/10-23-24-0'
16-
EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/09-04-24-0'
17-
ES_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/09-25-24-0'
18-
ES_EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/08-30-24-0'
19-
FR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-07-25-0'
20-
HU_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/07-16-24-0'
21-
PT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
22-
RU_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
23-
VI_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
24-
SV_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
25-
ZH_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/11-13-24-0'
26-
IT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/08-22-24-0'
27-
HY_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-0'
28-
MR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-1'
29-
JA_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/10-17-24-1'
30-
HI_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-22-25-0'
31-
DEFAULT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
14+
AR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-24-24-0'
15+
DE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-23-24-0'
16+
EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-04-24-0'
17+
ES_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-25-24-0'
18+
ES_EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/08-30-24-0'
19+
FR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-07-25-0'
20+
HU_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-16-24-0'
21+
PT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
22+
RU_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
23+
VI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
24+
SV_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
25+
ZH_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/11-13-24-0'
26+
IT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/08-22-24-0'
27+
HY_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-0'
28+
MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1'
29+
JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1'
30+
HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-22-25-0'
31+
DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
3232
}
3333
stages {
3434

35-
stage('Add git safe directory'){
36-
steps{
37-
sh 'git config --global --add safe.directory /var/lib/jenkins/workspace/NTP_$GIT_BRANCH'
38-
sh 'git config --global --add safe.directory /home/jenkinsci/workspace/NTP_$GIT_BRANCH'
39-
}
40-
}
4135

4236
stage('PyTorch version') {
4337
steps {
@@ -46,14 +40,6 @@ pipeline {
4640
}
4741
}
4842

49-
stage('Install test requirements') {
50-
steps {
51-
sh 'apt-get update && apt-get install -y bc'
52-
}
53-
}
54-
55-
56-
5743
stage('NeMo Installation') {
5844
steps {
5945
sh './reinstall.sh release'
@@ -65,7 +51,10 @@ pipeline {
6551
when {
6652
anyOf {
6753
branch 'main'
54+
branch 'staging/**'
55+
branch 'staging_*'
6856
changeRequest target: 'main'
57+
6958
}
7059
}
7160
failFast true
@@ -97,6 +86,8 @@ pipeline {
9786
when {
9887
anyOf {
9988
branch 'main'
89+
branch 'staging/**'
90+
branch 'staging_*'
10091
changeRequest target: 'main'
10192
}
10293
}
@@ -120,6 +111,8 @@ pipeline {
120111
when {
121112
anyOf {
122113
branch 'main'
114+
branch 'staging/**'
115+
branch 'staging_*'
123116
changeRequest target: 'main'
124117
}
125118
}
@@ -156,7 +149,9 @@ pipeline {
156149
stage('L0: Create AR TN/ITN Grammars') {
157150
when {
158151
anyOf {
159-
branch 'main'
152+
branch 'main'
153+
branch 'staging/**'
154+
branch 'staging_*'
160155
changeRequest target: 'main'
161156
}
162157
}
@@ -179,7 +174,9 @@ pipeline {
179174
stage('L0: Create FR TN/ITN & VI TN/ITN & HU TN & IT TN') {
180175
when {
181176
anyOf {
182-
branch 'main'
177+
branch 'main'
178+
branch 'staging/**'
179+
branch 'staging_*'
183180
changeRequest target: 'main'
184181
}
185182
}
@@ -221,7 +218,9 @@ pipeline {
221218
stage('L0: Create RU TN/ITN Grammars & SV & PT') {
222219
when {
223220
anyOf {
224-
branch 'main'
221+
branch 'main'
222+
branch 'staging/**'
223+
branch 'staging_*'
225224
changeRequest target: 'main'
226225
}
227226
}
@@ -263,7 +262,9 @@ pipeline {
263262
stage('L0: Create HY TN/ITN Grammars & MR') {
264263
when {
265264
anyOf {
266-
branch 'main'
265+
branch 'main'
266+
branch 'staging/**'
267+
branch 'staging_*'
267268
changeRequest target: 'main'
268269
}
269270
}
@@ -289,7 +290,9 @@ pipeline {
289290
stage('L0: Create ZH TN/ITN Grammar') {
290291
when {
291292
anyOf {
292-
branch 'main'
293+
branch 'main'
294+
branch 'staging/**'
295+
branch 'staging_*'
293296
changeRequest target: 'main'
294297
}
295298
}
@@ -310,7 +313,9 @@ pipeline {
310313
stage('L0: Create JA ITN Grammars') {
311314
when {
312315
anyOf {
313-
branch 'main'
316+
branch 'main'
317+
branch 'staging/**'
318+
branch 'staging_*'
314319
changeRequest target: 'main'
315320
}
316321
}
@@ -330,7 +335,9 @@ pipeline {
330335
stage('L1: TN/ITN Tests CPU') {
331336
when {
332337
anyOf {
333-
branch 'main'
338+
branch 'main'
339+
branch 'staging/**'
340+
branch 'staging_*'
334341
changeRequest target: 'main'
335342
}
336343
}
@@ -414,10 +421,12 @@ pipeline {
414421
}
415422
}
416423

417-
stage('L2: Sparrowhawk Tests') {
424+
stage('L2: EN Sparrowhawk Tests') {
418425
when {
419426
anyOf {
420-
branch 'main'
427+
branch 'main'
428+
branch 'staging/**'
429+
branch 'staging_*'
421430
changeRequest target: 'main'
422431
}
423432
}
@@ -446,54 +455,56 @@ pipeline {
446455

447456
}
448457
}
449-
458+
450459
stage('L2: NeMo text processing') {
451460
when {
452461
anyOf {
453-
branch 'main'
462+
branch 'main'
463+
branch 'staging/**'
464+
branch 'staging_*'
454465
changeRequest target: 'main'
455466
}
456467
}
457468
failFast true
458469
parallel {
459470
stage('L2: Eng TN') {
460471
steps {
461-
sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_norm/output_${TIME} && \
472+
sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkins/TestData/text_norm/output_${TIME} && \
462473
cd tools/text_processing_deployment && python pynini_export.py --output=$NORM_OUTPUT_DIR --grammars=tn_grammars --cache_dir ${EN_TN_CACHE} --language=en && ls -R $NORM_OUTPUT_DIR && echo ".far files created "|| exit 1'
463-
sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_norm/output_${TIME} && mkdir $NORM_OUTPUT_DIR && \
464-
cd nemo_text_processing/text_normalization/ && python normalize.py --input_file=/home/jenkinsci/TestData/text_norm/ci/test.txt --input_case="lower_cased" --language=en --output_file=$NORM_OUTPUT_DIR/test.pynini.txt --verbose && \
474+
sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkins/TestData/text_norm/output_${TIME} && mkdir $NORM_OUTPUT_DIR && \
475+
cd nemo_text_processing/text_normalization/ && python normalize.py --input_file=/home/jenkins/TestData/text_norm/ci/test.txt --input_case="lower_cased" --language=en --output_file=$NORM_OUTPUT_DIR/test.pynini.txt --verbose && \
465476
cat $NORM_OUTPUT_DIR/test.pynini.txt && \
466-
cmp --silent $NORM_OUTPUT_DIR/test.pynini.txt /home/jenkinsci/TestData/text_norm/ci/test_goal_py.txt || exit 1 && \
477+
cmp --silent $NORM_OUTPUT_DIR/test.pynini.txt /home/jenkins/TestData/text_norm/ci/test_goal_py.txt || exit 1 && \
467478
rm -rf $NORM_OUTPUT_DIR'
468479
}
469480
}
470481

471482
stage('L2: Eng ITN export') {
472483
steps {
473-
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_denorm/output_${TIME} && \
484+
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkins/TestData/text_denorm/output_${TIME} && \
474485
cd tools/text_processing_deployment && python pynini_export.py --output=$DENORM_OUTPUT_DIR --grammars=itn_grammars --cache_dir ${EN_TN_CACHE} --language=en && ls -R $DENORM_OUTPUT_DIR && echo ".far files created "|| exit 1'
475-
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_denorm/output_${TIME} && mkdir $DENORM_OUTPUT_DIR && \
476-
cd nemo_text_processing/inverse_text_normalization/ && python inverse_normalize.py --input_file=/home/jenkinsci/TestData/text_denorm/ci/test.txt --language=en --output_file=$DENORM_OUTPUT_DIR/test.pynini.txt --verbose && \
477-
cmp --silent $DENORM_OUTPUT_DIR/test.pynini.txt /home/jenkinsci/TestData/text_denorm/ci/test_goal_py.txt || exit 1 && \
486+
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkins/TestData/text_denorm/output_${TIME} && mkdir $DENORM_OUTPUT_DIR && \
487+
cd nemo_text_processing/inverse_text_normalization/ && python inverse_normalize.py --input_file=/home/jenkins/TestData/text_denorm/ci/test.txt --language=en --output_file=$DENORM_OUTPUT_DIR/test.pynini.txt --verbose && \
488+
cmp --silent $DENORM_OUTPUT_DIR/test.pynini.txt /home/jenkins/TestData/text_denorm/ci/test_goal_py.txt || exit 1 && \
478489
rm -rf $DENORM_OUTPUT_DIR'
479490
}
480491
}
481492

482493

483494
stage('L2: Eng alignment TN') {
484495
steps {
485-
sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_norm/output_${TIME} && mkdir $NORM_OUTPUT_DIR && \
496+
sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkins/TestData/text_norm/output_${TIME} && mkdir $NORM_OUTPUT_DIR && \
486497
cd nemo_text_processing/fst_alignment && python alignment.py --text="2615 Forest Av, 90501 CA, Santa Clara. 10kg, 12/16/2018" --grammar=tn --rule=tokenize_and_classify --fst=${EN_TN_CACHE}/en_tn_True_deterministic_cased__tokenize.far 2>&1 | tee $NORM_OUTPUT_DIR/pred.txt && \
487-
cmp --silent $NORM_OUTPUT_DIR/pred.txt /home/jenkinsci/TestData/text_norm/ci/alignment_gold.txt || exit 1 && \
498+
cmp --silent $NORM_OUTPUT_DIR/pred.txt /home/jenkins/TestData/text_norm/ci/alignment_gold.txt || exit 1 && \
488499
rm -rf $NORM_OUTPUT_DIR'
489500
}
490501
}
491502

492503
stage('L2: Eng alignment ITN') {
493504
steps {
494-
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_denorm/output_${TIME} && mkdir $DENORM_OUTPUT_DIR && \
505+
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkins/TestData/text_denorm/output_${TIME} && mkdir $DENORM_OUTPUT_DIR && \
495506
cd nemo_text_processing/fst_alignment && python alignment.py --text="one million twenty three thousand two hundred eleven ten kilograms one hundred twenty three dollars and twenty five cents" --grammar=itn --rule=tokenize_and_classify --fst=${EN_TN_CACHE}/en_itn_lower_cased.far 2>&1 | tee $DENORM_OUTPUT_DIR/pred.txt && \
496-
cmp --silent $DENORM_OUTPUT_DIR/pred.txt /home/jenkinsci/TestData/text_denorm/ci/alignment_gold.txt || exit 1 && \
507+
cmp --silent $DENORM_OUTPUT_DIR/pred.txt /home/jenkins/TestData/text_denorm/ci/alignment_gold.txt || exit 1 && \
497508
rm -rf $DENORM_OUTPUT_DIR'
498509
}
499510
}

‎requirements/requirements_test.txt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
black==19.10b0
2-
click==8.0.2
3-
isort[requirements]>5.1.0,<6.0.0
1+
black==25.1.0
2+
click>=8.0.2
3+
isort[requirements]>5.1.0,<=6.0.1
44
parameterized
55
pynini==2.1.6.post1
66
pytest

0 commit comments

Comments
 (0)