Skip to content

Commit f93f668

Browse files
anand-nvmgrafu
anand-nv
authored and
mgrafu
committed
Fix Jenkinsfile for CI (#325)
* Fix Jenkinsfile for CI Signed-off-by: Anand Joseph <[email protected]> * Fix requirements for test Signed-off-by: Anand Joseph <[email protected]> * Update paths and docker Signed-off-by: Anand Joseph <[email protected]> * Fix docker name Signed-off-by: Anand Joseph <[email protected]> * Fix click version Signed-off-by: Anand Joseph <[email protected]> * Change path of grammars for sparrowhawk tests Signed-off-by: Anand Joseph <[email protected]> * Update paths in sh_test.sh Signed-off-by: Anand Joseph <[email protected]> * Update paths Signed-off-by: Anand Joseph <[email protected]> * Revert paths Signed-off-by: Anand Joseph <[email protected]> --------- Signed-off-by: Anand Joseph <[email protected]>
1 parent dc4958b commit f93f668

File tree

3 files changed

+72
-61
lines changed

3 files changed

+72
-61
lines changed

‎.pre-commit-config.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,4 +50,4 @@ repos:
5050
- id: black
5151
name: Format code
5252
args: [--skip-string-normalization, --line-length=119]
53-
additional_dependencies: ['click==8.0.2']
53+
additional_dependencies: ['click>=8.0.2']

‎Jenkinsfile‎

Lines changed: 68 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
pipeline {
22
agent {
33
docker {
4-
image 'tnitn_ci:py310'
5-
args '--user 0:128 -v /home/jenkinsci:/home/jenkinsci -v $HOME/.cache:/root/.cache --shm-size=4g --entrypoint=""'
4+
image 'tnitn_ci_py310:24.07'
5+
args '-v /mnt/jenkins/jenkinsci:/home/jenkins -v $HOME/.cache:/root/.cache --shm-size=4g --entrypoint=""'
66
}
77
}
88
options {
@@ -11,33 +11,27 @@ pipeline {
1111
}
1212
environment {
1313

14-
AR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-24-24-0'
15-
DE_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/10-23-24-0'
16-
EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/09-04-24-0'
17-
ES_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/09-25-24-0'
18-
ES_EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/08-30-24-0'
19-
FR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-07-25-0'
20-
HU_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/07-16-24-0'
21-
PT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
22-
RU_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
23-
VI_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
24-
SV_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
25-
ZH_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/11-13-24-0'
26-
IT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/08-22-24-0'
27-
HY_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-0'
28-
MR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-1'
29-
JA_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/10-17-24-1'
30-
HI_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-22-25-0'
31-
DEFAULT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
14+
AR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-24-24-0'
15+
DE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-23-24-0'
16+
EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-04-24-0'
17+
ES_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-25-24-0'
18+
ES_EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/08-30-24-0'
19+
FR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-07-25-0'
20+
HU_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-16-24-0'
21+
PT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
22+
RU_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
23+
VI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
24+
SV_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
25+
ZH_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/11-13-24-0'
26+
IT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/08-22-24-0'
27+
HY_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-0'
28+
MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1'
29+
JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1'
30+
HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-22-25-0'
31+
DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
3232
}
3333
stages {
3434

35-
stage('Add git safe directory'){
36-
steps{
37-
sh 'git config --global --add safe.directory /var/lib/jenkins/workspace/NTP_$GIT_BRANCH'
38-
sh 'git config --global --add safe.directory /home/jenkinsci/workspace/NTP_$GIT_BRANCH'
39-
}
40-
}
4135

4236
stage('PyTorch version') {
4337
steps {
@@ -46,14 +40,6 @@ pipeline {
4640
}
4741
}
4842

49-
stage('Install test requirements') {
50-
steps {
51-
sh 'apt-get update && apt-get install -y bc'
52-
}
53-
}
54-
55-
56-
5743
stage('NeMo Installation') {
5844
steps {
5945
sh './reinstall.sh release'
@@ -65,7 +51,10 @@ pipeline {
6551
when {
6652
anyOf {
6753
branch 'main'
54+
branch 'staging/**'
55+
branch 'staging_*'
6856
changeRequest target: 'main'
57+
6958
}
7059
}
7160
failFast true
@@ -97,6 +86,8 @@ pipeline {
9786
when {
9887
anyOf {
9988
branch 'main'
89+
branch 'staging/**'
90+
branch 'staging_*'
10091
changeRequest target: 'main'
10192
}
10293
}
@@ -120,6 +111,8 @@ pipeline {
120111
when {
121112
anyOf {
122113
branch 'main'
114+
branch 'staging/**'
115+
branch 'staging_*'
123116
changeRequest target: 'main'
124117
}
125118
}
@@ -156,7 +149,9 @@ pipeline {
156149
stage('L0: Create AR TN/ITN Grammars') {
157150
when {
158151
anyOf {
159-
branch 'main'
152+
branch 'main'
153+
branch 'staging/**'
154+
branch 'staging_*'
160155
changeRequest target: 'main'
161156
}
162157
}
@@ -179,7 +174,9 @@ pipeline {
179174
stage('L0: Create FR TN/ITN & VI ITN & HU TN & IT TN') {
180175
when {
181176
anyOf {
182-
branch 'main'
177+
branch 'main'
178+
branch 'staging/**'
179+
branch 'staging_*'
183180
changeRequest target: 'main'
184181
}
185182
}
@@ -216,7 +213,9 @@ pipeline {
216213
stage('L0: Create RU TN/ITN Grammars & SV & PT') {
217214
when {
218215
anyOf {
219-
branch 'main'
216+
branch 'main'
217+
branch 'staging/**'
218+
branch 'staging_*'
220219
changeRequest target: 'main'
221220
}
222221
}
@@ -258,7 +257,9 @@ pipeline {
258257
stage('L0: Create HY TN/ITN Grammars & MR') {
259258
when {
260259
anyOf {
261-
branch 'main'
260+
branch 'main'
261+
branch 'staging/**'
262+
branch 'staging_*'
262263
changeRequest target: 'main'
263264
}
264265
}
@@ -284,7 +285,9 @@ pipeline {
284285
stage('L0: Create ZH TN/ITN Grammar') {
285286
when {
286287
anyOf {
287-
branch 'main'
288+
branch 'main'
289+
branch 'staging/**'
290+
branch 'staging_*'
288291
changeRequest target: 'main'
289292
}
290293
}
@@ -305,7 +308,9 @@ pipeline {
305308
stage('L0: Create JA ITN Grammars') {
306309
when {
307310
anyOf {
308-
branch 'main'
311+
branch 'main'
312+
branch 'staging/**'
313+
branch 'staging_*'
309314
changeRequest target: 'main'
310315
}
311316
}
@@ -325,7 +330,9 @@ pipeline {
325330
stage('L1: TN/ITN Tests CPU') {
326331
when {
327332
anyOf {
328-
branch 'main'
333+
branch 'main'
334+
branch 'staging/**'
335+
branch 'staging_*'
329336
changeRequest target: 'main'
330337
}
331338
}
@@ -409,10 +416,12 @@ pipeline {
409416
}
410417
}
411418

412-
stage('L2: Sparrowhawk Tests') {
419+
stage('L2: EN Sparrowhawk Tests') {
413420
when {
414421
anyOf {
415-
branch 'main'
422+
branch 'main'
423+
branch 'staging/**'
424+
branch 'staging_*'
416425
changeRequest target: 'main'
417426
}
418427
}
@@ -441,54 +450,56 @@ pipeline {
441450

442451
}
443452
}
444-
453+
445454
stage('L2: NeMo text processing') {
446455
when {
447456
anyOf {
448-
branch 'main'
457+
branch 'main'
458+
branch 'staging/**'
459+
branch 'staging_*'
449460
changeRequest target: 'main'
450461
}
451462
}
452463
failFast true
453464
parallel {
454465
stage('L2: Eng TN') {
455466
steps {
456-
sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_norm/output_${TIME} && \
467+
sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkins/TestData/text_norm/output_${TIME} && \
457468
cd tools/text_processing_deployment && python pynini_export.py --output=$NORM_OUTPUT_DIR --grammars=tn_grammars --cache_dir ${EN_TN_CACHE} --language=en && ls -R $NORM_OUTPUT_DIR && echo ".far files created "|| exit 1'
458-
sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_norm/output_${TIME} && mkdir $NORM_OUTPUT_DIR && \
459-
cd nemo_text_processing/text_normalization/ && python normalize.py --input_file=/home/jenkinsci/TestData/text_norm/ci/test.txt --input_case="lower_cased" --language=en --output_file=$NORM_OUTPUT_DIR/test.pynini.txt --verbose && \
469+
sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkins/TestData/text_norm/output_${TIME} && mkdir $NORM_OUTPUT_DIR && \
470+
cd nemo_text_processing/text_normalization/ && python normalize.py --input_file=/home/jenkins/TestData/text_norm/ci/test.txt --input_case="lower_cased" --language=en --output_file=$NORM_OUTPUT_DIR/test.pynini.txt --verbose && \
460471
cat $NORM_OUTPUT_DIR/test.pynini.txt && \
461-
cmp --silent $NORM_OUTPUT_DIR/test.pynini.txt /home/jenkinsci/TestData/text_norm/ci/test_goal_py.txt || exit 1 && \
472+
cmp --silent $NORM_OUTPUT_DIR/test.pynini.txt /home/jenkins/TestData/text_norm/ci/test_goal_py.txt || exit 1 && \
462473
rm -rf $NORM_OUTPUT_DIR'
463474
}
464475
}
465476

466477
stage('L2: Eng ITN export') {
467478
steps {
468-
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_denorm/output_${TIME} && \
479+
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkins/TestData/text_denorm/output_${TIME} && \
469480
cd tools/text_processing_deployment && python pynini_export.py --output=$DENORM_OUTPUT_DIR --grammars=itn_grammars --cache_dir ${EN_TN_CACHE} --language=en && ls -R $DENORM_OUTPUT_DIR && echo ".far files created "|| exit 1'
470-
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_denorm/output_${TIME} && mkdir $DENORM_OUTPUT_DIR && \
471-
cd nemo_text_processing/inverse_text_normalization/ && python inverse_normalize.py --input_file=/home/jenkinsci/TestData/text_denorm/ci/test.txt --language=en --output_file=$DENORM_OUTPUT_DIR/test.pynini.txt --verbose && \
472-
cmp --silent $DENORM_OUTPUT_DIR/test.pynini.txt /home/jenkinsci/TestData/text_denorm/ci/test_goal_py.txt || exit 1 && \
481+
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkins/TestData/text_denorm/output_${TIME} && mkdir $DENORM_OUTPUT_DIR && \
482+
cd nemo_text_processing/inverse_text_normalization/ && python inverse_normalize.py --input_file=/home/jenkins/TestData/text_denorm/ci/test.txt --language=en --output_file=$DENORM_OUTPUT_DIR/test.pynini.txt --verbose && \
483+
cmp --silent $DENORM_OUTPUT_DIR/test.pynini.txt /home/jenkins/TestData/text_denorm/ci/test_goal_py.txt || exit 1 && \
473484
rm -rf $DENORM_OUTPUT_DIR'
474485
}
475486
}
476487

477488

478489
stage('L2: Eng alignment TN') {
479490
steps {
480-
sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_norm/output_${TIME} && mkdir $NORM_OUTPUT_DIR && \
491+
sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkins/TestData/text_norm/output_${TIME} && mkdir $NORM_OUTPUT_DIR && \
481492
cd nemo_text_processing/fst_alignment && python alignment.py --text="2615 Forest Av, 90501 CA, Santa Clara. 10kg, 12/16/2018" --grammar=tn --rule=tokenize_and_classify --fst=${EN_TN_CACHE}/en_tn_True_deterministic_cased__tokenize.far 2>&1 | tee $NORM_OUTPUT_DIR/pred.txt && \
482-
cmp --silent $NORM_OUTPUT_DIR/pred.txt /home/jenkinsci/TestData/text_norm/ci/alignment_gold.txt || exit 1 && \
493+
cmp --silent $NORM_OUTPUT_DIR/pred.txt /home/jenkins/TestData/text_norm/ci/alignment_gold.txt || exit 1 && \
483494
rm -rf $NORM_OUTPUT_DIR'
484495
}
485496
}
486497

487498
stage('L2: Eng alignment ITN') {
488499
steps {
489-
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_denorm/output_${TIME} && mkdir $DENORM_OUTPUT_DIR && \
500+
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkins/TestData/text_denorm/output_${TIME} && mkdir $DENORM_OUTPUT_DIR && \
490501
cd nemo_text_processing/fst_alignment && python alignment.py --text="one million twenty three thousand two hundred eleven ten kilograms one hundred twenty three dollars and twenty five cents" --grammar=itn --rule=tokenize_and_classify --fst=${EN_TN_CACHE}/en_itn_lower_cased.far 2>&1 | tee $DENORM_OUTPUT_DIR/pred.txt && \
491-
cmp --silent $DENORM_OUTPUT_DIR/pred.txt /home/jenkinsci/TestData/text_denorm/ci/alignment_gold.txt || exit 1 && \
502+
cmp --silent $DENORM_OUTPUT_DIR/pred.txt /home/jenkins/TestData/text_denorm/ci/alignment_gold.txt || exit 1 && \
492503
rm -rf $DENORM_OUTPUT_DIR'
493504
}
494505
}

‎requirements/requirements_test.txt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
black==19.10b0
2-
click==8.0.2
3-
isort[requirements]>5.1.0,<6.0.0
1+
black==25.1.0
2+
click>=8.0.2
3+
isort[requirements]>5.1.0,<=6.0.1
44
parameterized
55
pynini==2.1.6.post1
66
pytest

0 commit comments

Comments
 (0)