Skip to content

Commit b65b05e

Browse files
TomWildenhain-Microsoftguschmue
TomWildenhain-Microsoft
and
guschmue
authored
Make keras2onnx tests use templates (#1605)
Signed-off-by: Tom Wildenhain <[email protected]> Co-authored-by: Guenther Schmuelling <[email protected]>
1 parent e9e1184 commit b65b05e

File tree

4 files changed

+226
-194
lines changed

4 files changed

+226
-194
lines changed

‎ci_build/azure_pipelines/keras2onnx_application_tests.yml

Lines changed: 6 additions & 109 deletions
Original file line numberDiff line numberDiff line change
@@ -56,60 +56,9 @@ jobs:
5656
NIGHTLY_BUILD_TEST: python run_all_v2.py
5757

5858
steps:
59-
- script: sudo install -d -m 0777 /home/vsts/.conda/envs
60-
displayName: Fix Conda permissions
61-
62-
- task: CondaEnvironment@1
63-
inputs:
64-
createCustomEnvironment: true
65-
environmentName: 'py$(python.version)'
66-
packageSpecs: 'python=$(python.version)'
67-
68-
- script: |
69-
python -m pip install --upgrade pip
70-
conda config --set always_yes yes --set changeps1 no
71-
pip install $(ONNX_PATH)
72-
pip install h5py==2.9.0
73-
pip install numpy==1.19
74-
pip install parameterized
75-
$(INSTALL_TENSORFLOW)
76-
$(INSTALL_KERAS)
77-
pip install git+https://github.com/microsoft/onnxconverter-common
78-
$(INSTALL_ORT)
79-
pip install Pillow==8.2.0
80-
pip install opencv-python
81-
pip install tqdm
82-
pip install keras-segmentation==0.2.0
83-
git clone https://github.com/matterport/Mask_RCNN
84-
cd Mask_RCNN
85-
pip install -r requirements.txt
86-
python setup.py install
87-
cd ..
88-
pip install matplotlib
89-
git clone https://github.com/qqwweee/keras-yolo3
90-
$(INSTALL_KERAS_RESNET)
91-
pip install git+https://www.github.com/keras-team/keras-contrib.git
92-
pip install keras-tcn==2.8.3
93-
$(UNINSTALL_KERAS)
94-
pip install git+https://github.com/qubvel/efficientnet
95-
$(INSTALL_TRANSFORMERS)
96-
pip install keras-self-attention
97-
pip install pytest pytest-cov pytest-runner
98-
displayName: 'Install dependencies'
99-
100-
- script: |
101-
pip install -e .
102-
python -c "import onnxruntime"
103-
pytest tests/keras2onnx_unit_tests --doctest-modules --junitxml=junit/test-results.xml
104-
cd tests/keras2onnx_applications/nightly_build
105-
$(NIGHTLY_BUILD_TEST)
106-
displayName: 'pytest'
107-
108-
- task: PublishTestResults@2
109-
inputs:
110-
testResultsFiles: '**/test-results-*.xml'
111-
testRunTitle: 'Python $(python.version)'
112-
condition: succeededOrFailed()
59+
- template: 'templates/keras2onnx_application_tests.yml'
60+
parameters:
61+
platform: 'linux'
11362

11463
- job: 'Win'
11564
timeoutInMinutes: 180
@@ -151,58 +100,6 @@ jobs:
151100
NIGHTLY_BUILD_TEST: python run_all.py --exclude "test_keras_applications_v2.py test_mask_rcnn.py"
152101

153102
steps:
154-
- task: UsePythonVersion@0
155-
inputs:
156-
versionSpec: '$(python.version)'
157-
architecture: 'x64'
158-
159-
- powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
160-
displayName: Add conda to PATH
161-
162-
- script: conda create --yes --quiet --name py$(python.version) -c conda-forge python=$(python.version) numpy protobuf
163-
displayName: Create Anaconda environment
164-
165-
- script: |
166-
call activate py$(python.version)
167-
python -m pip install --upgrade pip numpy
168-
echo Test numpy installation... && python -c "import numpy"
169-
pip install %ONNX_PATH%
170-
pip uninstall -y protobuf
171-
pip install protobuf
172-
pip install h5py==2.9.0
173-
pip install numpy==1.19
174-
pip install parameterized
175-
%INSTALL_TENSORFLOW%
176-
%INSTALL_KERAS%
177-
pip install git+https://github.com/microsoft/onnxconverter-common
178-
%INSTALL_ORT%
179-
pip install Pillow==8.2.0
180-
pip install opencv-python
181-
pip install tqdm
182-
pip install keras-segmentation==0.2.0
183-
pip install matplotlib
184-
git clone https://github.com/qqwweee/keras-yolo3
185-
%INSTALL_KERAS_RESNET%
186-
pip install git+https://www.github.com/keras-team/keras-contrib.git
187-
pip install keras-tcn==2.8.3
188-
%UNINSTALL_KERAS%
189-
pip install git+https://github.com/qubvel/efficientnet
190-
%INSTALL_TRANSFORMERS%
191-
pip install keras-self-attention
192-
pip install pytest pytest-cov pytest-runner
193-
displayName: 'Install dependencies'
194-
195-
- script: |
196-
call activate py$(python.version)
197-
pip install -e .
198-
echo Test onnxruntime installation... && python -c "import onnxruntime"
199-
pytest tests/keras2onnx_unit_tests --doctest-modules --junitxml=junit/test-results.xml
200-
cd tests/keras2onnx_applications/nightly_build
201-
%NIGHTLY_BUILD_TEST%
202-
displayName: 'pytest'
203-
204-
- task: PublishTestResults@2
205-
inputs:
206-
testResultsFiles: '**/test-results-*.xml'
207-
testRunTitle: 'Python $(python.version)'
208-
condition: succeededOrFailed()
103+
- template: 'templates/keras2onnx_application_tests.yml'
104+
parameters:
105+
platform: 'windows'

‎ci_build/azure_pipelines/keras2onnx_unit_test.yml

Lines changed: 6 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -70,50 +70,9 @@ jobs:
7070
INSTALL_ORT: pip install onnxruntime==1.8.0
7171

7272
steps:
73-
- script: sudo install -d -m 0777 /home/vsts/.conda/envs
74-
displayName: Fix Conda permissions
75-
76-
- task: CondaEnvironment@1
77-
inputs:
78-
createCustomEnvironment: true
79-
environmentName: 'py$(python.version)'
80-
packageSpecs: 'python=$(python.version)'
81-
82-
- script: |
83-
python -m pip install --upgrade pip
84-
conda config --set always_yes yes --set changeps1 no
85-
pip install $(ONNX_PATH)
86-
pip install h5py==2.9.0
87-
pip install numpy==1.19
88-
pip install parameterized
89-
pip install $(TENSORFLOW_PATH)
90-
if [[ ! -z $KERAS ]];
91-
then
92-
pip install $(KERAS)
93-
fi
94-
pip install git+https://github.com/microsoft/onnxconverter-common
95-
pip install -r requirements.txt
96-
pip install -r requirements-dev.txt
97-
pip install pytest pytest-cov pytest-runner
98-
$(INSTALL_ORT)
99-
displayName: 'Install dependencies'
100-
101-
- script: |
102-
pip install -e .
103-
python -c "import onnxruntime"
104-
python -c "import onnxconverter_common"
105-
if [[ ! -z $KERAS ]];
106-
then
107-
export TF_KERAS=0
108-
fi
109-
pytest tests/keras2onnx_unit_tests --doctest-modules --junitxml=junit/test-results.xml
110-
displayName: 'pytest'
111-
112-
- task: PublishTestResults@2
113-
inputs:
114-
testResultsFiles: '**/test-results.xml'
115-
testRunTitle: 'Python $(python.version)'
116-
condition: succeededOrFailed()
73+
- template: 'templates/keras2onnx_unit_test.yml'
74+
parameters:
75+
platform: 'linux'
11776

11877
- job: 'Win'
11978
pool:
@@ -180,44 +139,6 @@ jobs:
180139
INSTALL_ORT: pip install onnxruntime==1.8.0
181140

182141
steps:
183-
- task: UsePythonVersion@0
184-
inputs:
185-
versionSpec: '$(python.version)'
186-
architecture: 'x64'
187-
188-
- powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
189-
displayName: Add conda to PATH
190-
191-
- script: conda create --yes --quiet --name py$(python.version) -c conda-forge python=$(python.version) numpy protobuf
192-
displayName: Create Anaconda environment
193-
194-
- script: |
195-
call activate py$(python.version)
196-
python -m pip install --upgrade pip numpy==1.19
197-
echo Test numpy installation... && python -c "import numpy"
198-
pip install %ONNX_PATH%
199-
pip uninstall -y protobuf
200-
pip install protobuf
201-
pip install h5py==2.9.0
202-
pip install parameterized
203-
pip install %TENSORFLOW_PATH%
204-
IF NOT "%KERAS%"=="" (pip install %KERAS%)
205-
pip install git+https://github.com/microsoft/onnxconverter-common
206-
pip install pytest pytest-cov pytest-runner
207-
%INSTALL_ORT%
208-
displayName: 'Install dependencies'
209-
210-
- script: |
211-
call activate py$(python.version)
212-
pip install -e .
213-
echo Test onnxruntime installation... && python -c "import onnxruntime"
214-
python -c "import onnxconverter_common"
215-
IF NOT "%KERAS%"=="" (set TF_KERAS=0)
216-
pytest tests/keras2onnx_unit_tests --doctest-modules --junitxml=junit/test-results.xml
217-
displayName: 'pytest'
218-
219-
- task: PublishTestResults@2
220-
inputs:
221-
testResultsFiles: '**/test-results.xml'
222-
testRunTitle: 'Python $(python.version)'
223-
condition: succeededOrFailed()
142+
- template: 'templates/keras2onnx_unit_test.yml'
143+
parameters:
144+
platform: 'windows'

‎ci_build/azure_pipelines/templates/keras2onnx_application_tests.yml

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
# Run applications tests from keras2onnx
2+
3+
parameters:
4+
platform: 'linux'
5+
6+
7+
steps:
8+
- ${{ if eq(parameters.platform, 'linux') }}:
9+
- script: sudo install -d -m 0777 /home/vsts/.conda/envs
10+
displayName: Fix Conda permissions
11+
12+
- task: CondaEnvironment@1
13+
inputs:
14+
createCustomEnvironment: true
15+
environmentName: 'py$(python.version)'
16+
packageSpecs: 'python=$(python.version)'
17+
18+
- script: |
19+
python -m pip install --upgrade pip
20+
conda config --set always_yes yes --set changeps1 no
21+
pip install $(ONNX_PATH)
22+
pip install h5py==2.9.0
23+
pip install numpy==1.19
24+
pip install parameterized
25+
$(INSTALL_TENSORFLOW)
26+
$(INSTALL_KERAS)
27+
pip install git+https://github.com/microsoft/onnxconverter-common
28+
$(INSTALL_ORT)
29+
pip install Pillow==8.2.0
30+
pip install opencv-python
31+
pip install tqdm
32+
pip install keras-segmentation==0.2.0
33+
git clone https://github.com/matterport/Mask_RCNN
34+
cd Mask_RCNN
35+
pip install -r requirements.txt
36+
python setup.py install
37+
cd ..
38+
pip install matplotlib
39+
git clone https://github.com/qqwweee/keras-yolo3
40+
$(INSTALL_KERAS_RESNET)
41+
pip install git+https://www.github.com/keras-team/keras-contrib.git
42+
pip install keras-tcn==2.8.3
43+
$(UNINSTALL_KERAS)
44+
pip install git+https://github.com/qubvel/efficientnet
45+
$(INSTALL_TRANSFORMERS)
46+
pip install keras-self-attention
47+
pip install pytest pytest-cov pytest-runner
48+
displayName: 'Install dependencies'
49+
50+
- script: |
51+
pip install -e .
52+
python -c "import onnxruntime"
53+
pytest tests/keras2onnx_unit_tests --doctest-modules --junitxml=junit/test-results.xml
54+
cd tests/keras2onnx_applications/nightly_build
55+
$(NIGHTLY_BUILD_TEST)
56+
displayName: 'pytest'
57+
58+
- task: PublishTestResults@2
59+
inputs:
60+
testResultsFiles: '**/test-results-*.xml'
61+
testRunTitle: 'Python $(python.version)'
62+
condition: succeededOrFailed()
63+
64+
- ${{ if eq(parameters.platform, 'windows') }}:
65+
- task: UsePythonVersion@0
66+
inputs:
67+
versionSpec: '$(python.version)'
68+
architecture: 'x64'
69+
70+
- powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
71+
displayName: Add conda to PATH
72+
73+
- script: conda create --yes --quiet --name py$(python.version) -c conda-forge python=$(python.version) numpy protobuf
74+
displayName: Create Anaconda environment
75+
76+
- script: |
77+
call activate py$(python.version)
78+
python -m pip install --upgrade pip numpy
79+
echo Test numpy installation... && python -c "import numpy"
80+
pip install %ONNX_PATH%
81+
pip uninstall -y protobuf
82+
pip install protobuf
83+
pip install h5py==2.9.0
84+
pip install numpy==1.19
85+
pip install parameterized
86+
%INSTALL_TENSORFLOW%
87+
%INSTALL_KERAS%
88+
pip install git+https://github.com/microsoft/onnxconverter-common
89+
%INSTALL_ORT%
90+
pip install Pillow==8.2.0
91+
pip install opencv-python
92+
pip install tqdm
93+
pip install keras-segmentation==0.2.0
94+
pip install matplotlib
95+
git clone https://github.com/qqwweee/keras-yolo3
96+
%INSTALL_KERAS_RESNET%
97+
pip install git+https://www.github.com/keras-team/keras-contrib.git
98+
pip install keras-tcn==2.8.3
99+
%UNINSTALL_KERAS%
100+
pip install git+https://github.com/qubvel/efficientnet
101+
%INSTALL_TRANSFORMERS%
102+
pip install keras-self-attention
103+
pip install pytest pytest-cov pytest-runner
104+
displayName: 'Install dependencies'
105+
106+
- script: |
107+
call activate py$(python.version)
108+
pip install -e .
109+
echo Test onnxruntime installation... && python -c "import onnxruntime"
110+
pytest tests/keras2onnx_unit_tests --doctest-modules --junitxml=junit/test-results.xml
111+
cd tests/keras2onnx_applications/nightly_build
112+
%NIGHTLY_BUILD_TEST%
113+
displayName: 'pytest'
114+
115+
- task: PublishTestResults@2
116+
inputs:
117+
testResultsFiles: '**/test-results-*.xml'
118+
testRunTitle: 'Python $(python.version)'
119+
condition: succeededOrFailed()

0 commit comments

Comments
 (0)