Skip to content

Commit d6e38ad

Browse files
lucienwang1009lucienwang1009
committed
combine and report unittest results
1 parent c39a471 commit d6e38ad

File tree

8 files changed

+198
-105
lines changed

8 files changed

+198
-105
lines changed

‎ci_build/azure_pipelines/coveragerc

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[paths]
2+
source =
3+
./

‎ci_build/azure_pipelines/onnxruntime_nightly_test.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,6 @@ jobs:
1212
- template: 'unit_test.yml'
1313
parameters:
1414
onnx_opsets: ['10', '9', '8', '7']
15+
report_coverage: 'True'
16+
17+
- template: 'templates/combine_test_coverage.yml'

‎ci_build/azure_pipelines/templates/combine_test_coverage.yml

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# combine and report unittest coverage
2+
3+
parameters:
4+
artifact_name: 'single_test_coverage'
5+
6+
stages:
7+
- stage:
8+
jobs:
9+
- job: 'combine_and_report_coverage'
10+
variables:
11+
CI_ARTIFACT_NAME: '${{ parameters.artifact_name }}'
12+
13+
pool:
14+
vmImage: 'ubuntu-16.04'
15+
16+
steps:
17+
- task: DownloadBuildArtifacts@0
18+
displayName: 'Download Single Test Coverage'
19+
inputs:
20+
artifactName: '${{ parameters.artifact_name }}'
21+
downloadPath: $(System.DefaultWorkingDirectory)
22+
23+
- task: CondaEnvironment@1
24+
inputs:
25+
createCustomEnvironment: 'true'
26+
environmentName: 'tf2onnx'
27+
packageSpecs: 'python=3.6'
28+
updateConda: 'false'
29+
30+
- bash: |
31+
pip install -U coverage
32+
condition: succeeded()
33+
displayName: 'Install Coverage'
34+
35+
- bash: |
36+
cat ${CI_ARTIFACT_NAME}/.coveragerc_paths* >> ci_build/azure_pipelines/coveragerc
37+
coverage combine --rcfile ci_build/azure_pipelines/coveragerc ${CI_ARTIFACT_NAME}
38+
coverage report
39+
coverage html -d ${BUILD_ARTIFACTSTAGINGDIRECTORY}/coverage_report
40+
condition: succeeded()
41+
displayName: 'Combine And Report Test Coverage'
42+
43+
- task: PublishBuildArtifacts@1
44+
condition: succeeded()
45+
inputs:
46+
pathtoPublish: '$(Build.ArtifactStagingDirectory)'
47+
artifactName: 'test_coverage_report'
48+
displayName: 'Deploy Test Coverage Report'

‎ci_build/azure_pipelines/templates/job_generator.yml

Lines changed: 85 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -9,73 +9,83 @@ parameters:
99
onnx_backends: {onnxruntime: ['0.4.0']}
1010
job: {}
1111
run_setup: 'True'
12+
report_coverage: 'False'
13+
artifact_name: 'single_test_coverage'
1214

1315
jobs:
14-
- ${{ each platform in parameters.platforms }}:
1516
- job: ${{ parameters.job.name }}
1617
pool:
17-
${{ if eq(platform, 'linux') }}:
18-
vmImage: 'ubuntu-16.04'
19-
${{ if eq(platform, 'windows') }}:
20-
vmImage: 'vs2017-win2016'
21-
${{ if eq(platform, 'mac') }}:
22-
vmImage: 'macOS-10.13'
18+
vmImage: $(CI_VM_IMAGE)
2319

2420
# Generate matrix
2521
strategy:
2622
matrix:
27-
${{ each python_version in parameters.python_versions }}:
28-
${{ each tf_version in parameters.tf_versions }}:
29-
${{ each onnx_version in parameters.onnx_versions }}:
30-
${{ each onnx_backend in parameters.onnx_backends }}:
31-
${{ each onnx_backend_version in onnx_backend.value }}:
32-
${{ each onnx_opset in parameters.onnx_opsets }}:
33-
${{ if ne(onnx_opset, '') }}:
34-
${{ format('{0} python{1} tf{2} onnx{3} opset{4} {5}{6}', platform, python_version, tf_version, onnx_version, onnx_opset, onnx_backend.key, onnx_backend_version) }}:
35-
CI_PYTHON_VERSION: '${{ python_version }}'
36-
CI_TF_VERSION: '${{ tf_version }}'
37-
CI_ONNX_VERSION: '${{ onnx_version }}'
38-
CI_ONNX_OPSET: '${{ onnx_opset }}'
39-
CI_ONNX_BACKEND: '${{ onnx_backend.key }}'
40-
CI_ONNX_BACKEND_VERSION: '${{ onnx_backend_version }}'
41-
42-
${{ if eq(tf_version, '') }}:
43-
CI_PIP_TF_NAME: 'tensorflow'
44-
${{ if ne(tf_version, '') }}:
45-
CI_PIP_TF_NAME: ${{ format('tensorflow=={0}', tf_version) }}
46-
47-
${{ if eq(onnx_version, '') }}:
48-
CI_PIP_ONNX_NAME: 'onnx'
49-
${{ if ne(onnx_version, '') }}:
50-
CI_PIP_ONNX_NAME: ${{ format('onnx=={0}', onnx_version) }}
51-
52-
${{ if eq(onnx_backend_version, '') }}:
53-
CI_PIP_ONNX_BACKEND_NAME: '${{ onnx_backend.key }}'
54-
${{ if ne(onnx_backend_version, '') }}:
55-
CI_PIP_ONNX_BACKEND_NAME: ${{ format('{0}=={1}', onnx_backend.key, onnx_backend_version) }}
56-
57-
${{ if eq(onnx_opset, '') }}:
58-
${{ format('{0} python{1} tf{2} onnx{3} {4}{5}', platform, python_version, tf_version, onnx_version, onnx_backend.key, onnx_backend_version) }}:
59-
CI_PYTHON_VERSION: '${{ python_version }}'
60-
CI_TF_VERSION: '${{ tf_version }}'
61-
CI_ONNX_VERSION: '${{ onnx_version }}'
62-
CI_ONNX_BACKEND: '${{ onnx_backend.key }}'
63-
CI_ONNX_BACKEND_VERSION: '${{ onnx_backend_version }}'
64-
65-
${{ if eq(tf_version, '') }}:
66-
CI_PIP_TF_NAME: 'tensorflow'
67-
${{ if ne(tf_version, '') }}:
68-
CI_PIP_TF_NAME: ${{ format('tensorflow=={0}', tf_version) }}
69-
70-
${{ if eq(onnx_version, '') }}:
71-
CI_PIP_ONNX_NAME: 'onnx'
72-
${{ if ne(onnx_version, '') }}:
73-
CI_PIP_ONNX_NAME: ${{ format('onnx=={0}', onnx_version) }}
74-
75-
${{ if eq(onnx_backend_version, '') }}:
76-
CI_PIP_ONNX_BACKEND_NAME: '${{ onnx_backend.key }}'
77-
${{ if ne(onnx_backend_version, '') }}:
78-
CI_PIP_ONNX_BACKEND_NAME: ${{ format('{0}=={1}', onnx_backend.key, onnx_backend_version) }}
23+
${{ each platform in parameters.platforms }}:
24+
${{ each python_version in parameters.python_versions }}:
25+
${{ each tf_version in parameters.tf_versions }}:
26+
${{ each onnx_version in parameters.onnx_versions }}:
27+
${{ each onnx_backend in parameters.onnx_backends }}:
28+
${{ each onnx_backend_version in onnx_backend.value }}:
29+
${{ each onnx_opset in parameters.onnx_opsets }}:
30+
${{ if ne(onnx_opset, '') }}:
31+
${{ format('{0} python{1} tf{2} onnx{3} opset{4} {5}{6}', platform, python_version, tf_version, onnx_version, onnx_opset, onnx_backend.key, onnx_backend_version) }}:
32+
${{ if eq(platform, 'linux') }}:
33+
CI_VM_IMAGE: 'ubuntu-16.04'
34+
${{ if eq(platform, 'windows') }}:
35+
CI_VM_IMAGE: 'vs2017-win2016'
36+
${{ if eq(platform, 'mac') }}:
37+
CI_VM_IMAGE: 'macOS-10.13'
38+
CI_PYTHON_VERSION: '${{ python_version }}'
39+
CI_TF_VERSION: '${{ tf_version }}'
40+
CI_ONNX_VERSION: '${{ onnx_version }}'
41+
CI_ONNX_OPSET: '${{ onnx_opset }}'
42+
CI_ONNX_BACKEND: '${{ onnx_backend.key }}'
43+
CI_ONNX_BACKEND_VERSION: '${{ onnx_backend_version }}'
44+
45+
${{ if eq(tf_version, '') }}:
46+
CI_PIP_TF_NAME: 'tensorflow'
47+
${{ if ne(tf_version, '') }}:
48+
CI_PIP_TF_NAME: ${{ format('tensorflow=={0}', tf_version) }}
49+
50+
${{ if eq(onnx_version, '') }}:
51+
CI_PIP_ONNX_NAME: 'onnx'
52+
${{ if ne(onnx_version, '') }}:
53+
CI_PIP_ONNX_NAME: ${{ format('onnx=={0}', onnx_version) }}
54+
55+
${{ if eq(onnx_backend_version, '') }}:
56+
CI_PIP_ONNX_BACKEND_NAME: '${{ onnx_backend.key }}'
57+
${{ if ne(onnx_backend_version, '') }}:
58+
CI_PIP_ONNX_BACKEND_NAME: ${{ format('{0}=={1}', onnx_backend.key, onnx_backend_version) }}
59+
60+
${{ if eq(onnx_opset, '') }}:
61+
${{ format('{0} python{1} tf{2} onnx{3} {4}{5}', platform, python_version, tf_version, onnx_version, onnx_backend.key, onnx_backend_version) }}:
62+
${{ if eq(platform, 'linux') }}:
63+
CI_VM_IMAGE: 'ubuntu-16.04'
64+
${{ if eq(platform, 'windows') }}:
65+
CI_VM_IMAGE: 'vs2017-win2016'
66+
${{ if eq(platform, 'mac') }}:
67+
CI_VM_IMAGE: 'macOS-10.13'
68+
CI_PLATFORM: '${{ platform }}'
69+
CI_PYTHON_VERSION: '${{ python_version }}'
70+
CI_TF_VERSION: '${{ tf_version }}'
71+
CI_ONNX_VERSION: '${{ onnx_version }}'
72+
CI_ONNX_BACKEND: '${{ onnx_backend.key }}'
73+
CI_ONNX_BACKEND_VERSION: '${{ onnx_backend_version }}'
74+
75+
${{ if eq(tf_version, '') }}:
76+
CI_PIP_TF_NAME: 'tensorflow'
77+
${{ if ne(tf_version, '') }}:
78+
CI_PIP_TF_NAME: ${{ format('tensorflow=={0}', tf_version) }}
79+
80+
${{ if eq(onnx_version, '') }}:
81+
CI_PIP_ONNX_NAME: 'onnx'
82+
${{ if ne(onnx_version, '') }}:
83+
CI_PIP_ONNX_NAME: ${{ format('onnx=={0}', onnx_version) }}
84+
85+
${{ if eq(onnx_backend_version, '') }}:
86+
CI_PIP_ONNX_BACKEND_NAME: '${{ onnx_backend.key }}'
87+
${{ if ne(onnx_backend_version, '') }}:
88+
CI_PIP_ONNX_BACKEND_NAME: ${{ format('{0}=={1}', onnx_backend.key, onnx_backend_version) }}
7989

8090
# Insert all properties other than pool/steps/strategy
8191
${{ each pair in parameters.job }}:
@@ -108,3 +118,19 @@ jobs:
108118

109119
# Insert original steps
110120
- ${{ parameters.job.steps }}
121+
122+
# Report and publish test coverage
123+
# Save source path for combining coverage cross platform
124+
- bash: |
125+
coverage report
126+
mv .coverage ${BUILD_ARTIFACTSTAGINGDIRECTORY}/.coverage.${CI_PLATFORM}_python${CI_PYTHON_VERSION}_tf${CI_TF_VERSION}_onnx${CI_ONNX_VERSION}_${CI_ONNX_BACKEND}${CI_ONNX_VERSION}
127+
echo " ${SYSTEM_DEFAULTWORKINGDIRECTORY}" > ${BUILD_ARTIFACTSTAGINGDIRECTORY}/.coveragerc_paths.${CI_PLATFORM}_python${CI_PYTHON_VERSION}_tf${CI_TF_VERSION}_onnx${CI_ONNX_VERSION}_${CI_ONNX_BACKEND}${CI_ONNX_VERSION}
128+
condition: and(succeededOrFailed(), eq(${{ parameters.report_coverage }}, 'True'))
129+
displayName: 'Report Single Test Coverage'
130+
131+
- task: PublishBuildArtifacts@1
132+
condition: succeeded()
133+
inputs:
134+
pathtoPublish: '$(Build.ArtifactStagingDirectory)'
135+
artifactName: '${{ parameters.artifact_name }}'
136+
displayName: 'Deploy Single Test Coverage'

‎ci_build/azure_pipelines/templates/setup.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
steps:
44
- bash: |
55
set -ex
6-
pip install pytest pytest-cov pytest-runner graphviz requests pyyaml pillow pandas parameterized
6+
pip install pytest pytest-cov pytest-runner coverage graphviz requests pyyaml pillow pandas parameterized
77
pip install $(CI_PIP_TF_NAME) $(CI_PIP_ONNX_NAME) $(CI_PIP_ONNX_BACKEND_NAME)
88
99
# TF 1.10 requires numpy <=1.14.5 and >=1.13.3, but onnxruntime 0.2.1 does not work with numpy <= 1.14.5

‎ci_build/azure_pipelines/templates/unit_test.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ steps:
88
- bash: |
99
export TF2ONNX_TEST_BACKEND=$CI_ONNX_BACKEND
1010
export TF2ONNX_TEST_OPSET=$CI_ONNX_OPSET
11-
python -m pytest --cov=tf2onnx --cov-report=term --disable-pytest-warnings -r s tests
11+
python -m pytest --cov=tf2onnx --cov-report=term --disable-pytest-warnings -r s tests --cov-append
1212
timeoutInMinutes: 5
1313
displayName: ${{ format('Run UnitTest - Opset{0}', onnx_opset) }}
1414
condition: succeededOrFailed()

‎ci_build/azure_pipelines/unit_test-matrix.yml

Lines changed: 25 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,28 @@
11
# Unit test, full matrix
22

3-
jobs:
4-
- template: 'templates/job_generator.yml'
5-
parameters:
6-
platforms: ['linux', 'windows', 'mac']
7-
python_versions: ['3.6', '3.5']
8-
tf_versions: ['1.12', '1.11', '1.10', '1.9', '1.8', '1.7', '1.6', '1.5']
9-
onnx_opsets: ['']
10-
job:
11-
steps:
12-
- template: 'unit_test.yml'
3+
stages:
4+
- stage:
5+
jobs:
6+
- template: 'templates/job_generator.yml'
7+
parameters:
8+
platforms: ['linux', 'windows', 'mac']
9+
python_versions: ['3.6', '3.5']
10+
tf_versions: ['1.12', '1.11', '1.10', '1.9', '1.8', '1.7', '1.6', '1.5']
11+
onnx_opsets: ['']
12+
job:
13+
steps:
14+
- template: 'unit_test.yml'
15+
report_coverage: 'True'
16+
17+
- template: 'templates/job_generator.yml'
18+
parameters:
19+
platforms: ['linux', 'windows', 'mac']
20+
python_versions: ['3.7']
21+
tf_versions: ['1.13.1']
22+
onnx_opsets: ['']
23+
job:
24+
steps:
25+
- template: 'unit_test.yml'
26+
report_coverage: 'True'
1327

14-
- template: 'templates/job_generator.yml'
15-
parameters:
16-
platforms: ['linux', 'windows', 'mac']
17-
python_versions: ['3.7']
18-
tf_versions: ['1.13.1']
19-
onnx_opsets: ['']
20-
job:
21-
steps:
22-
- template: 'unit_test.yml'
28+
- template: 'templates/combine_test_coverage.yml'

‎ci_build/azure_pipelines/unit_test.yml

Lines changed: 32 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,35 @@
11
# Unit test
22

3-
jobs:
4-
- template: 'templates/job_generator.yml'
5-
parameters:
6-
python_versions: ['3.7', '3.6', '3.5']
7-
tf_versions: ['1.13.1']
8-
onnx_opsets: ['']
9-
job:
10-
steps:
11-
- template: 'unit_test.yml'
3+
stages:
4+
- stage:
5+
jobs:
6+
- template: 'templates/job_generator.yml'
7+
parameters:
8+
python_versions: ['3.7', '3.6', '3.5']
9+
tf_versions: ['1.13.1']
10+
onnx_opsets: ['']
11+
job:
12+
steps:
13+
- template: 'unit_test.yml'
14+
report_coverage: 'True'
15+
16+
- template: 'templates/job_generator.yml'
17+
parameters:
18+
tf_versions: ['1.12', '1.11', '1.10', '1.9', '1.8', '1.7', '1.6', '1.5']
19+
onnx_opsets: ['']
20+
job:
21+
steps:
22+
- template: 'unit_test.yml'
23+
report_coverage: 'True'
24+
25+
- template: 'templates/job_generator.yml'
26+
parameters:
27+
platforms: ['windows', 'mac']
28+
tf_versions: ['1.13.1']
29+
onnx_opsets: ['']
30+
job:
31+
steps:
32+
- template: 'unit_test.yml'
33+
report_coverage: 'True'
1234

13-
- template: 'templates/job_generator.yml'
14-
parameters:
15-
tf_versions: ['1.12', '1.11', '1.10', '1.9', '1.8', '1.7', '1.6', '1.5']
16-
onnx_opsets: ['']
17-
job:
18-
steps:
19-
- template: 'unit_test.yml'
20-
21-
- template: 'templates/job_generator.yml'
22-
parameters:
23-
platforms: ['windows', 'mac']
24-
tf_versions: ['1.13.1']
25-
onnx_opsets: ['']
26-
job:
27-
steps:
28-
- template: 'unit_test.yml'
35+
- template: 'templates/combine_test_coverage.yml'

0 commit comments

Comments
 (0)