Skip to content

Commit d9c1e34

Browse files
oiteloitel
authored
CI: Generate unit tests' performance report (#5629)
* CI: Generate unit tests' performance report * WIP * Build static Google Test library on Ubuntu * WIP * Fix PowerShell syntax * WIP * WIP * Do not fail on report generation error
1 parent e222794 commit d9c1e34

File tree

9 files changed

+160
-18
lines changed

9 files changed

+160
-18
lines changed

‎.github/workflows/build-test-linux-vcpkg.yml‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,8 @@ jobs:
179179
run: xvfb-run -a ./build/${{ matrix.config }}/bin/MeshViewer -hidden -noEventLoop -unloadPluginsAtEnd
180180

181181
- name: Unit Tests
182+
env:
183+
GTEST_OUTPUT: 'xml:unit_tests_report_gtest.xml'
182184
run: ./build/${{ matrix.config }}/bin/MRTest
183185

184186
- name: C Unit Tests (old bindings)
@@ -192,19 +194,25 @@ jobs:
192194
if: ${{ inputs.mrbind }}
193195
timeout-minutes: 8
194196
working-directory: ./build/${{ matrix.config }}/bin
195-
run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python'
197+
run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python' -a ' --junit-xml=../unit_tests_report_pytest.xml'
196198

197199
- name: Python Regression Tests
198200
if: ${{ inputs.internal_build && inputs.mrbind }}
199201
uses: ./.github/actions/python-regression-tests
200202
with:
201203
build_config: ${{ matrix.config }}
202204
mrbind: ${{ inputs.mrbind }}
203-
pytest_args: "--run-cuda=negative"
205+
pytest_args: "--run-cuda=negative --junit-xml=../unit_tests_report_regression.xml"
204206
smoke: ${{ !inputs.full_config_build && matrix.config == 'Debug' }}
205207
test_artifacts_path: linux-vcpkg-${{ matrix.arch }}/${{ matrix.config }}/${{ matrix.compiler }}
206208
upload_test_artifacts: ${{ inputs.upload_test_artifacts }}
207209

210+
- name: Generate Test Performance Report
211+
continue-on-error: true
212+
run: |
213+
scripts/junit_to_csv.py unit_tests_report.csv unit_tests_report_*.xml
214+
cat unit_tests_report.csv
215+
208216
- name: Create Package
209217
if: ${{ matrix.config == 'Release' && matrix.compiler == 'Clang 20' }}
210218
run: |

‎.github/workflows/build-test-macos.yml‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,8 @@ jobs:
173173
run: ./build/${{ matrix.config }}/bin/MeshViewer -tryHidden -noEventLoop -unloadPluginsAtEnd
174174

175175
- name: Unit Tests
176+
env:
177+
GTEST_OUTPUT: 'xml:unit_tests_report_gtest.xml'
176178
run: ./build/${{ matrix.config }}/bin/MRTest
177179

178180
- name: C Unit Tests (old bindings)
@@ -185,12 +187,12 @@ jobs:
185187
- name: Python Sanity Tests
186188
timeout-minutes: 8
187189
working-directory: ./build/${{ matrix.config }}/bin
188-
run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python'
190+
run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python' -a ' --junit-xml=../unit_tests_report_pytest.xml'
189191

190192
- name: Python Regression Tests
191193
if: ${{ inputs.internal_build }}
192194
env:
193-
RUN_CUDA_ARG: "--run-cuda=negative"
195+
RUN_CUDA_ARG: "--run-cuda=negative --junit-xml=../unit_tests_report_regression.xml"
194196
uses: ./.github/actions/python-regression-tests
195197
with:
196198
build_config: ${{ matrix.config }}
@@ -199,6 +201,12 @@ jobs:
199201
test_artifacts_path: macos/${{ matrix.os }}
200202
upload_test_artifacts: ${{ inputs.upload_test_artifacts }}
201203

204+
- name: Generate Test Performance Report
205+
continue-on-error: true
206+
run: |
207+
scripts/junit_to_csv.py unit_tests_report.csv unit_tests_report_*.xml
208+
cat unit_tests_report.csv
209+
202210
- name: Create Pkg
203211
if: ${{ matrix.config == 'Release' }}
204212
run: |

‎.github/workflows/build-test-ubuntu-arm64.yml‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,8 @@ jobs:
192192
run: xvfb-run -a ./build/${{ matrix.config }}/bin/MeshViewer -hidden -noEventLoop -unloadPluginsAtEnd
193193

194194
- name: Unit Tests
195+
env:
196+
GTEST_OUTPUT: 'xml:unit_tests_report_gtest.xml'
195197
run: ./build/${{ matrix.config }}/bin/MRTest
196198

197199
- name: C Unit Tests (old bindings)
@@ -205,18 +207,24 @@ jobs:
205207
if: ${{ inputs.mrbind }}
206208
timeout-minutes: 8
207209
working-directory: ./build/${{ matrix.config }}/bin
208-
run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python'
210+
run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python' -a ' --junit-xml=../unit_tests_report_pytest.xml'
209211

210212
- name: Python Regression Tests
211213
if: ${{ inputs.internal_build && inputs.mrbind }}
212214
uses: ./.github/actions/python-regression-tests
213215
with:
214216
build_config: ${{ matrix.config }}
215-
pytest_args: "--run-cuda=negative"
217+
pytest_args: "--run-cuda=negative --junit-xml=../unit_tests_report_regression.xml"
216218
smoke: ${{ !inputs.full_config_build && matrix.config == 'Debug' }}
217219
test_artifacts_path: ubuntu_arm64/${{ matrix.os }}
218220
upload_test_artifacts: ${{ inputs.upload_test_artifacts }}
219221

222+
- name: Generate Test Performance Report
223+
continue-on-error: true
224+
run: |
225+
scripts/junit_to_csv.py unit_tests_report.csv unit_tests_report_*.xml
226+
cat unit_tests_report.csv
227+
220228
- name: Create Deb
221229
env:
222230
MESHLIB_BUILD_RELEASE: "ON"

‎.github/workflows/build-test-ubuntu-x64.yml‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,8 @@ jobs:
142142
run: xvfb-run -a ./build/${{ matrix.config }}/bin/MeshViewer -hidden -noEventLoop -unloadPluginsAtEnd
143143

144144
- name: Unit Tests
145+
env:
146+
GTEST_OUTPUT: 'xml:unit_tests_report_gtest.xml'
145147
run: ./build/${{ matrix.config }}/bin/MRTest
146148

147149
- name: C Unit Tests (old bindings)
@@ -155,12 +157,12 @@ jobs:
155157
if: ${{ inputs.mrbind }}
156158
timeout-minutes: 8
157159
working-directory: ./build/${{ matrix.config }}/bin
158-
run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python'
160+
run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python' -a ' --junit-xml=../unit_tests_report_pytest.xml'
159161

160162
- name: Python Regression Tests
161163
if: ${{ inputs.internal_build && inputs.mrbind }}
162164
env:
163-
RUN_CUDA_ARG: "--run-cuda=negative"
165+
RUN_CUDA_ARG: "--run-cuda=negative --junit-xml=../unit_tests_report_regression.xml"
164166
uses: ./.github/actions/python-regression-tests
165167
with:
166168
build_config: ${{ matrix.config }}
@@ -169,6 +171,12 @@ jobs:
169171
test_artifacts_path: ${{ matrix.os }}/${{matrix.config}}/${{matrix.compiler}}
170172
upload_test_artifacts: ${{ inputs.upload_test_artifacts }}
171173

174+
- name: Generate Test Performance Report
175+
continue-on-error: true
176+
run: |
177+
scripts/junit_to_csv.py unit_tests_report.csv unit_tests_report_*.xml
178+
cat unit_tests_report.csv
179+
172180
- name: Create Deb
173181
if: ${{ matrix.compiler == 'GCC' && matrix.config == 'Release' }}
174182
env:

‎.github/workflows/build-test-windows.yml‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,8 @@ jobs:
238238
shell: cmd
239239

240240
- name: Unit Tests
241+
env:
242+
GTEST_OUTPUT: 'xml:unit_tests_report_gtest.xml'
241243
run: source\x64\${{ matrix.config }}\MRTest.exe
242244

243245
- name: C Unit Tests (old bindings)
@@ -249,24 +251,37 @@ jobs:
249251

250252
- name: C# Unit Tests
251253
if: ${{ env.BUILD_C_SHARP == 'true' }}
252-
run: dotnet run --project source/MRDotNet2Test -c ${{ matrix.config }}
254+
run: |
255+
dotnet build source\MRDotNet2Test -c ${{ matrix.config }}
256+
source\x64\${{ matrix.config }}\MRDotNet2Test.exe --result="_report_nunit.xml"
257+
# convert report to JUnit format
258+
[Environment]::CurrentDirectory = $pwd
259+
$xslt = New-Object System.Xml.Xsl.XslCompiledTransform
260+
$xslt.Load("scripts\devops\nunit3-junit.xslt")
261+
$xslt.Transform("_report_nunit.xml", "unit_tests_report_nunit.xml")
253262
254263
- name: Python Sanity Tests
255264
timeout-minutes: 8
256265
working-directory: source\x64\${{ matrix.config }}
257-
run: py -3 ..\..\..\scripts\run_python_test_script.py -d '..\test_python'
266+
run: py -3 ..\..\..\scripts\run_python_test_script.py -d '..\test_python' -a ' --junit-xml=../unit_tests_report_pytest.xml'
258267

259268
- name: Python Regression Tests
260269
if: ${{ inputs.internal_build }}
261270
uses: ./.github/actions/python-regression-tests
262271
with:
263272
build_config: ${{ matrix.config }}
264273
mrbind: ${{ inputs.mrbind }}
265-
pytest_args: "${{ env.BUILD_C_SHARP == 'true' && env.PYTEST_C_SHARP_ARGS || '' }} --run-cuda=negative"
274+
pytest_args: "${{ env.BUILD_C_SHARP == 'true' && env.PYTEST_C_SHARP_ARGS || '' }} --run-cuda=negative --junit-xml=../unit_tests_report_regression.xml"
266275
smoke: ${{ !inputs.full_config_build && matrix.config == 'Debug' }}
267276
test_artifacts_path: ${{ matrix.test_artifacts_path }}
268277
upload_test_artifacts: ${{ inputs.upload_test_artifacts }}
269278

279+
- name: Generate Test Performance Report
280+
continue-on-error: true
281+
run: |
282+
py -3 scripts/junit_to_csv.py unit_tests_report.csv (get-item unit_tests_report_*.xml)
283+
cat unit_tests_report.csv
284+
270285
- name: Delete unwanted files
271286
# Those files should not exist when using CMake.
272287
shell: cmd

‎requirements/ubuntu.txt‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ libfmt-dev
99
libfreetype-dev
1010
libgdcm-dev
1111
libglfw3-dev
12-
libgtest-dev
1312
libgtk-3-dev
1413
libhidapi-dev
1514
libhpdf-dev

‎scripts/devops/nunit3-junit.xslt‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<xsl:stylesheet version="2.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
3+
<xsl:output method="xml" indent="yes"/>
4+
5+
<xsl:template match="/test-run">
6+
<testsuites tests="{@testcasecount}" failures="{@failed}" disabled="{@skipped}" time="{@duration}">
7+
<xsl:apply-templates/>
8+
</testsuites>
9+
</xsl:template>
10+
11+
<xsl:template match="test-suite">
12+
<xsl:if test="test-case">
13+
<testsuite tests="{@testcasecount}" time="{@duration}" errors="{@testcasecount - @passed - @skipped - @failed}" failures="{@failed}" skipped="{@skipped}" timestamp="{@start-time}">
14+
<xsl:attribute name="name">
15+
<xsl:for-each select="ancestor-or-self::test-suite[@type='TestSuite']/@name">
16+
<xsl:value-of select="concat(., '.')"/>
17+
</xsl:for-each>
18+
</xsl:attribute>
19+
<xsl:apply-templates select="test-case"/>
20+
</testsuite>
21+
<xsl:apply-templates select="test-suite"/>
22+
</xsl:if>
23+
<xsl:if test="not(test-case)">
24+
<xsl:apply-templates/>
25+
</xsl:if>
26+
</xsl:template>
27+
28+
<xsl:template match="test-case">
29+
<testcase name="{@name}" assertions="{@asserts}" time="{@duration}" status="{@result}" classname="{@classname}">
30+
<xsl:if test="@runstate = 'Skipped' or @runstate = 'Ignored'">
31+
<skipped/>
32+
</xsl:if>
33+
34+
<xsl:apply-templates/>
35+
</testcase>
36+
</xsl:template>
37+
38+
<xsl:template match="command-line"/>
39+
<xsl:template match="settings"/>
40+
<xsl:template match="filter"/>
41+
42+
<xsl:template match="output">
43+
<system-out>
44+
<xsl:value-of select="."/>
45+
</system-out>
46+
</xsl:template>
47+
48+
<xsl:template match="stack-trace">
49+
</xsl:template>
50+
51+
<xsl:template match="test-case/failure">
52+
<failure message="{./message}">
53+
<xsl:value-of select="./stack-trace"/>
54+
</failure>
55+
</xsl:template>
56+
57+
<xsl:template match="test-suite/failure"/>
58+
59+
<xsl:template match="test-case/reason">
60+
<xsl:if test="./message != null">
61+
<skipped message="{./message}"/>
62+
</xsl:if>
63+
</xsl:template>
64+
65+
<xsl:template match="test-case/assertions">
66+
</xsl:template>
67+
68+
<xsl:template match="test-suite/reason"/>
69+
70+
<xsl:template match="properties"/>
71+
</xsl:stylesheet>
72+

‎scripts/junit_to_csv.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/usr/bin/env python3
2+
import csv
3+
import sys
4+
import xml.etree.ElementTree as ET
5+
6+
7+
def gather_info(testsuite):
8+
suite_name = testsuite.attrib['name']
9+
for testcase in testsuite:
10+
class_name = testcase.attrib['classname']
11+
name = testcase.attrib['name']
12+
time = testcase.attrib['time']
13+
yield [suite_name, class_name, name, time]
14+
15+
16+
if __name__ == "__main__":
17+
with open(sys.argv[1], 'w') as output:
18+
writer = csv.writer(output, delimiter=',', quotechar='"')
19+
writer.writerow(["suite_name", "class_name", "name", "time"])
20+
for arg in sys.argv[2:]:
21+
parser = ET.XMLParser(target=ET.TreeBuilder(insert_comments=True))
22+
report = ET.parse(arg, parser)
23+
for testsuite in report.getroot():
24+
for info in gather_info(testsuite):
25+
writer.writerow(info)

‎thirdparty/CMakeLists.txt‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,11 @@ IF(MR_EMSCRIPTEN)
3838
ENDIF()
3939
ENDIF()
4040

41-
IF(APPLE)
42-
set(INSTALL_GTEST ON)
41+
set(INSTALL_GTEST ON)
42+
IF(EMSCRIPTEN)
43+
add_compile_definitions(GTEST_HAS_CXXABI_H_=0)
44+
add_subdirectory(./googletest ./googletest)
45+
ELSE()
4346
# https://stackoverflow.com/a/42643260/7325599
4447
set(BUILD_SHARED_LIBS_OLD ${BUILD_SHARED_LIBS})
4548
set(BUILD_SHARED_LIBS ON CACHE BOOL "" FORCED)
@@ -94,10 +97,6 @@ IF(EMSCRIPTEN)
9497
set(GDCM_BUILD_DOCBOOK_MANPAGES OFF)
9598
add_subdirectory(./GDCM)
9699

97-
add_compile_definitions(GTEST_HAS_CXXABI_H_=0)
98-
set(INSTALL_GTEST ON)
99-
add_subdirectory(./googletest)
100-
101100
set(BUILD_TOOLS OFF CACHE BOOL "")
102101
set(BUILD_REGRESS OFF CACHE BOOL "")
103102
set(BUILD_EXAMPLES OFF CACHE BOOL "")

0 commit comments

Comments
 (0)