Skip to content

Commit 5d13f4b

Browse files
MionszMionsz
committed
Fix: Update validation-tests.yml pipenv install issues
Fix: Update validation-tests.yml pipenv install issues and run issues Add: Multiple test cases run inputs Add: Dropdown for pre-release use cases Add: Fixtures.py time.sleep between runs Signed-off-by: Milosz Linkiewicz <[email protected]>
1 parent 31c21a1 commit 5d13f4b

File tree

3 files changed

+231
-32
lines changed

3 files changed

+231
-32
lines changed

‎.github/workflows/validation-tests.yml‎

Lines changed: 163 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -9,23 +9,88 @@ on:
99
default: 'main'
1010
required: false
1111
description: 'Branch name to use'
12-
validation-tests:
12+
validation-tests-1:
1313
type: string
1414
default: 'tests/single/video/pacing'
1515
required: true
16-
description: 'Validation tests to run'
16+
description: '1st validation tests to run'
17+
validation-tests-2:
18+
type: string
19+
default: ''
20+
required: false
21+
description: '2nd validation tests to run'
22+
validation-tests-3:
23+
type: string
24+
default: ''
25+
required: false
26+
description: '3th validation tests to run'
27+
validation-tests-4:
28+
type: string
29+
default: ''
30+
required: false
31+
description: '4th validation tests to run'
32+
validation-tests-5:
33+
type: string
34+
default: ''
35+
required: false
36+
description: '5th validation tests to run'
37+
validation-pre-release-1:
38+
description: 'Select from pre-release group tests nr-1'
39+
required: false
40+
type: choice
41+
options:
42+
- NONE
43+
- ancillary
44+
- kernel-socket
45+
- rss-mode
46+
- st20p
47+
- st30p
48+
- st41
49+
- udp
50+
- video
51+
- xdp
52+
validation-pre-release-2:
53+
description: 'Select from pre-release group tests nr-2'
54+
required: false
55+
type: choice
56+
options:
57+
- NONE
58+
- ffmpeg-plugin
59+
- fuzzy-tests
60+
- performance
61+
- ptp
62+
- rx-timing
63+
- vero
64+
- virtio-enable
65+
- wrong-parameter
66+
validation-pre-release-3:
67+
description: 'Select from pre-release group tests nr-3'
68+
required: false
69+
type: choice
70+
options:
71+
- NONE
72+
- gpu-direct
73+
- gpu-enabling
1774

1875
env:
1976
# Customize the env if
2077
BUILD_TYPE: 'Release'
2178
DPDK_VERSION: '23.11'
22-
# Bellow ENV variables are required to be defined on runner side:
23-
# TEST_PF_PORT_P: '0000:49:00.0'
24-
# TEST_PF_PORT_R: '0000:49:00.1'
25-
# TEST_PORT_P: '0000:49:01.2'
26-
# TEST_PORT_R: '0000:49:01.3'
27-
# TEST_DMA_PORT_P: '0000:6a:01.0'
28-
# TEST_DMA_PORT_R: '0000:6f:01.0'
79+
# Bellow ENV variables example set is defined on the self-hosted runner side:
80+
# TEST_PF_PORT_P=0000:49:00.0
81+
# TEST_PF_PORT_R=0000:49:00.1
82+
# TEST_PORT_P=0000:49:01.2
83+
# TEST_PORT_R=0000:49:01.3
84+
# TEST_DMA_PORT_P=0000:80:04.0
85+
# TEST_DMA_PORT_R=0000:80:04.1
86+
# TEST_VF_PORT_P_0=0000:49:01.0
87+
# TEST_VF_PORT_P_1=0000:49:01.1
88+
# TEST_VF_PORT_P_2=0000:49:01.2
89+
# TEST_VF_PORT_P_3=0000:49:01.3
90+
# TEST_VF_PORT_R_0=0000:49:11.0
91+
# TEST_VF_PORT_R_1=0000:49:11.1
92+
# TEST_VF_PORT_R_2=0000:49:11.2
93+
# TEST_VF_PORT_R_3=0000:49:11.3
2994

3095
permissions:
3196
contents: read
@@ -34,78 +99,149 @@ jobs:
3499
validation-build-mtl:
35100
runs-on: [Linux, self-hosted, DPDK]
36101
timeout-minutes: 60
37-
102+
outputs:
103+
pipenv-activate: ${{ steps.pipenv-install.outputs.VIRTUAL_ENV }}
38104
steps:
39-
- name: Harden Runner
105+
- name: 'preparation: Harden Runner'
40106
uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
41107
with:
42108
egress-policy: audit
43109

44-
- name: Checkout MTL
110+
- name: 'preparation: Restore valid owner to repository and directories'
111+
if: always()
112+
run: |
113+
sudo chown -R "${USER}" "$(pwd)"
114+
115+
- name: 'preparation: Checkout MTL'
45116
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
46117
with:
47118
ref: '${{ inputs.branch-to-checkout }}'
48119

49-
- name: Checkout DPDK
120+
- name: 'preparation: Checkout DPDK'
50121
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
51122
with:
52123
repository: 'DPDK/dpdk'
53124
ref: 'v${{ env.DPDK_VERSION }}'
54125
path: 'dpdk'
55126

56-
- name: Install the build dependency
127+
- name: 'configuration: Install the build dependency'
57128
run: |
58129
sudo apt-get install -y git gcc meson python3 python3-pyelftools pkg-config libnuma-dev libjson-c-dev libpcap-dev libgtest-dev libsdl2-dev libsdl2-ttf-dev libssl-dev
59-
sudo apt-get install -y systemtap-sdt-dev pipenv
130+
sudo apt-get install -y systemtap-sdt-dev
60131
61-
- name: Apply dpdk patches
132+
- name: 'configuration: Apply dpdk patches'
62133
run: |
63134
patch -d "dpdk" -p1 -i <(cat patches/dpdk/${{ env.DPDK_VERSION }}/*.patch)
64135
65-
- name: Build dpdk
136+
- name: 'installation: Build dpdk'
66137
run: |
67138
cd dpdk
68139
meson build
69140
ninja -C build
70141
cd build
71142
sudo ninja install
72143
73-
- name: Build
144+
- name: 'installation: Build mtl'
74145
run: |
75146
./build.sh
76147
sudo ldconfig
77148
78-
- name: Prepare pipenv environment
149+
- name: 'installation: Install pipenv environment'
79150
working-directory: tests/validation
151+
id: pipenv-install
80152
run: |
81-
pipenv install -r requirements.txt
153+
sudo apt remove -y pipenv || true
154+
sudo apt install -y python3-virtualenv python3-pip
155+
python3 -m pip install pipenv
156+
python3 -m pipenv install -r requirements.txt
157+
echo "VIRTUAL_ENV=$(python3 -m pipenv --venv)/bin/activate" >> "$GITHUB_ENV"
82158
83159
validation-run-tests:
84160
needs: [validation-build-mtl]
85161
runs-on: [Linux, self-hosted, DPDK]
162+
env:
163+
PYTEST_ALIAS: 'sudo --preserve-env python3 -m pipenv run pytest'
164+
PYTEST_PARAMS: '--nic="${TEST_PORT_P},${TEST_PORT_R}" --media=/mnt/media --build="../.."'
86165
steps:
87-
- name: Harden Runner
166+
- name: 'preparation: Harden Runner'
88167
uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
89168
with:
90169
egress-policy: audit
91170

92-
- name: Kill previous pytest routine
171+
- name: 'preparation: Kill MtlManager and pytest routines'
93172
run: |
94173
sudo killall -SIGINT pipenv || true
95174
sudo killall -SIGINT pytest || true
96175
sudo killall -SIGINT MtlManager || true
97176
98-
- name: Binding network adapter
177+
- name: 'preparation: Binding network adapter'
99178
run: |
100-
sudo ./script/nicctl.sh create_vf "${TEST_PF_PORT_P}" || true
179+
sudo rmmod irdma || true
180+
sudo ./script/nicctl.sh create_tvf "${TEST_PF_PORT_P}" || true
181+
sudo ./script/nicctl.sh create_tvf "${TEST_PF_PORT_R}" || true
101182
sudo ./dpdk/usertools/dpdk-devbind.py -b vfio-pci "${TEST_DMA_PORT_P}" || true
102183
sudo ./dpdk/usertools/dpdk-devbind.py -b vfio-pci "${TEST_DMA_PORT_R}" || true
103184
104-
- name: Start MtlManager at background
185+
- name: 'preparation: Start MtlManager at background'
105186
run: |
106187
sudo MtlManager &
107188
108-
- name: Run tests in pipenv environment
189+
- name: 'execution: Run validation-tests-1 in pipenv environment'
190+
if: always() && inputs.validation-tests-1 != ''
191+
working-directory: tests/validation
192+
run: |
193+
${{ env.PYTEST_ALIAS }} '${{ inputs.validation-tests-1 }}' ${{ env.PYTEST_PARAMS }}
194+
195+
- name: 'execution: Run validation-tests-2 in pipenv environment'
196+
if: always() && inputs.validation-tests-2 != ''
197+
working-directory: tests/validation
198+
run: |
199+
${{ env.PYTEST_ALIAS }} '${{ inputs.validation-tests-2 }}' ${{ env.PYTEST_PARAMS }}
200+
201+
- name: 'execution: Run validation-tests-3 in pipenv environment'
202+
if: always() && inputs.validation-tests-3 != ''
203+
working-directory: tests/validation
204+
run: |
205+
${{ env.PYTEST_ALIAS }} '${{ inputs.validation-tests-3 }}' ${{ env.PYTEST_PARAMS }}
206+
207+
- name: 'execution: Run validation-tests-4 in pipenv environment'
208+
if: always() && inputs.validation-tests-4 != ''
209+
working-directory: tests/validation
210+
run: |
211+
${{ env.PYTEST_ALIAS }} '${{ inputs.validation-tests-4 }}' ${{ env.PYTEST_PARAMS }}
212+
213+
- name: 'execution: Run validation-tests-5 in pipenv environment'
214+
if: always() && inputs.validation-tests-5 != ''
215+
working-directory: tests/validation
216+
run: |
217+
${{ env.PYTEST_ALIAS }} '${{ inputs.validation-tests-5 }}' ${{ env.PYTEST_PARAMS }}
218+
219+
- name: 'execution: Run validation-pre-release-1 in pipenv environment'
220+
if: always() && inputs.validation-pre-release-1 != 'NONE'
221+
working-directory: tests/validation
222+
run: |
223+
echo "== TO BE IMPLEMENTED ${{ inputs.inputs.validation-pre-release-1 }} =="
224+
225+
- name: 'execution: Run validation-pre-release-2 in pipenv environment'
226+
if: always() && inputs.validation-pre-release-2 != 'NONE'
109227
working-directory: tests/validation
110228
run: |
111-
sudo pipenv run pytest '${{ inputs.validation-tests }}' --nic="${TEST_PORT_P},${TEST_PORT_R}" --media=/mnt/media
229+
echo "== TO BE IMPLEMENTED ${{ inputs.inputs.validation-pre-release-2 }} =="
230+
231+
- name: 'execution: Run validation-pre-release-3 in pipenv environment'
232+
if: always() && inputs.validation-pre-release-3 != 'NONE'
233+
working-directory: tests/validation
234+
run: |
235+
echo "== TO BE IMPLEMENTED ${{ inputs.inputs.validation-pre-release-3 }} =="
236+
237+
- name: 'cleanup: Kill MtlManager and pytest routines'
238+
if: always()
239+
run: |
240+
sudo killall -SIGINT pipenv || true
241+
sudo killall -SIGINT pytest || true
242+
sudo killall -SIGINT MtlManager || true
243+
244+
- name: 'cleanup: Restore valid owner to repository and directories'
245+
if: always()
246+
run: |
247+
sudo chown -R "${USER}" "$(pwd)"

‎tests/validation/README.md‎

Lines changed: 61 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,19 +27,75 @@ Folder `tests`. Logs from executions are available in `logs` folder. Latest resu
2727

2828
Test engine is based on pytest.
2929

30-
Installation:
30+
Installation can be done in 2, similar in outcome, ways:
31+
32+
Approach using pure venv:
3133

3234
```bash
33-
python -m venv .venv
35+
# Install mandatory system packages
36+
sudo apt update
37+
sudo apt install -y python3-dev python3-virtualenv python3-venv python3-pip
38+
# Create virtual environment (venv) for python3
39+
python3 -m venv .venv
40+
# Activate venv for python3
3441
source .venv/bin/activate
35-
pip install -r requirements.txt
42+
# Install required python3 modules
43+
python3 -m pip install -r requirements.txt
44+
# User now can run command like pytest using:
45+
sudo --preserve-env python3 -m pytest
46+
```
47+
48+
Approach using pipenv:
49+
50+
```bash
51+
# Install mandatory system packages
52+
sudo apt update
53+
sudo apt install -y python3-dev python3-virtualenv python3-venv python3-pip
54+
# Install pipenv for python3
55+
python3 -m pip install pipenv
56+
# Install venv using pipenv for python3
57+
python3 -m pipenv install -r requirements.txt
58+
# User now can run command like pytest using:
59+
sudo --preserve-env python3 -m pipenv run pytest
60+
# User can also activate pipenv shell to have it persistent:
61+
sudo --preserve-env python3 -m pipenv shell
62+
```
63+
64+
Testcases should be run using `sudo` or root account. The basic and minimal approach command takes `--nic="${TEST_PORT_P},${TEST_PORT_R}"` as input.
65+
Self-hosted runners will have an media directory that is available under `/mnt/media` path (this is the default value for media parameter: `--media=/mnt/media`).
66+
Simple example when being run from tests/validation subdirectory will look like for pipenv:
67+
68+
```bash
69+
sudo \
70+
--preserve-env python3 \
71+
-m pipenv run pytest \
72+
--nic="${TEST_PORT_P},${TEST_PORT_R}" \
73+
--media=/mnt/media \
74+
--build="../.."
3675
```
3776

3877
Content of tests repository:
3978

4079
- invalid - negative tests to check library response to wrong values
4180
- single - functional tests of different features. These tests are using single host and can be run on PF or VF.
42-
- dpdk - tests using DPDK driver
81+
- xdp
82+
- kernel
83+
- ancillary
84+
- audio
85+
- dma
86+
- ffmpeg
87+
- gstreamer
88+
- kernel_socket - tests using kernel sockets
89+
- performance
90+
- ptp
91+
- rss_mode
92+
- rx_timing
93+
- st20p
94+
- st22p
95+
- st30p
96+
- st41
97+
- udp
98+
- video
99+
- virtio_user
43100
- xdp - tests using XDP driver mode
44-
- kernel - tests using kernel sockets
45101
- dual - functional, load and stress tests. These tests require dual host setup with switch.

‎tests/validation/tests/Engine/fixtures.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
from typing import Dict
55

66
import pytest
7+
import time
78

89
from .stash import clear_result_media, remove_result_media
910

@@ -96,3 +97,9 @@ def test_time(request):
9697
if test_time is None:
9798
return 30
9899
return int(test_time)
100+
101+
@pytest.fixture(autouse=True)
102+
def delay_between_tests():
103+
yield
104+
print("Sleeping 5s between tests cases runs.")
105+
time.sleep(5)

0 commit comments

Comments
 (0)