Skip to content

Commit 61fee7f

Browse files
committed
Merge remote-tracking branch 'remotes/philmd-gitlab/tags/acceptance-testing-20200622' into staging
Acceptance tests patches - List acceptance test reviewers in MAINTAINERS - Record/Replay tests from Pavel Dovgalyuk Example of use: $ avocado --show=app,replay run -t machine:vexpress-a9 tests/acceptance/replay_kernel.py Fetching asset from tests/acceptance/replay_kernel.py:ReplayKernel.test_arm_vexpressa9 (1/1) tests/acceptance/replay_kernel.py:ReplayKernel.test_arm_vexpressa9: replay: recording the execution... replay: finished the recording with log size 204784 bytes replay: elapsed time 6.44 sec replay: replaying the execution... replay: successfully finished the replay replay: elapsed time 7.97 sec replay: replay overhead 23.86% PASS (14.67 s) Travis-CI: https://travis-ci.org/github/philmd/qemu/jobs/700787719 # gpg: Signature made Mon 22 Jun 2020 09:58:13 BST # gpg: using RSA key FAABE75E12917221DCFD6BB2E3E32C2CDEADC0DE # gpg: Good signature from "Philippe Mathieu-Daudé (F4BUG) <[email protected]>" [full] # Primary key fingerprint: FAAB E75E 1291 7221 DCFD 6BB2 E3E3 2C2C DEAD C0DE * remotes/philmd-gitlab/tags/acceptance-testing-20200622: tests/acceptance: record/replay tests with advcal images tests/acceptance: add record/replay test for m68k tests/acceptance: add record/replay test for ppc64 tests/acceptance: add record/replay test for arm tests/acceptance: add record/replay test for aarch64 tests/acceptance: add kernel record/replay test for x86_64 tests/acceptance: add base class record/replay kernel tests MAINTAINERS: Add an entry to review Avocado based acceptance tests Signed-off-by: Peter Maydell <[email protected]>
2 parents 171199f + b52d7e2 commit 61fee7f

File tree

2 files changed

+309
-0
lines changed

2 files changed

+309
-0
lines changed

MAINTAINERS

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2538,6 +2538,7 @@ F: net/filter-replay.c
25382538
F: include/sysemu/replay.h
25392539
F: docs/replay.txt
25402540
F: stubs/replay.c
2541+
F: tests/acceptance/replay_kernel.py
25412542

25422543
IOVA Tree
25432544
M: Peter Xu <[email protected]>
@@ -2945,6 +2946,14 @@ S: Maintained
29452946
F: tests/tcg/Makefile
29462947
F: tests/tcg/Makefile.include
29472948

2949+
Acceptance (Integration) Testing with the Avocado framework
2950+
W: https://trello.com/b/6Qi1pxVn/avocado-qemu
2951+
R: Cleber Rosa <[email protected]>
2952+
R: Philippe Mathieu-Daudé <[email protected]>
2953+
R: Wainer dos Santos Moschetta <[email protected]>
2954+
S: Odd Fixes
2955+
F: tests/acceptance/
2956+
29482957
Documentation
29492958
-------------
29502959
Build system architecture

tests/acceptance/replay_kernel.py

Lines changed: 300 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,300 @@
1+
# Record/replay test that boots a Linux kernel
2+
#
3+
# Copyright (c) 2020 ISP RAS
4+
#
5+
# Author:
6+
# Pavel Dovgalyuk <[email protected]>
7+
#
8+
# This work is licensed under the terms of the GNU GPL, version 2 or
9+
# later. See the COPYING file in the top-level directory.
10+
11+
import os
12+
import logging
13+
import time
14+
15+
from avocado import skipIf
16+
from avocado_qemu import wait_for_console_pattern
17+
from avocado.utils import archive
18+
from avocado.utils import process
19+
from boot_linux_console import LinuxKernelTest
20+
21+
class ReplayKernel(LinuxKernelTest):
22+
"""
23+
Boots a Linux kernel in record mode and checks that the console
24+
is operational and the kernel command line is properly passed
25+
from QEMU to the kernel.
26+
Then replays the same scenario and verifies, that QEMU correctly
27+
terminates.
28+
"""
29+
30+
timeout = 90
31+
KERNEL_COMMON_COMMAND_LINE = 'printk.time=1 panic=-1 '
32+
33+
def run_vm(self, kernel_path, kernel_command_line, console_pattern,
34+
record, shift, args, replay_path):
35+
logger = logging.getLogger('replay')
36+
start_time = time.time()
37+
vm = self.get_vm()
38+
vm.set_console()
39+
if record:
40+
logger.info('recording the execution...')
41+
mode = 'record'
42+
else:
43+
logger.info('replaying the execution...')
44+
mode = 'replay'
45+
vm.add_args('-icount', 'shift=%s,rr=%s,rrfile=%s' %
46+
(shift, mode, replay_path),
47+
'-kernel', kernel_path,
48+
'-append', kernel_command_line,
49+
'-net', 'none',
50+
'-no-reboot')
51+
if args:
52+
vm.add_args(*args)
53+
vm.launch()
54+
self.wait_for_console_pattern(console_pattern, vm)
55+
if record:
56+
vm.shutdown()
57+
logger.info('finished the recording with log size %s bytes'
58+
% os.path.getsize(replay_path))
59+
else:
60+
vm.wait()
61+
logger.info('successfully finished the replay')
62+
elapsed = time.time() - start_time
63+
logger.info('elapsed time %.2f sec' % elapsed)
64+
return elapsed
65+
66+
def run_rr(self, kernel_path, kernel_command_line, console_pattern,
67+
shift=7, args=None):
68+
replay_path = os.path.join(self.workdir, 'replay.bin')
69+
t1 = self.run_vm(kernel_path, kernel_command_line, console_pattern,
70+
True, shift, args, replay_path)
71+
t2 = self.run_vm(kernel_path, kernel_command_line, console_pattern,
72+
False, shift, args, replay_path)
73+
logger = logging.getLogger('replay')
74+
logger.info('replay overhead {:.2%}'.format(t2 / t1 - 1))
75+
76+
@skipIf(os.getenv('CONTINUOUS_INTEGRATION'), 'Running on Travis-CI')
77+
def test_x86_64_pc(self):
78+
"""
79+
:avocado: tags=arch:x86_64
80+
:avocado: tags=machine:pc
81+
"""
82+
kernel_url = ('https://archives.fedoraproject.org/pub/archive/fedora'
83+
'/linux/releases/29/Everything/x86_64/os/images/pxeboot'
84+
'/vmlinuz')
85+
kernel_hash = '23bebd2680757891cf7adedb033532163a792495'
86+
kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
87+
88+
kernel_command_line = self.KERNEL_COMMON_COMMAND_LINE + 'console=ttyS0'
89+
console_pattern = 'VFS: Cannot open root device'
90+
91+
self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5)
92+
93+
def test_aarch64_virt(self):
94+
"""
95+
:avocado: tags=arch:aarch64
96+
:avocado: tags=machine:virt
97+
:avocado: tags=cpu:cortex-a53
98+
"""
99+
kernel_url = ('https://archives.fedoraproject.org/pub/archive/fedora'
100+
'/linux/releases/29/Everything/aarch64/os/images/pxeboot'
101+
'/vmlinuz')
102+
kernel_hash = '8c73e469fc6ea06a58dc83a628fc695b693b8493'
103+
kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
104+
105+
kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
106+
'console=ttyAMA0')
107+
console_pattern = 'VFS: Cannot open root device'
108+
109+
self.run_rr(kernel_path, kernel_command_line, console_pattern,
110+
args=('-cpu', 'cortex-a53'))
111+
112+
def test_arm_virt(self):
113+
"""
114+
:avocado: tags=arch:arm
115+
:avocado: tags=machine:virt
116+
"""
117+
kernel_url = ('https://archives.fedoraproject.org/pub/archive/fedora'
118+
'/linux/releases/29/Everything/armhfp/os/images/pxeboot'
119+
'/vmlinuz')
120+
kernel_hash = 'e9826d741b4fb04cadba8d4824d1ed3b7fb8b4d4'
121+
kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
122+
123+
kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
124+
'console=ttyAMA0')
125+
console_pattern = 'VFS: Cannot open root device'
126+
127+
self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=1)
128+
129+
def test_arm_cubieboard_initrd(self):
130+
"""
131+
:avocado: tags=arch:arm
132+
:avocado: tags=machine:cubieboard
133+
"""
134+
deb_url = ('https://apt.armbian.com/pool/main/l/'
135+
'linux-4.20.7-sunxi/linux-image-dev-sunxi_5.75_armhf.deb')
136+
deb_hash = '1334c29c44d984ffa05ed10de8c3361f33d78315'
137+
deb_path = self.fetch_asset(deb_url, asset_hash=deb_hash)
138+
kernel_path = self.extract_from_deb(deb_path,
139+
'/boot/vmlinuz-4.20.7-sunxi')
140+
dtb_path = '/usr/lib/linux-image-dev-sunxi/sun4i-a10-cubieboard.dtb'
141+
dtb_path = self.extract_from_deb(deb_path, dtb_path)
142+
initrd_url = ('https://github.com/groeck/linux-build-test/raw/'
143+
'2eb0a73b5d5a28df3170c546ddaaa9757e1e0848/rootfs/'
144+
'arm/rootfs-armv5.cpio.gz')
145+
initrd_hash = '2b50f1873e113523967806f4da2afe385462ff9b'
146+
initrd_path_gz = self.fetch_asset(initrd_url, asset_hash=initrd_hash)
147+
initrd_path = os.path.join(self.workdir, 'rootfs.cpio')
148+
archive.gzip_uncompress(initrd_path_gz, initrd_path)
149+
150+
kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
151+
'console=ttyS0,115200 '
152+
'usbcore.nousb '
153+
'panic=-1 noreboot')
154+
console_pattern = 'Boot successful.'
155+
self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=1,
156+
args=('-dtb', dtb_path,
157+
'-initrd', initrd_path,
158+
'-no-reboot'))
159+
160+
def test_ppc64_pseries(self):
161+
"""
162+
:avocado: tags=arch:ppc64
163+
:avocado: tags=machine:pseries
164+
"""
165+
kernel_url = ('https://archives.fedoraproject.org/pub/archive'
166+
'/fedora-secondary/releases/29/Everything/ppc64le/os'
167+
'/ppc/ppc64/vmlinuz')
168+
kernel_hash = '3fe04abfc852b66653b8c3c897a59a689270bc77'
169+
kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
170+
171+
kernel_command_line = self.KERNEL_COMMON_COMMAND_LINE + 'console=hvc0'
172+
# icount is not good enough for PPC64 for complete boot yet
173+
console_pattern = 'Kernel command line: %s' % kernel_command_line
174+
self.run_rr(kernel_path, kernel_command_line, console_pattern)
175+
176+
def test_m68k_q800(self):
177+
"""
178+
:avocado: tags=arch:m68k
179+
:avocado: tags=machine:q800
180+
"""
181+
deb_url = ('https://snapshot.debian.org/archive/debian-ports'
182+
'/20191021T083923Z/pool-m68k/main'
183+
'/l/linux/kernel-image-5.3.0-1-m68k-di_5.3.7-1_m68k.udeb')
184+
deb_hash = '044954bb9be4160a3ce81f8bc1b5e856b75cccd1'
185+
deb_path = self.fetch_asset(deb_url, asset_hash=deb_hash)
186+
kernel_path = self.extract_from_deb(deb_path,
187+
'/boot/vmlinux-5.3.0-1-m68k')
188+
189+
kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
190+
'console=ttyS0 vga=off')
191+
console_pattern = 'No filesystem could mount root'
192+
self.run_rr(kernel_path, kernel_command_line, console_pattern)
193+
194+
def do_test_advcal_2018(self, file_path, kernel_name, args=None):
195+
archive.extract(file_path, self.workdir)
196+
197+
for entry in os.scandir(self.workdir):
198+
if entry.name.startswith('day') and entry.is_dir():
199+
kernel_path = os.path.join(entry.path, kernel_name)
200+
break
201+
202+
kernel_command_line = ''
203+
console_pattern = 'QEMU advent calendar'
204+
self.run_rr(kernel_path, kernel_command_line, console_pattern,
205+
args=args)
206+
207+
def test_arm_vexpressa9(self):
208+
"""
209+
:avocado: tags=arch:arm
210+
:avocado: tags=machine:vexpress-a9
211+
"""
212+
tar_hash = '32b7677ce8b6f1471fb0059865f451169934245b'
213+
tar_url = ('https://www.qemu-advent-calendar.org'
214+
'/2018/download/day16.tar.xz')
215+
file_path = self.fetch_asset(tar_url, asset_hash=tar_hash)
216+
dtb_path = self.workdir + '/day16/vexpress-v2p-ca9.dtb'
217+
self.do_test_advcal_2018(file_path, 'winter.zImage',
218+
args=('-dtb', dtb_path))
219+
220+
def test_m68k_mcf5208evb(self):
221+
"""
222+
:avocado: tags=arch:m68k
223+
:avocado: tags=machine:mcf5208evb
224+
"""
225+
tar_hash = 'ac688fd00561a2b6ce1359f9ff6aa2b98c9a570c'
226+
tar_url = ('https://www.qemu-advent-calendar.org'
227+
'/2018/download/day07.tar.xz')
228+
file_path = self.fetch_asset(tar_url, asset_hash=tar_hash)
229+
self.do_test_advcal_2018(file_path, 'sanity-clause.elf')
230+
231+
def test_microblaze_s3adsp1800(self):
232+
"""
233+
:avocado: tags=arch:microblaze
234+
:avocado: tags=machine:petalogix-s3adsp1800
235+
"""
236+
tar_hash = '08bf3e3bfb6b6c7ce1e54ab65d54e189f2caf13f'
237+
tar_url = ('https://www.qemu-advent-calendar.org'
238+
'/2018/download/day17.tar.xz')
239+
file_path = self.fetch_asset(tar_url, asset_hash=tar_hash)
240+
self.do_test_advcal_2018(file_path, 'ballerina.bin')
241+
242+
def test_ppc64_e500(self):
243+
"""
244+
:avocado: tags=arch:ppc64
245+
:avocado: tags=machine:ppce500
246+
:avocado: tags=cpu:e5500
247+
"""
248+
tar_hash = '6951d86d644b302898da2fd701739c9406527fe1'
249+
tar_url = ('https://www.qemu-advent-calendar.org'
250+
'/2018/download/day19.tar.xz')
251+
file_path = self.fetch_asset(tar_url, asset_hash=tar_hash)
252+
self.do_test_advcal_2018(file_path, 'uImage', ('-cpu', 'e5500'))
253+
254+
def test_ppc_g3beige(self):
255+
"""
256+
:avocado: tags=arch:ppc
257+
:avocado: tags=machine:g3beige
258+
"""
259+
tar_hash = 'e0b872a5eb8fdc5bed19bd43ffe863900ebcedfc'
260+
tar_url = ('https://www.qemu-advent-calendar.org'
261+
'/2018/download/day15.tar.xz')
262+
file_path = self.fetch_asset(tar_url, asset_hash=tar_hash)
263+
self.do_test_advcal_2018(file_path, 'invaders.elf',
264+
args=('-M', 'graphics=off'))
265+
266+
def test_ppc_mac99(self):
267+
"""
268+
:avocado: tags=arch:ppc
269+
:avocado: tags=machine:mac99
270+
"""
271+
tar_hash = 'e0b872a5eb8fdc5bed19bd43ffe863900ebcedfc'
272+
tar_url = ('https://www.qemu-advent-calendar.org'
273+
'/2018/download/day15.tar.xz')
274+
file_path = self.fetch_asset(tar_url, asset_hash=tar_hash)
275+
self.do_test_advcal_2018(file_path, 'invaders.elf',
276+
args=('-M', 'graphics=off'))
277+
278+
def test_sparc_ss20(self):
279+
"""
280+
:avocado: tags=arch:sparc
281+
:avocado: tags=machine:SS-20
282+
"""
283+
tar_hash = 'b18550d5d61c7615d989a06edace051017726a9f'
284+
tar_url = ('https://www.qemu-advent-calendar.org'
285+
'/2018/download/day11.tar.xz')
286+
file_path = self.fetch_asset(tar_url, asset_hash=tar_hash)
287+
self.do_test_advcal_2018(file_path, 'zImage.elf')
288+
289+
def test_xtensa_lx60(self):
290+
"""
291+
:avocado: tags=arch:xtensa
292+
:avocado: tags=machine:lx60
293+
:avocado: tags=cpu:dc233c
294+
"""
295+
tar_hash = '49e88d9933742f0164b60839886c9739cb7a0d34'
296+
tar_url = ('https://www.qemu-advent-calendar.org'
297+
'/2018/download/day02.tar.xz')
298+
file_path = self.fetch_asset(tar_url, asset_hash=tar_hash)
299+
self.do_test_advcal_2018(file_path, 'santas-sleigh-ride.elf',
300+
args=('-cpu', 'dc233c'))

0 commit comments

Comments
 (0)