mirror of
https://github.com/Motorhead1991/qemu.git
synced 2025-07-27 04:13:53 -06:00
build: use "meson test" as the test harness
"meson test" starting with version 0.57 is just as capable and easy to use as QEMU's own TAP driver. All existing options for "make check" work. The only required code change involves how to mark "slow" tests; they need to belong to an additional "slow" suite. The rules for .tap output are replaced by JUnit XML; GitLab is able to parse that output and present it in the CI pipeline report. Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
This commit is contained in:
Paolo Bonzini
parent
2bf40d0841
commit
3d2f73ef75
6 changed files with 51 additions and 561 deletions
|
|
@ -13,101 +13,79 @@ import sys
|
|||
|
||||
class Suite(object):
|
||||
def __init__(self):
|
||||
self.tests = list()
|
||||
self.slow_tests = list()
|
||||
self.executables = set()
|
||||
self.deps = set()
|
||||
self.speeds = ['quick']
|
||||
|
||||
def names(self, base):
|
||||
return [base if speed == 'quick' else f'{base}-{speed}' for speed in self.speeds]
|
||||
|
||||
|
||||
print('''
|
||||
SPEED = quick
|
||||
|
||||
# $1 = environment, $2 = test command, $3 = test name, $4 = dir
|
||||
.test-human-tap = $1 $(if $4,(cd $4 && $2),$2) -m $(SPEED) < /dev/null | ./scripts/tap-driver.pl --test-name="$3" $(if $(V),,--show-failures-only)
|
||||
.test-human-exitcode = $1 $(PYTHON) scripts/test-driver.py $(if $4,-C$4) $(if $(V),--verbose) -- $2 < /dev/null
|
||||
.test-tap-tap = $1 $(if $4,(cd $4 && $2),$2) < /dev/null | sed "s/^[a-z][a-z]* [0-9]*/& $3/" || true
|
||||
.test-tap-exitcode = printf "%s\\n" 1..1 "`$1 $(if $4,(cd $4 && $2),$2) < /dev/null > /dev/null || echo "not "`ok 1 $3"
|
||||
.test.human-print = echo $(if $(V),'$1 $2','Running test $3') &&
|
||||
.test.env = MALLOC_PERTURB_=$${MALLOC_PERTURB_:-$$(( $${RANDOM:-0} % 255 + 1))}
|
||||
.speed.quick = $(foreach s,$(sort $(filter-out %-slow, $1)), --suite $s)
|
||||
.speed.slow = $(foreach s,$(sort $1), --suite $s)
|
||||
|
||||
# $1 = test name, $2 = test target (human or tap)
|
||||
.test.run = $(call .test.$2-print,$(.test.env.$1),$(.test.cmd.$1),$(.test.name.$1)) $(call .test-$2-$(.test.driver.$1),$(.test.env.$1),$(.test.cmd.$1),$(.test.name.$1),$(.test.dir.$1))
|
||||
.mtestargs = --no-rebuild -t 0
|
||||
ifneq ($(SPEED), quick)
|
||||
.mtestargs += --setup $(SPEED)
|
||||
endif
|
||||
.mtestargs += $(subst -j,--num-processes , $(filter-out -j, $(lastword -j1 $(filter -j%, $(MAKEFLAGS)))))
|
||||
|
||||
.test.output-format = human
|
||||
''')
|
||||
.check.mtestargs = $(MTESTARGS) $(.mtestargs) $(if $(V),--verbose,--print-errorlogs)
|
||||
.bench.mtestargs = $(MTESTARGS) $(.mtestargs) --benchmark --verbose''')
|
||||
|
||||
introspect = json.load(sys.stdin)
|
||||
i = 0
|
||||
|
||||
def process_tests(test, targets, suites):
|
||||
global i
|
||||
env = ' '.join(('%s=%s' % (shlex.quote(k), shlex.quote(v))
|
||||
for k, v in test['env'].items()))
|
||||
executable = test['cmd'][0]
|
||||
try:
|
||||
executable = os.path.relpath(executable)
|
||||
except:
|
||||
pass
|
||||
if test['workdir'] is not None:
|
||||
try:
|
||||
test['cmd'][0] = os.path.relpath(executable, test['workdir'])
|
||||
except:
|
||||
test['cmd'][0] = executable
|
||||
else:
|
||||
test['cmd'][0] = executable
|
||||
cmd = ' '.join((shlex.quote(x) for x in test['cmd']))
|
||||
driver = test['protocol'] if 'protocol' in test else 'exitcode'
|
||||
|
||||
i += 1
|
||||
if test['workdir'] is not None:
|
||||
print('.test.dir.%d := %s' % (i, shlex.quote(test['workdir'])))
|
||||
|
||||
deps = (targets.get(x, []) for x in test['depends'])
|
||||
deps = itertools.chain.from_iterable(deps)
|
||||
|
||||
print('.test.name.%d := %s' % (i, test['name']))
|
||||
print('.test.driver.%d := %s' % (i, driver))
|
||||
print('.test.env.%d := $(.test.env) %s' % (i, env))
|
||||
print('.test.cmd.%d := %s' % (i, cmd))
|
||||
print('.test.deps.%d := %s' % (i, ' '.join(deps)))
|
||||
print('.PHONY: run-test-%d' % (i,))
|
||||
print('run-test-%d: $(.test.deps.%d)' % (i,i))
|
||||
print('\t@$(call .test.run,%d,$(.test.output-format))' % (i,))
|
||||
deps = list(deps)
|
||||
|
||||
test_suites = test['suite'] or ['default']
|
||||
is_slow = any(s.endswith('-slow') for s in test_suites)
|
||||
for s in test_suites:
|
||||
# The suite name in the introspection info is "PROJECT:SUITE"
|
||||
s = s.split(':')[1]
|
||||
if s == 'slow':
|
||||
continue
|
||||
if s.endswith('-slow'):
|
||||
s = s[:-5]
|
||||
if is_slow:
|
||||
suites[s].slow_tests.append(i)
|
||||
else:
|
||||
suites[s].tests.append(i)
|
||||
suites[s].executables.add(executable)
|
||||
suites[s].speeds.append('slow')
|
||||
suites[s].deps.update(deps)
|
||||
|
||||
def emit_prolog(suites, prefix):
|
||||
all_tap = ' '.join(('%s-report-%s.tap' % (prefix, k) for k in suites.keys()))
|
||||
print('.PHONY: %s %s-report.tap %s' % (prefix, prefix, all_tap))
|
||||
print('%s: run-tests' % (prefix,))
|
||||
print('%s-report.tap %s: %s-report%%.tap: all' % (prefix, all_tap, prefix))
|
||||
print('''\t$(MAKE) .test.output-format=tap --quiet -Otarget V=1 %s$* | ./scripts/tap-merge.pl | tee "$@" \\
|
||||
| ./scripts/tap-driver.pl $(if $(V),, --show-failures-only)''' % (prefix, ))
|
||||
all_targets = ' '.join((f'{prefix}-{k}' for k in suites.keys()))
|
||||
all_xml = ' '.join((f'{prefix}-report-{k}.junit.xml' for k in suites.keys()))
|
||||
print()
|
||||
print(f'all-{prefix}-targets = {all_targets}')
|
||||
print(f'all-{prefix}-xml = {all_xml}')
|
||||
print(f'.PHONY: {prefix} do-meson-{prefix} {prefix}-report.junit.xml $(all-{prefix}-targets) $(all-{prefix}-xml)')
|
||||
print(f'ifeq ($(filter {prefix}, $(MAKECMDGOALS)),)')
|
||||
print(f'.{prefix}.mtestargs += $(call .speed.$(SPEED), $(.{prefix}.mtest-suites))')
|
||||
print(f'endif')
|
||||
print(f'{prefix}-build: run-ninja')
|
||||
print(f'{prefix} $(all-{prefix}-targets): do-meson-{prefix}')
|
||||
print(f'do-meson-{prefix}: run-ninja; $(if $(MAKE.n),,+)$(MESON) test $(.{prefix}.mtestargs)')
|
||||
print(f'{prefix}-report.junit.xml $(all-{prefix}-xml): {prefix}-report%.junit.xml: run-ninja')
|
||||
print(f'\t$(MAKE) {prefix}$* MTESTARGS="$(MTESTARGS) --logbase {prefix}-report$*" && ln -f meson-logs/$@ .')
|
||||
|
||||
def emit_suite(name, suite, prefix):
|
||||
executables = ' '.join(suite.executables)
|
||||
slow_test_numbers = ' '.join((str(x) for x in suite.slow_tests))
|
||||
test_numbers = ' '.join((str(x) for x in suite.tests))
|
||||
target = '%s-%s' % (prefix, name)
|
||||
print('.test.quick.%s := %s' % (target, test_numbers))
|
||||
print('.test.slow.%s := $(.test.quick.%s) %s' % (target, target, slow_test_numbers))
|
||||
print('%s-build: %s' % (prefix, executables))
|
||||
print('.PHONY: %s' % (target, ))
|
||||
print('.PHONY: %s-report-%s.tap' % (prefix, name))
|
||||
print('%s: run-tests' % (target, ))
|
||||
print('ifneq ($(filter %s %s, $(MAKECMDGOALS)),)' % (target, prefix))
|
||||
print('.tests += $(.test.$(SPEED).%s)' % (target, ))
|
||||
print('endif')
|
||||
print('all-%s-targets += %s' % (prefix, target))
|
||||
deps = ' '.join(suite.deps)
|
||||
targets = f'{prefix}-{name} {prefix}-report-{name}.junit.xml {prefix} {prefix}-report.junit.xml'
|
||||
print()
|
||||
print(f'.{prefix}-{name}.deps = {deps}')
|
||||
print(f'ifneq ($(filter {prefix}-build {targets}, $(MAKECMDGOALS)),)')
|
||||
print(f'.{prefix}.build-suites += {name}')
|
||||
print(f'endif')
|
||||
print(f'ifneq ($(filter {targets}, $(MAKECMDGOALS)),)')
|
||||
print(f'.{prefix}.mtest-suites += ' + ' '.join(suite.names(name)))
|
||||
print(f'endif')
|
||||
|
||||
targets = {t['id']: [os.path.relpath(f) for f in t['filename']]
|
||||
for t in introspect['targets']}
|
||||
|
|
@ -125,5 +103,3 @@ for test in introspect['benchmarks']:
|
|||
emit_prolog(benchsuites, 'bench')
|
||||
for name, suite in benchsuites.items():
|
||||
emit_suite(name, suite, 'bench')
|
||||
|
||||
print('run-tests: $(patsubst %, run-test-%, $(.tests))')
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue