convert pactest to TAP output
Each test produces a single TAP result with the rules run in a sub-test. This reduces output when run under automake and makes it possible to continue setting expectfailure at the test level rather than per-rule. Signed-off-by: Andrew Gregory <andrew.gregory.8@gmail.com> Signed-off-by: Allan McRae <allan@archlinux.org>
This commit is contained in:
Andrew Gregory
Allan McRae
parent
429b956fb2
commit
1152052b3e
|
|
@ -26,6 +26,7 @@
|
|||
import tempfile
|
||||
|
||||
import pmenv
|
||||
import tap
|
||||
import util
|
||||
|
||||
__author__ = "Aurelien FORET"
|
||||
|
|
@ -110,7 +111,7 @@ def create_parser():
|
|||
env.pacman["ldconfig"] = opts.ldconfig
|
||||
|
||||
if opts.testcases is None or len(opts.testcases) == 0:
|
||||
print "no tests defined, nothing to do"
|
||||
tap.bail("no tests defined, nothing to do")
|
||||
os.rmdir(root_path)
|
||||
sys.exit(2)
|
||||
|
||||
|
|
@ -124,7 +125,7 @@ def create_parser():
|
|||
if not opts.keeproot:
|
||||
shutil.rmtree(root_path)
|
||||
else:
|
||||
print "pacman testing root saved: %s" % root_path
|
||||
tap.diag("pacman testing root saved: %s" % root_path)
|
||||
|
||||
if env.failed > 0:
|
||||
sys.exit(1)
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@
|
|||
import tarfile
|
||||
|
||||
import pmpkg
|
||||
import tap
|
||||
import util
|
||||
|
||||
def _getsection(fd):
|
||||
|
|
@ -105,7 +106,7 @@ def db_read(self, name):
|
|||
# desc
|
||||
filename = os.path.join(path, "desc")
|
||||
if not os.path.isfile(filename):
|
||||
print "invalid db entry found (desc missing) for pkg", pkgname
|
||||
tap.bail("invalid db entry found (desc missing) for pkg " + pkgname)
|
||||
return None
|
||||
fd = open(filename, "r")
|
||||
while 1:
|
||||
|
|
@ -160,7 +161,7 @@ def db_read(self, name):
|
|||
# files
|
||||
filename = os.path.join(path, "files")
|
||||
if not os.path.isfile(filename):
|
||||
print "invalid db entry found (files missing) for pkg", pkgname
|
||||
tap.bail("invalid db entry found (files missing) for pkg " + pkgname)
|
||||
return None
|
||||
fd = open(filename, "r")
|
||||
while 1:
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@
|
|||
import os
|
||||
|
||||
import pmtest
|
||||
import tap
|
||||
|
||||
|
||||
class pmenv(object):
|
||||
|
|
@ -58,26 +59,18 @@ def addtest(self, testcase):
|
|||
def run(self):
|
||||
"""
|
||||
"""
|
||||
|
||||
tap.plan(len(self.testcases))
|
||||
for t in self.testcases:
|
||||
print "=========="*8
|
||||
print "Running '%s'" % t.testname
|
||||
tap.diag("==========" * 8)
|
||||
tap.diag("Running '%s'" % t.testname)
|
||||
|
||||
t.load()
|
||||
print t.description
|
||||
print "----------"*8
|
||||
|
||||
t.generate(self.pacman)
|
||||
|
||||
t.run(self.pacman)
|
||||
|
||||
t.check()
|
||||
print "==> Test result"
|
||||
if t.result["fail"] == 0:
|
||||
print "\tPASS"
|
||||
else:
|
||||
print "\tFAIL"
|
||||
print
|
||||
tap.diag("==> Checking rules")
|
||||
tap.todo = t.expectfailure
|
||||
tap.subtest(lambda: t.check(), t.description)
|
||||
|
||||
def results(self):
|
||||
"""
|
||||
|
|
@ -109,40 +102,42 @@ def _printtest(t):
|
|||
result = "[PASS]"
|
||||
else:
|
||||
result = "[FAIL]"
|
||||
print result,
|
||||
print "%s Rules: OK = %2u FAIL = %2u" \
|
||||
% (t.testname.ljust(34), success, fail)
|
||||
tap.diag("%s %s Rules: OK = %2u FAIL = %2u" \
|
||||
% (result, t.testname.ljust(34), success, fail))
|
||||
if fail != 0:
|
||||
# print test description if test failed
|
||||
print " ", t.description
|
||||
tap.diag(" " + t.description)
|
||||
|
||||
print "=========="*8
|
||||
print "Results"
|
||||
print "----------"*8
|
||||
print " Passed:"
|
||||
tap.diag("==========" * 8)
|
||||
tap.diag("Results")
|
||||
tap.diag("----------" * 8)
|
||||
tap.diag(" Passed:")
|
||||
for test in tpassed:
|
||||
_printtest(test)
|
||||
print "----------"*8
|
||||
print " Expected Failures:"
|
||||
tap.diag("----------" * 8)
|
||||
tap.diag(" Expected Failures:")
|
||||
for test in texpectedfail:
|
||||
_printtest(test)
|
||||
print "----------"*8
|
||||
print " Unexpected Passes:"
|
||||
tap.diag("----------" * 8)
|
||||
tap.diag(" Unexpected Passes:")
|
||||
for test in tunexpectedpass:
|
||||
_printtest(test)
|
||||
print "----------"*8
|
||||
print " Failed:"
|
||||
tap.diag("----------" * 8)
|
||||
tap.diag(" Failed:")
|
||||
for test in tfailed:
|
||||
_printtest(test)
|
||||
print "----------"*8
|
||||
tap.diag("----------" * 8)
|
||||
|
||||
total = len(self.testcases)
|
||||
print "Total = %3u" % total
|
||||
tap.diag("Total = %3u" % total)
|
||||
if total:
|
||||
print "Pass = %3u (%6.2f%%)" % (self.passed, float(self.passed) * 100 / total)
|
||||
print "Expected Fail = %3u (%6.2f%%)" % (self.expectedfail, float(self.expectedfail) * 100 / total)
|
||||
print "Unexpected Pass = %3u (%6.2f%%)" % (self.unexpectedpass, float(self.unexpectedpass) * 100 / total)
|
||||
print "Fail = %3u (%6.2f%%)" % (self.failed, float(self.failed) * 100 / total)
|
||||
print ""
|
||||
tap.diag("Pass = %3u (%6.2f%%)" % (self.passed,
|
||||
float(self.passed) * 100 / total))
|
||||
tap.diag("Expected Fail = %3u (%6.2f%%)" % (self.expectedfail,
|
||||
float(self.expectedfail) * 100 / total))
|
||||
tap.diag("Unexpected Pass = %3u (%6.2f%%)" % (self.unexpectedpass,
|
||||
float(self.unexpectedpass) * 100 / total))
|
||||
tap.diag("Fail = %3u (%6.2f%%)" % (self.failed,
|
||||
float(self.failed) * 100 / total))
|
||||
|
||||
# vim: set ts=4 sw=4 et:
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@
|
|||
import os
|
||||
import stat
|
||||
|
||||
import tap
|
||||
import util
|
||||
|
||||
class pmrule(object):
|
||||
|
|
@ -57,12 +58,12 @@ def check(self, test):
|
|||
elif case == "OUTPUT":
|
||||
logfile = os.path.join(test.root, util.LOGFILE)
|
||||
if not os.access(logfile, os.F_OK):
|
||||
print "LOGFILE not found, cannot validate 'OUTPUT' rule"
|
||||
tap.diag("LOGFILE not found, cannot validate 'OUTPUT' rule")
|
||||
success = 0
|
||||
elif not util.grep(logfile, key):
|
||||
success = 0
|
||||
else:
|
||||
print "PACMAN rule '%s' not found" % case
|
||||
tap.diag("PACMAN rule '%s' not found" % case)
|
||||
success = -1
|
||||
elif kind == "PKG":
|
||||
localdb = test.db["local"]
|
||||
|
|
@ -108,7 +109,7 @@ def check(self, test):
|
|||
if not found:
|
||||
success = 0
|
||||
else:
|
||||
print "PKG rule '%s' not found" % case
|
||||
tap.diag("PKG rule '%s' not found" % case)
|
||||
success = -1
|
||||
elif kind == "FILE":
|
||||
filename = os.path.join(test.root, key)
|
||||
|
|
@ -148,7 +149,7 @@ def check(self, test):
|
|||
if not os.path.isfile("%s.pacsave" % filename):
|
||||
success = 0
|
||||
else:
|
||||
print "FILE rule '%s' not found" % case
|
||||
tap.diag("FILE rule '%s' not found" % case)
|
||||
success = -1
|
||||
elif kind == "DIR":
|
||||
filename = os.path.join(test.root, key)
|
||||
|
|
@ -156,7 +157,7 @@ def check(self, test):
|
|||
if not os.path.isdir(filename):
|
||||
success = 0
|
||||
else:
|
||||
print "DIR rule '%s' not found" % case
|
||||
tap.diag("DIR rule '%s' not found" % case)
|
||||
success = -1
|
||||
elif kind == "LINK":
|
||||
filename = os.path.join(test.root, key)
|
||||
|
|
@ -164,7 +165,7 @@ def check(self, test):
|
|||
if not os.path.islink(filename):
|
||||
success = 0
|
||||
else:
|
||||
print "LINK rule '%s' not found" % case
|
||||
tap.diag("LINK rule '%s' not found" % case)
|
||||
success = -1
|
||||
elif kind == "CACHE":
|
||||
cachedir = os.path.join(test.root, util.PM_CACHEDIR)
|
||||
|
|
@ -174,7 +175,7 @@ def check(self, test):
|
|||
os.path.join(cachedir, pkg.filename())):
|
||||
success = 0
|
||||
else:
|
||||
print "Rule kind '%s' not found" % kind
|
||||
tap.diag("Rule kind '%s' not found" % kind)
|
||||
success = -1
|
||||
|
||||
if self.false and success != -1:
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@
|
|||
import pmrule
|
||||
import pmdb
|
||||
import pmfile
|
||||
import tap
|
||||
import util
|
||||
from util import vprint
|
||||
|
||||
|
|
@ -104,7 +105,7 @@ def load(self):
|
|||
raise IOError("file %s does not exist!" % self.name)
|
||||
|
||||
def generate(self, pacman):
|
||||
print "==> Generating test environment"
|
||||
tap.diag("==> Generating test environment")
|
||||
|
||||
# Cleanup leftover files from a previous test session
|
||||
if os.path.isdir(self.root):
|
||||
|
|
@ -192,23 +193,23 @@ def generate(self, pacman):
|
|||
|
||||
def run(self, pacman):
|
||||
if os.path.isfile(util.PM_LOCK):
|
||||
print "\tERROR: another pacman session is on-going -- skipping"
|
||||
tap.bail("\tERROR: another pacman session is on-going -- skipping")
|
||||
return
|
||||
|
||||
print "==> Running test"
|
||||
tap.diag("==> Running test")
|
||||
vprint("\tpacman %s" % self.args)
|
||||
|
||||
cmd = []
|
||||
if os.geteuid() != 0:
|
||||
fakeroot = util.which("fakeroot")
|
||||
if not fakeroot:
|
||||
print "WARNING: fakeroot not found!"
|
||||
tap.diag("WARNING: fakeroot not found!")
|
||||
else:
|
||||
cmd.append("fakeroot")
|
||||
|
||||
fakechroot = util.which("fakechroot")
|
||||
if not fakechroot:
|
||||
print "WARNING: fakechroot not found!"
|
||||
tap.diag("WARNING: fakechroot not found!")
|
||||
else:
|
||||
cmd.append("fakechroot")
|
||||
|
||||
|
|
@ -252,23 +253,20 @@ def run(self, pacman):
|
|||
|
||||
# Check if the lock is still there
|
||||
if os.path.isfile(util.PM_LOCK):
|
||||
print "\tERROR: %s not removed" % util.PM_LOCK
|
||||
tap.diag("\tERROR: %s not removed" % util.PM_LOCK)
|
||||
os.unlink(util.PM_LOCK)
|
||||
# Look for a core file
|
||||
if os.path.isfile(os.path.join(self.root, util.TMPDIR, "core")):
|
||||
print "\tERROR: pacman dumped a core file"
|
||||
tap.diag("\tERROR: pacman dumped a core file")
|
||||
|
||||
def check(self):
|
||||
print "==> Checking rules"
|
||||
|
||||
tap.plan(len(self.rules))
|
||||
for i in self.rules:
|
||||
success = i.check(self)
|
||||
if success == 1:
|
||||
msg = " OK "
|
||||
self.result["success"] += 1
|
||||
else:
|
||||
msg = "FAIL"
|
||||
self.result["fail"] += 1
|
||||
print "\t[%s] %s" % (msg, i)
|
||||
tap.ok(success, i)
|
||||
|
||||
# vim: set ts=4 sw=4 et:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,64 @@
|
|||
# Copyright (c) 2013 Pacman Development Team <pacman-dev@archlinux.org>
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
todo = None
|
||||
count = 0
|
||||
level = 0
|
||||
failed = 0
|
||||
|
||||
def _output(msg):
|
||||
print("%s%s" % (" "*level, msg))
|
||||
|
||||
def ok(ok, description=""):
|
||||
global count, failed
|
||||
count += 1
|
||||
if not ok:
|
||||
failed += 1
|
||||
directive = " # TODO" if todo else ""
|
||||
_output("%s %d - %s%s" % ("ok" if ok else "not ok", count,
|
||||
description, directive))
|
||||
|
||||
def plan(count):
|
||||
_output("1..%d" % (count))
|
||||
|
||||
def diag(msg):
|
||||
_output("# %s" % (msg))
|
||||
|
||||
def bail(reason=""):
|
||||
_output("Bail out! %s" % (reason))
|
||||
|
||||
def subtest(func, description=""):
|
||||
global todo, count, level, failed
|
||||
|
||||
save_todo = todo
|
||||
save_count = count
|
||||
save_level = level
|
||||
save_failed = failed
|
||||
|
||||
todo = None
|
||||
count = 0
|
||||
level += 1
|
||||
failed = 0
|
||||
|
||||
func()
|
||||
|
||||
subtest_ok = not failed
|
||||
|
||||
todo = save_todo
|
||||
count = save_count
|
||||
level = save_level
|
||||
failed = save_failed
|
||||
|
||||
ok(subtest_ok, description)
|
||||
|
|
@ -21,6 +21,8 @@
|
|||
import re
|
||||
import hashlib
|
||||
|
||||
import tap
|
||||
|
||||
SELFPATH = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
# ALPM
|
||||
|
|
@ -43,7 +45,7 @@
|
|||
|
||||
def vprint(msg):
|
||||
if verbose:
|
||||
print msg
|
||||
tap.diag(msg)
|
||||
|
||||
#
|
||||
# Methods to generate files
|
||||
|
|
|
|||
Loading…
Reference in New Issue