pactest: add test.expectfailure option for tests

This will enable us to mark tests we know currently fail to differentiate
them from those that we know should pass. Regressions should be easier to
spot this way.

Signed-off-by: Dan McGee <dan@archlinux.org>

pactest/pmenv.py

@@ -86,13 +86,25 @@ class pmenv:
         """
         passed = 0
         tpassed = []
+        failed = 0
         tfailed = []
+        expectedfail = 0
+        texpectedfail = []
+        unexpectedpass = 0
+        tunexpectedpass = []
         for test in self.testcases:
             fail = test.result["fail"]
-            if fail == 0:
+            if fail == 0 and not test.expectfailure:
                 passed += 1
                 tpassed.append(test)
+            elif fail != 0 and test.expectfailure:
+                expectedfail += 1
+                texpectedfail.append(test)
+            elif fail == 0: # and not test.expectfail
+                unexpectedpass += 1
+                tunexpectedpass.append(test)
             else:
+                failed += 1
                 tfailed.append(test)
 
         def _printtest(t):
@@ -114,17 +126,26 @@ class pmenv:
         print "=========="*8
         print "Results"
         print "----------"*8
+        print " Passed:"
         for test in tpassed: _printtest(test)
         print "----------"*8
+        print " Expected Failures:"
+        for test in texpectedfail: _printtest(test)
+        print "----------"*8
+        print " Unexpected Passes:"
+        for test in tunexpectedpass: _printtest(test)
+        print "----------"*8
+        print " Failed:"
         for test in tfailed: _printtest(test)
         print "----------"*8
 
         total = len(self.testcases)
-        failed = total - passed
-        print "TOTAL = %3u" % total
+        print "Total            = %3u" % total
         if total:
-            print "PASS  = %3u (%6.2f%%)" % (passed, float(passed) * 100 / total)
-            print "FAIL  = %3u (%6.2f%%)" % (failed, float(failed) * 100 / total)
+            print "Pass             = %3u (%6.2f%%)" % (passed, float(passed) * 100 / total)
+            print "Expected Fail    = %3u (%6.2f%%)" % (expectedfail, float(expectedfail) * 100 / total)
+            print "Unexpected Pass  = %3u (%6.2f%%)" % (unexpectedpass, float(unexpectedpass) * 100 / total)
+            print "Fail             = %3u (%6.2f%%)" % (failed, float(failed) * 100 / total)
         print ""
 
 if __name__ == "__main__":

pactest/pmtest.py

@@ -90,6 +90,7 @@ class pmtest:
         # Test rules
         self.rules = []
         self.files = []
+        self.expectfailure = False
         
         if os.path.isfile(self.name):
             execfile(self.name)

pactest/tests/fileconflict001.py

@@ -18,3 +18,5 @@ self.args = "-U %s" % " ".join([p.filename() for p in p1, p2])
 self.addrule("PACMAN_RETCODE=1")
 self.addrule("!PKG_EXIST=pkg1")
 self.addrule("!PKG_EXIST=pkg2")
+
+self.expectfailure = True

pactest/tests/fileconflict002.py

@@ -14,3 +14,5 @@ self.args = "-U %s" % " ".join([p.filename() for p in p1, p2])
 self.addrule("PACMAN_RETCODE=1")
 self.addrule("!PKG_EXIST=pkg1")
 self.addrule("!PKG_EXIST=pkg2")
+
+self.expectfailure = True

pactest/tests/fileconflict004.py

@@ -17,3 +17,5 @@ self.addrule("PACMAN_RETCODE=0")
 self.addrule("PKG_EXIST=pkg1")
 self.addrule("PKG_VERSION=pkg1|2.0-1")
 self.addrule("FILE_TYPE=test|link")
+
+self.expectfailure = True

pactest/tests/sync403.py

@@ -19,3 +19,5 @@ self.addrule("PACMAN_RETCODE=0")
 self.addrule("PKG_EXIST=pkg1")
 self.addrule("!PKG_EXIST=pkg2")
 self.addrule("PKG_EXIST=pkg3")
+
+self.expectfailure = True

pactest/tests/trans001.py

@@ -18,3 +18,5 @@ self.addrule("PACMAN_RETCODE=1")
 self.addrule("!PKG_EXIST=pkg1")
 self.addrule("PKG_EXIST=pkg2")
 self.addrule("PKG_EXIST=pkg3")
+
+self.expectfailure = True

pactest/tests/upgrade046.py

@@ -29,3 +29,5 @@ self.addrule("FILE_MODIFIED=bin/dummy")
 self.addrule("FILE_MODIFIED=bin/foobar")
 self.addrule("FILE_EXIST=usr/share/file")
 self.addrule("FILE_MODIFIED=usr/share/file")
+
+self.expectfailure = True

pactest/tests/upgrade051.py

@@ -13,3 +13,5 @@ self.args = "-U %s" % p.filename()
 self.addrule("PACMAN_RETCODE=0")
 self.addrule("!PKG_EXIST=pkg1")
 self.addrule("PKG_EXIST=pkg2")
+
+self.expectfailure = True