diff mbox series

[pacman-dev] meson: handle XFAIL tests outside of TAP

Message ID 20200502161232.390493-1-dreisner@archlinux.org
State Accepted, archived
Headers show
Series [pacman-dev] meson: handle XFAIL tests outside of TAP | expand

Commit Message

Dave Reisner May 2, 2020, 4:12 p.m. UTC 
This change causes expected fail tests to actually fail by eliding the
'# TODO' from the test plan. In turn, we can now properly use
'should_fail' in the meson test() rule and see these expected fail
tests in the output:

Before:
  ...
  320/332 upgrade077.py                   OK 0.12679290771484375 s
  321/332 upgrade078.py                   OK 0.12620115280151367 s
  322/332 upgrade080.py                   OK 0.1252129077911377 s
  ...

  Ok:                 332
  Expected Fail:      0
  Fail:               0
  Unexpected Pass:    0
  Skipped:            0
  Timeout:            0

After:
  ...
  320/332 upgrade077.py                   OK 0.12679290771484375 s
  321/332 upgrade078.py                   EXPECTEDFAIL0.12620115280151367 s
  322/332 upgrade080.py                   OK 0.1252129077911377 s
  ...

  Ok:                 326
  Expected Fail:      6
  Fail:               0
  Unexpected Pass:    0
  Skipped:            0
  Timeout:            0
---
 test/pacman/meson.build | 15 ++++++++++++++-
 test/pacman/pmenv.py    |  7 ++++++-
 2 files changed, 20 insertions(+), 2 deletions(-)
diff mbox series

Patch

diff --git a/test/pacman/meson.build b/test/pacman/meson.build
index 9de4f5a1..fd0dbf90 100644
--- a/test/pacman/meson.build
+++ b/test/pacman/meson.build
@@ -330,6 +330,17 @@  pacman_tests = [
   'tests/xfercommand001.py',
 ]
 
+xfail_tests = {
+  'tests/deprange001.py': true,
+  # expect failure on 32 bit machines
+  'tests/query006.py': cc.sizeof('ssize_t') < 8,
+  'tests/replace110.py': true,
+  'tests/sync-update-package-removing-required-provides.py': true,
+  'tests/sync403.py': true,
+  'tests/sync406.py': true,
+  'tests/upgrade078.py': true,
+}
+
 foreach input : pacman_tests
   test_name = input.split('/')[1]
   args = [
@@ -351,7 +362,9 @@  foreach input : pacman_tests
     test_name,
     PYTHON,
     protocol : 'tap',
+    env : ['RUNNING_UNDER_MESON=1'],
     args : args,
-    timeout: 120,
+    timeout : 120,
+    should_fail : xfail_tests.get(input, false),
     depends : [pacman_bin])
 endforeach
diff --git a/test/pacman/pmenv.py b/test/pacman/pmenv.py
index 24437b61..d87193f2 100644
--- a/test/pacman/pmenv.py
+++ b/test/pacman/pmenv.py
@@ -70,5 +70,10 @@  def run(self):
                 t.run(self.pacman)
 
                 tap.diag("==> Checking rules")
-                tap.todo = t.expectfailure
+                # When running under meson, we don't emit 'todo' in the plan and instead
+                # handle expected failures in the test() objects. This really should be
+                # fixed in meson:
+                # https://github.com/mesonbuild/meson/issues/2923#issuecomment-614647076
+                tap.todo = (t.expectfailure and
+                        not 'RUNNING_UNDER_MESON' in os.environ)
                 tap.subtest(lambda: t.check(), t.description)