From 918d83e6ed250c3e6379e02c3d0e328bb11ec195 Mon Sep 17 00:00:00 2001
From: dan sinclair <dj2@everburning.com>
Date: Sun, 17 Mar 2019 19:06:11 -0400
Subject: [testing] Add suppressions to test runner (#373)

This CL adds a list of suppressed test cases to the test runner. This
allows suppressing failures on a per platform basis. This is needed for
MoltenVK where tessellation and geometry shaders are not supported yet.
The output looks like:

  Summary of Failures:
  ./tests/cases/compute_accumulated_ubo_definition.vkscript
  ./tests/cases/compute_mat2x2.vkscript
  ./tests/cases/compute_mat3x2.vkscript
  ./tests/cases/compute_ubo_and_ssbo.vkscript
  ./tests/cases/draw_triangle_list_with_depth.vkscript
  ./tests/cases/multiple_ssbo_update_with_graphics_pipeline.vkscript
  ./tests/cases/multiple_ubo_update_with_graphics_pipeline.vkscript

  Summary of Suppressions:
  ./tests/cases/draw_triangle_list_using_geom_shader.vkscript
  ./tests/cases/draw_triangle_list_using_tessellation.vkscript

  Test cases executed: 58
    Successes:  49
    Failures:   7
    Suppressed: 2
---
 tests/run_tests.py | 46 +++++++++++++++++++++++++++++++++++++---------
 1 file changed, 37 insertions(+), 9 deletions(-)

(limited to 'tests')

diff --git a/tests/run_tests.py b/tests/run_tests.py
index cf536cf..fd3b3e4 100755
--- a/tests/run_tests.py
+++ b/tests/run_tests.py
@@ -17,11 +17,21 @@ import base64
 import difflib
 import optparse
 import os
+import platform
 import re
 import subprocess
 import sys
 import tempfile
 
+SUPPRESSIONS = {
+  "Darwin": [
+    # No geometry shader on MoltenVK
+    "draw_triangle_list_using_geom_shader.vkscript",
+    # No tessellation shader on MoltenVK
+    "draw_triangle_list_using_tessellation.vkscript"
+  ]
+}
+
 class TestCase:
   def __init__(self, input_path, parse_only):
     self.input_path = input_path
@@ -33,6 +43,12 @@ class TestCase:
     fail_re = re.compile('^.+[.]expect_fail[.][amber|vkscript]')
     return fail_re.match(self.GetInputPath())
 
+  def IsSuppressed(self):
+    system = platform.system()
+    if system in SUPPRESSIONS.keys():
+      return os.path.basename(self.input_path) in SUPPRESSIONS[system]
+    return False
+
   def IsParseOnly(self):
     return self.parse_only
 
@@ -54,13 +70,13 @@ class TestRunner:
 
     try:
       err = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
-      if err != "" and not tc.IsExpectedFail():
+      if err != "" and not tc.IsExpectedFail() and not tc.IsSuppressed():
         sys.stdout.write(err)
         return False
 
     except Exception as e:
       print e.output
-      if not tc.IsExpectedFail():
+      if not tc.IsExpectedFail() and not tc.IsSuppressed():
         print e
       return False
 
@@ -71,11 +87,14 @@ class TestRunner:
     for tc in self.test_cases:
       result = self.RunTest(tc)
 
-      if not tc.IsExpectedFail() and not result:
-        self.failures.append(tc.GetInputPath())
-      elif tc.IsExpectedFail() and result:
-        print("Expected: " + tc.GetInputPath() + " to fail but passed.")
-        self.failures.append(tc.GetInputPath())
+      if tc.IsSuppressed():
+        self.suppressed.append(tc.GetInputPath())
+      else:
+        if not tc.IsExpectedFail() and not result:
+          self.failures.append(tc.GetInputPath())
+        elif tc.IsExpectedFail() and result:
+          print("Expected: " + tc.GetInputPath() + " to fail but passed.")
+          self.failures.append(tc.GetInputPath())
 
   def SummarizeResults(self):
     if len(self.failures) > 0:
@@ -85,10 +104,18 @@ class TestRunner:
       for failure in self.failures:
         print failure
 
+    if len(self.suppressed) > 0:
+      self.suppressed.sort()
+
+      print '\nSummary of Suppressions:'
+      for suppression in self.suppressed:
+        print suppression
+
     print
     print 'Test cases executed: %d' % len(self.test_cases)
-    print '  Successes: %d' % (len(self.test_cases) - len(self.failures))
-    print '  Failures:  %d' % len(self.failures)
+    print '  Successes:  %d' % (len(self.test_cases) - len(self.suppressed) - len(self.failures))
+    print '  Failures:   %d' % len(self.failures)
+    print '  Suppressed: %d' % len(self.suppressed)
     print
 
 
@@ -145,6 +172,7 @@ class TestRunner:
                   TestCase(input_path, self.options.parse_only))
 
     self.failures = []
+    self.suppressed = []
 
     self.RunTests()
     self.SummarizeResults()
-- 
cgit v1.2.3