diff options
author | dan sinclair <dj2@everburning.com> | 2019-03-17 19:06:11 -0400 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-03-17 19:06:11 -0400 |
commit | 918d83e6ed250c3e6379e02c3d0e328bb11ec195 (patch) | |
tree | 3797c2532247fd969f8169778095202be6ae3801 /tests | |
parent | e822b18aacc7680dc0454c9614ae3d6348fd5125 (diff) | |
download | amber-918d83e6ed250c3e6379e02c3d0e328bb11ec195.tar.gz |
[testing] Add suppressions to test runner (#373)
This CL adds a list of suppressed test cases to the test runner. This
allows suppressing failures on a per platform basis. This is needed for
MoltenVK where tessellation and geometry shaders are not supported yet.
The output looks like:
Summary of Failures:
./tests/cases/compute_accumulated_ubo_definition.vkscript
./tests/cases/compute_mat2x2.vkscript
./tests/cases/compute_mat3x2.vkscript
./tests/cases/compute_ubo_and_ssbo.vkscript
./tests/cases/draw_triangle_list_with_depth.vkscript
./tests/cases/multiple_ssbo_update_with_graphics_pipeline.vkscript
./tests/cases/multiple_ubo_update_with_graphics_pipeline.vkscript
Summary of Suppressions:
./tests/cases/draw_triangle_list_using_geom_shader.vkscript
./tests/cases/draw_triangle_list_using_tessellation.vkscript
Test cases executed: 58
Successes: 49
Failures: 7
Suppressed: 2
Diffstat (limited to 'tests')
-rwxr-xr-x | tests/run_tests.py | 46 |
1 files changed, 37 insertions, 9 deletions
diff --git a/tests/run_tests.py b/tests/run_tests.py index cf536cf..fd3b3e4 100755 --- a/tests/run_tests.py +++ b/tests/run_tests.py @@ -17,11 +17,21 @@ import base64 import difflib import optparse import os +import platform import re import subprocess import sys import tempfile +SUPPRESSIONS = { + "Darwin": [ + # No geometry shader on MoltenVK + "draw_triangle_list_using_geom_shader.vkscript", + # No tessellation shader on MoltenVK + "draw_triangle_list_using_tessellation.vkscript" + ] +} + class TestCase: def __init__(self, input_path, parse_only): self.input_path = input_path @@ -33,6 +43,12 @@ class TestCase: fail_re = re.compile('^.+[.]expect_fail[.][amber|vkscript]') return fail_re.match(self.GetInputPath()) + def IsSuppressed(self): + system = platform.system() + if system in SUPPRESSIONS.keys(): + return os.path.basename(self.input_path) in SUPPRESSIONS[system] + return False + def IsParseOnly(self): return self.parse_only @@ -54,13 +70,13 @@ class TestRunner: try: err = subprocess.check_output(cmd, stderr=subprocess.STDOUT) - if err != "" and not tc.IsExpectedFail(): + if err != "" and not tc.IsExpectedFail() and not tc.IsSuppressed(): sys.stdout.write(err) return False except Exception as e: print e.output - if not tc.IsExpectedFail(): + if not tc.IsExpectedFail() and not tc.IsSuppressed(): print e return False @@ -71,11 +87,14 @@ class TestRunner: for tc in self.test_cases: result = self.RunTest(tc) - if not tc.IsExpectedFail() and not result: - self.failures.append(tc.GetInputPath()) - elif tc.IsExpectedFail() and result: - print("Expected: " + tc.GetInputPath() + " to fail but passed.") - self.failures.append(tc.GetInputPath()) + if tc.IsSuppressed(): + self.suppressed.append(tc.GetInputPath()) + else: + if not tc.IsExpectedFail() and not result: + self.failures.append(tc.GetInputPath()) + elif tc.IsExpectedFail() and result: + print("Expected: " + tc.GetInputPath() + " to fail but passed.") + self.failures.append(tc.GetInputPath()) def SummarizeResults(self): if len(self.failures) > 0: @@ -85,10 +104,18 @@ class TestRunner: for failure in self.failures: print failure + if len(self.suppressed) > 0: + self.suppressed.sort() + + print '\nSummary of Suppressions:' + for suppression in self.suppressed: + print suppression + print print 'Test cases executed: %d' % len(self.test_cases) - print ' Successes: %d' % (len(self.test_cases) - len(self.failures)) - print ' Failures: %d' % len(self.failures) + print ' Successes: %d' % (len(self.test_cases) - len(self.suppressed) - len(self.failures)) + print ' Failures: %d' % len(self.failures) + print ' Suppressed: %d' % len(self.suppressed) print @@ -145,6 +172,7 @@ class TestRunner: TestCase(input_path, self.options.parse_only)) self.failures = [] + self.suppressed = [] self.RunTests() self.SummarizeResults() |