diff options
Diffstat (limited to 'googletest/test/googletest-filter-unittest.py')
| -rwxr-xr-x | googletest/test/googletest-filter-unittest.py | 439 |
1 files changed, 273 insertions, 166 deletions
diff --git a/googletest/test/googletest-filter-unittest.py b/googletest/test/googletest-filter-unittest.py index 6b32f2d2..f1f3c7a5 100755 --- a/googletest/test/googletest-filter-unittest.py +++ b/googletest/test/googletest-filter-unittest.py @@ -42,12 +42,13 @@ we test that here also. import os import re + try: from sets import Set as set # For Python 2.3 compatibility except ImportError: pass import sys -import gtest_test_utils +from googletest.test import gtest_test_utils # Constants. @@ -60,7 +61,8 @@ CAN_PASS_EMPTY_ENV = False if sys.executable: os.environ['EMPTY_VAR'] = '' child = gtest_test_utils.Subprocess( - [sys.executable, '-c', 'import os; print(\'EMPTY_VAR\' in os.environ)']) + [sys.executable, '-c', "import os; print('EMPTY_VAR' in os.environ)"] + ) CAN_PASS_EMPTY_ENV = eval(child.output) @@ -75,8 +77,8 @@ if sys.executable: os.environ['UNSET_VAR'] = 'X' del os.environ['UNSET_VAR'] child = gtest_test_utils.Subprocess( - [sys.executable, '-c', 'import os; print(\'UNSET_VAR\' not in os.environ)' - ]) + [sys.executable, '-c', "import os; print('UNSET_VAR' not in os.environ)"] + ) CAN_UNSET_ENV = eval(child.output) @@ -84,7 +86,7 @@ if sys.executable: # make sense on platforms that cannot pass empty env variables (Win32) # and on platforms that cannot unset variables (since we cannot tell # the difference between "" and NULL -- Borland and Solaris < 5.10) -CAN_TEST_EMPTY_FILTER = (CAN_PASS_EMPTY_ENV and CAN_UNSET_ENV) +CAN_TEST_EMPTY_FILTER = CAN_PASS_EMPTY_ENV and CAN_UNSET_ENV # The environment variable for specifying the test filters. @@ -113,13 +115,18 @@ TEST_CASE_REGEX = re.compile(r'^\[\-+\] \d+ tests? from (\w+(/\w+)?)') # Regex for parsing test names from Google Test's output. TEST_REGEX = re.compile(r'^\[\s*RUN\s*\].*\.(\w+(/\w+)?)') +# Regex for parsing disabled banner from Google Test's output +DISABLED_BANNER_REGEX = re.compile(r'^\[\s*DISABLED\s*\] (.*)') + # The command line flag to tell Google Test to output the list of tests it # will run. LIST_TESTS_FLAG = '--gtest_list_tests' # Indicates whether Google Test supports death tests. -SUPPORTS_DEATH_TESTS = 'HasDeathTest' in gtest_test_utils.Subprocess( - [COMMAND, LIST_TESTS_FLAG]).output +SUPPORTS_DEATH_TESTS = ( + 'HasDeathTest' + in gtest_test_utils.Subprocess([COMMAND, LIST_TESTS_FLAG]).output +) # Full names of all tests in googletest-filter-unittests_. PARAM_TESTS = [ @@ -131,7 +138,7 @@ PARAM_TESTS = [ 'SeqQ/ParamTest.TestX/1', 'SeqQ/ParamTest.TestY/0', 'SeqQ/ParamTest.TestY/1', - ] +] DISABLED_TESTS = [ 'BarTest.DISABLED_TestFour', @@ -140,29 +147,31 @@ DISABLED_TESTS = [ 'DISABLED_FoobarTest.Test1', 'DISABLED_FoobarTest.DISABLED_Test2', 'DISABLED_FoobarbazTest.TestA', - ] +] if SUPPORTS_DEATH_TESTS: DEATH_TESTS = [ - 'HasDeathTest.Test1', - 'HasDeathTest.Test2', - ] + 'HasDeathTest.Test1', + 'HasDeathTest.Test2', + ] else: DEATH_TESTS = [] # All the non-disabled tests. -ACTIVE_TESTS = [ - 'FooTest.Abc', - 'FooTest.Xyz', - - 'BarTest.TestOne', - 'BarTest.TestTwo', - 'BarTest.TestThree', - - 'BazTest.TestOne', - 'BazTest.TestA', - 'BazTest.TestB', - ] + DEATH_TESTS + PARAM_TESTS +ACTIVE_TESTS = ( + [ + 'FooTest.Abc', + 'FooTest.Xyz', + 'BarTest.TestOne', + 'BarTest.TestTwo', + 'BarTest.TestThree', + 'BazTest.TestOne', + 'BazTest.TestA', + 'BazTest.TestB', + ] + + DEATH_TESTS + + PARAM_TESTS +) param_tests_present = None @@ -180,14 +189,15 @@ def SetEnvVar(env_var, value): del environ[env_var] -def RunAndReturnOutput(args = None): +def RunAndReturnOutput(args=None): """Runs the test program and returns its output.""" - return gtest_test_utils.Subprocess([COMMAND] + (args or []), - env=environ).output + return gtest_test_utils.Subprocess( + [COMMAND] + (args or []), env=environ + ).output -def RunAndExtractTestList(args = None): +def RunAndExtractTestList(args=None): """Runs the test program and returns its exit code and a list of tests run.""" p = gtest_test_utils.Subprocess([COMMAND] + (args or []), env=environ) @@ -206,6 +216,17 @@ def RunAndExtractTestList(args = None): return (tests_run, p.exit_code) +def RunAndExtractDisabledBannerList(args=None): + """Runs the test program and returns tests that printed a disabled banner.""" + p = gtest_test_utils.Subprocess([COMMAND] + (args or []), env=environ) + banners_printed = [] + for line in p.output.split('\n'): + match = DISABLED_BANNER_REGEX.match(line) + if match is not None: + banners_printed.append(match.group(1)) + return banners_printed + + def InvokeWithModifiedEnv(extra_env, function, *args, **kwargs): """Runs the given function and arguments in a modified environment.""" try: @@ -220,10 +241,13 @@ def InvokeWithModifiedEnv(extra_env, function, *args, **kwargs): def RunWithSharding(total_shards, shard_index, command): """Runs a test program shard and returns exit code and a list of tests run.""" - extra_env = {SHARD_INDEX_ENV_VAR: str(shard_index), - TOTAL_SHARDS_ENV_VAR: str(total_shards)} + extra_env = { + SHARD_INDEX_ENV_VAR: str(shard_index), + TOTAL_SHARDS_ENV_VAR: str(total_shards), + } return InvokeWithModifiedEnv(extra_env, RunAndExtractTestList, command) + # The unit test. @@ -236,10 +260,10 @@ class GTestFilterUnitTest(gtest_test_utils.TestCase): """Asserts that two sets are equal.""" for elem in lhs: - self.assert_(elem in rhs, '%s in %s' % (elem, rhs)) + self.assertTrue(elem in rhs, '%s in %s' % (elem, rhs)) for elem in rhs: - self.assert_(elem in lhs, '%s in %s' % (elem, lhs)) + self.assertTrue(elem in lhs, '%s in %s' % (elem, lhs)) def AssertPartitionIsValid(self, set_var, list_of_sets): """Asserts that list_of_sets is a valid partition of set_var.""" @@ -271,13 +295,13 @@ class GTestFilterUnitTest(gtest_test_utils.TestCase): # into a process using the environment variable. However, we can still # test the case when the variable is not supplied (i.e., gtest_filter is # None). - # pylint: disable-msg=C6403 + # pylint: disable=g-explicit-bool-comparison if CAN_TEST_EMPTY_FILTER or gtest_filter != '': SetEnvVar(FILTER_ENV_VAR, gtest_filter) tests_run = RunAndExtractTestList()[0] SetEnvVar(FILTER_ENV_VAR, None) self.AssertSetEqual(tests_run, tests_to_run) - # pylint: enable-msg=C6403 + # pylint: enable=g-explicit-bool-comparison # Next, tests using the command line flag. @@ -289,8 +313,14 @@ class GTestFilterUnitTest(gtest_test_utils.TestCase): tests_run = RunAndExtractTestList(args)[0] self.AssertSetEqual(tests_run, tests_to_run) - def RunAndVerifyWithSharding(self, gtest_filter, total_shards, tests_to_run, - args=None, check_exit_0=False): + def RunAndVerifyWithSharding( + self, + gtest_filter, + total_shards, + tests_to_run, + args=None, + check_exit_0=False, + ): """Checks that binary runs correct tests for the given filter and shard. Runs all shards of googletest-filter-unittest_ with the given filter, and @@ -302,9 +332,9 @@ class GTestFilterUnitTest(gtest_test_utils.TestCase): gtest_filter: A filter to apply to the tests. total_shards: A total number of shards to split test run into. tests_to_run: A set of tests expected to run. - args : Arguments to pass to the to the test binary. - check_exit_0: When set to a true value, make sure that all shards - return 0. + args: Arguments to pass to the to the test binary. + check_exit_0: When set to a true value, make sure that all shards return + 0. """ tests_to_run = self.AdjustForParameterizedTests(tests_to_run) @@ -314,7 +344,7 @@ class GTestFilterUnitTest(gtest_test_utils.TestCase): # into a process using the environment variable. However, we can still # test the case when the variable is not supplied (i.e., gtest_filter is # None). - # pylint: disable-msg=C6403 + # pylint: disable=g-explicit-bool-comparison if CAN_TEST_EMPTY_FILTER or gtest_filter != '': SetEnvVar(FILTER_ENV_VAR, gtest_filter) partition = [] @@ -326,7 +356,7 @@ class GTestFilterUnitTest(gtest_test_utils.TestCase): self.AssertPartitionIsValid(tests_to_run, partition) SetEnvVar(FILTER_ENV_VAR, None) - # pylint: enable-msg=C6403 + # pylint: enable=g-explicit-bool-comparison def RunAndVerifyAllowingDisabled(self, gtest_filter, tests_to_run): """Checks that the binary runs correct set of tests for the given filter. @@ -358,8 +388,9 @@ class GTestFilterUnitTest(gtest_test_utils.TestCase): global param_tests_present if param_tests_present is None: - param_tests_present = PARAM_TEST_REGEX.search( - RunAndReturnOutput()) is not None + param_tests_present = ( + PARAM_TEST_REGEX.search(RunAndReturnOutput()) is not None + ) def testDefaultBehavior(self): """Tests the behavior of not specifying the filter.""" @@ -411,8 +442,9 @@ class GTestFilterUnitTest(gtest_test_utils.TestCase): BAZ_TESTS = ['BazTest.TestOne', 'BazTest.TestA', 'BazTest.TestB'] self.RunAndVerify('BazTest.*', BAZ_TESTS) - self.RunAndVerifyAllowingDisabled('BazTest.*', - BAZ_TESTS + ['BazTest.DISABLED_TestC']) + self.RunAndVerifyAllowingDisabled( + 'BazTest.*', BAZ_TESTS + ['BazTest.DISABLED_TestC'] + ) def testFilterByTest(self): """Tests filtering by test name.""" @@ -423,38 +455,50 @@ class GTestFilterUnitTest(gtest_test_utils.TestCase): """Select only the disabled tests to run.""" self.RunAndVerify('DISABLED_FoobarTest.Test1', []) - self.RunAndVerifyAllowingDisabled('DISABLED_FoobarTest.Test1', - ['DISABLED_FoobarTest.Test1']) + self.RunAndVerifyAllowingDisabled( + 'DISABLED_FoobarTest.Test1', ['DISABLED_FoobarTest.Test1'] + ) self.RunAndVerify('*DISABLED_*', []) self.RunAndVerifyAllowingDisabled('*DISABLED_*', DISABLED_TESTS) self.RunAndVerify('*.DISABLED_*', []) - self.RunAndVerifyAllowingDisabled('*.DISABLED_*', [ - 'BarTest.DISABLED_TestFour', - 'BarTest.DISABLED_TestFive', - 'BazTest.DISABLED_TestC', - 'DISABLED_FoobarTest.DISABLED_Test2', - ]) + self.RunAndVerifyAllowingDisabled( + '*.DISABLED_*', + [ + 'BarTest.DISABLED_TestFour', + 'BarTest.DISABLED_TestFive', + 'BazTest.DISABLED_TestC', + 'DISABLED_FoobarTest.DISABLED_Test2', + ], + ) self.RunAndVerify('DISABLED_*', []) - self.RunAndVerifyAllowingDisabled('DISABLED_*', [ - 'DISABLED_FoobarTest.Test1', - 'DISABLED_FoobarTest.DISABLED_Test2', - 'DISABLED_FoobarbazTest.TestA', - ]) + self.RunAndVerifyAllowingDisabled( + 'DISABLED_*', + [ + 'DISABLED_FoobarTest.Test1', + 'DISABLED_FoobarTest.DISABLED_Test2', + 'DISABLED_FoobarbazTest.TestA', + ], + ) def testWildcardInTestCaseName(self): """Tests using wildcard in the test case name.""" - self.RunAndVerify('*a*.*', [ - 'BarTest.TestOne', - 'BarTest.TestTwo', - 'BarTest.TestThree', - - 'BazTest.TestOne', - 'BazTest.TestA', - 'BazTest.TestB', ] + DEATH_TESTS + PARAM_TESTS) + self.RunAndVerify( + '*a*.*', + [ + 'BarTest.TestOne', + 'BarTest.TestTwo', + 'BarTest.TestThree', + 'BazTest.TestOne', + 'BazTest.TestA', + 'BazTest.TestB', + ] + + DEATH_TESTS + + PARAM_TESTS, + ) def testWildcardInTestName(self): """Tests using wildcard in the test name.""" @@ -464,23 +508,27 @@ class GTestFilterUnitTest(gtest_test_utils.TestCase): def testFilterWithoutDot(self): """Tests a filter that has no '.' in it.""" - self.RunAndVerify('*z*', [ - 'FooTest.Xyz', - - 'BazTest.TestOne', - 'BazTest.TestA', - 'BazTest.TestB', - ]) + self.RunAndVerify( + '*z*', + [ + 'FooTest.Xyz', + 'BazTest.TestOne', + 'BazTest.TestA', + 'BazTest.TestB', + ], + ) def testTwoPatterns(self): """Tests filters that consist of two patterns.""" - self.RunAndVerify('Foo*.*:*A*', [ - 'FooTest.Abc', - 'FooTest.Xyz', - - 'BazTest.TestA', - ]) + self.RunAndVerify( + 'Foo*.*:*A*', + [ + 'FooTest.Abc', + 'FooTest.Xyz', + 'BazTest.TestA', + ], + ) # An empty pattern + a non-empty one self.RunAndVerify(':*A*', ['FooTest.Abc', 'BazTest.TestA']) @@ -488,83 +536,109 @@ class GTestFilterUnitTest(gtest_test_utils.TestCase): def testThreePatterns(self): """Tests filters that consist of three patterns.""" - self.RunAndVerify('*oo*:*A*:*One', [ - 'FooTest.Abc', - 'FooTest.Xyz', - - 'BarTest.TestOne', - - 'BazTest.TestOne', - 'BazTest.TestA', - ]) + self.RunAndVerify( + '*oo*:*A*:*One', + [ + 'FooTest.Abc', + 'FooTest.Xyz', + 'BarTest.TestOne', + 'BazTest.TestOne', + 'BazTest.TestA', + ], + ) # The 2nd pattern is empty. - self.RunAndVerify('*oo*::*One', [ - 'FooTest.Abc', - 'FooTest.Xyz', - - 'BarTest.TestOne', - - 'BazTest.TestOne', - ]) + self.RunAndVerify( + '*oo*::*One', + [ + 'FooTest.Abc', + 'FooTest.Xyz', + 'BarTest.TestOne', + 'BazTest.TestOne', + ], + ) # The last 2 patterns are empty. - self.RunAndVerify('*oo*::', [ - 'FooTest.Abc', - 'FooTest.Xyz', - ]) + self.RunAndVerify( + '*oo*::', + [ + 'FooTest.Abc', + 'FooTest.Xyz', + ], + ) def testNegativeFilters(self): - self.RunAndVerify('*-BazTest.TestOne', [ - 'FooTest.Abc', - 'FooTest.Xyz', - - 'BarTest.TestOne', - 'BarTest.TestTwo', - 'BarTest.TestThree', - - 'BazTest.TestA', - 'BazTest.TestB', - ] + DEATH_TESTS + PARAM_TESTS) - - self.RunAndVerify('*-FooTest.Abc:BazTest.*', [ - 'FooTest.Xyz', - - 'BarTest.TestOne', - 'BarTest.TestTwo', - 'BarTest.TestThree', - ] + DEATH_TESTS + PARAM_TESTS) - - self.RunAndVerify('BarTest.*-BarTest.TestOne', [ - 'BarTest.TestTwo', - 'BarTest.TestThree', - ]) + self.RunAndVerify( + '*-BazTest.TestOne', + [ + 'FooTest.Abc', + 'FooTest.Xyz', + 'BarTest.TestOne', + 'BarTest.TestTwo', + 'BarTest.TestThree', + 'BazTest.TestA', + 'BazTest.TestB', + ] + + DEATH_TESTS + + PARAM_TESTS, + ) + + self.RunAndVerify( + '*-FooTest.Abc:BazTest.*', + [ + 'FooTest.Xyz', + 'BarTest.TestOne', + 'BarTest.TestTwo', + 'BarTest.TestThree', + ] + + DEATH_TESTS + + PARAM_TESTS, + ) + + self.RunAndVerify( + 'BarTest.*-BarTest.TestOne', + [ + 'BarTest.TestTwo', + 'BarTest.TestThree', + ], + ) # Tests without leading '*'. - self.RunAndVerify('-FooTest.Abc:FooTest.Xyz:BazTest.*', [ - 'BarTest.TestOne', - 'BarTest.TestTwo', - 'BarTest.TestThree', - ] + DEATH_TESTS + PARAM_TESTS) + self.RunAndVerify( + '-FooTest.Abc:FooTest.Xyz:BazTest.*', + [ + 'BarTest.TestOne', + 'BarTest.TestTwo', + 'BarTest.TestThree', + ] + + DEATH_TESTS + + PARAM_TESTS, + ) # Value parameterized tests. self.RunAndVerify('*/*', PARAM_TESTS) # Value parameterized tests filtering by the sequence name. - self.RunAndVerify('SeqP/*', [ - 'SeqP/ParamTest.TestX/0', - 'SeqP/ParamTest.TestX/1', - 'SeqP/ParamTest.TestY/0', - 'SeqP/ParamTest.TestY/1', - ]) + self.RunAndVerify( + 'SeqP/*', + [ + 'SeqP/ParamTest.TestX/0', + 'SeqP/ParamTest.TestX/1', + 'SeqP/ParamTest.TestY/0', + 'SeqP/ParamTest.TestY/1', + ], + ) # Value parameterized tests filtering by the test name. - self.RunAndVerify('*/0', [ - 'SeqP/ParamTest.TestX/0', - 'SeqP/ParamTest.TestY/0', - 'SeqQ/ParamTest.TestX/0', - 'SeqQ/ParamTest.TestY/0', - ]) + self.RunAndVerify( + '*/0', + [ + 'SeqP/ParamTest.TestX/0', + 'SeqP/ParamTest.TestY/0', + 'SeqQ/ParamTest.TestX/0', + 'SeqQ/ParamTest.TestY/0', + ], + ) def testFlagOverridesEnvVar(self): """Tests that the filter flag overrides the filtering env. variable.""" @@ -579,41 +653,70 @@ class GTestFilterUnitTest(gtest_test_utils.TestCase): def testShardStatusFileIsCreated(self): """Tests that the shard file is created if specified in the environment.""" - shard_status_file = os.path.join(gtest_test_utils.GetTempDir(), - 'shard_status_file') - self.assert_(not os.path.exists(shard_status_file)) + shard_status_file = os.path.join( + gtest_test_utils.GetTempDir(), 'shard_status_file' + ) + self.assertTrue(not os.path.exists(shard_status_file)) extra_env = {SHARD_STATUS_FILE_ENV_VAR: shard_status_file} try: InvokeWithModifiedEnv(extra_env, RunAndReturnOutput) finally: - self.assert_(os.path.exists(shard_status_file)) + self.assertTrue(os.path.exists(shard_status_file)) os.remove(shard_status_file) def testShardStatusFileIsCreatedWithListTests(self): """Tests that the shard file is created with the "list_tests" flag.""" - shard_status_file = os.path.join(gtest_test_utils.GetTempDir(), - 'shard_status_file2') - self.assert_(not os.path.exists(shard_status_file)) + shard_status_file = os.path.join( + gtest_test_utils.GetTempDir(), 'shard_status_file2' + ) + self.assertTrue(not os.path.exists(shard_status_file)) extra_env = {SHARD_STATUS_FILE_ENV_VAR: shard_status_file} try: - output = InvokeWithModifiedEnv(extra_env, - RunAndReturnOutput, - [LIST_TESTS_FLAG]) + output = InvokeWithModifiedEnv( + extra_env, RunAndReturnOutput, [LIST_TESTS_FLAG] + ) finally: # This assertion ensures that Google Test enumerated the tests as # opposed to running them. - self.assert_('[==========]' not in output, - 'Unexpected output during test enumeration.\n' - 'Please ensure that LIST_TESTS_FLAG is assigned the\n' - 'correct flag value for listing Google Test tests.') - - self.assert_(os.path.exists(shard_status_file)) + self.assertTrue( + '[==========]' not in output, + ( + 'Unexpected output during test enumeration.\n' + 'Please ensure that LIST_TESTS_FLAG is assigned the\n' + 'correct flag value for listing Google Test tests.' + ), + ) + + self.assertTrue(os.path.exists(shard_status_file)) os.remove(shard_status_file) + def testDisabledBanner(self): + """Tests that the disabled banner prints only tests that match filter.""" + make_filter = lambda s: ['--%s=%s' % (FILTER_FLAG, s)] + + banners = RunAndExtractDisabledBannerList(make_filter('*')) + self.AssertSetEqual( + banners, + [ + 'BarTest.DISABLED_TestFour', + 'BarTest.DISABLED_TestFive', + 'BazTest.DISABLED_TestC', + ], + ) + + banners = RunAndExtractDisabledBannerList(make_filter('Bar*')) + self.AssertSetEqual( + banners, ['BarTest.DISABLED_TestFour', 'BarTest.DISABLED_TestFive'] + ) + + banners = RunAndExtractDisabledBannerList(make_filter('*-Bar*')) + self.AssertSetEqual(banners, ['BazTest.DISABLED_TestC']) + if SUPPORTS_DEATH_TESTS: + def testShardingWorksWithDeathTests(self): """Tests integration with death tests and sharding.""" @@ -621,19 +724,23 @@ class GTestFilterUnitTest(gtest_test_utils.TestCase): expected_tests = [ 'HasDeathTest.Test1', 'HasDeathTest.Test2', - 'SeqP/ParamTest.TestX/0', 'SeqP/ParamTest.TestX/1', 'SeqP/ParamTest.TestY/0', 'SeqP/ParamTest.TestY/1', - ] - - for flag in ['--gtest_death_test_style=threadsafe', - '--gtest_death_test_style=fast']: - self.RunAndVerifyWithSharding(gtest_filter, 3, expected_tests, - check_exit_0=True, args=[flag]) - self.RunAndVerifyWithSharding(gtest_filter, 5, expected_tests, - check_exit_0=True, args=[flag]) + ] + + for flag in [ + '--gtest_death_test_style=threadsafe', + '--gtest_death_test_style=fast', + ]: + self.RunAndVerifyWithSharding( + gtest_filter, 3, expected_tests, check_exit_0=True, args=[flag] + ) + self.RunAndVerifyWithSharding( + gtest_filter, 5, expected_tests, check_exit_0=True, args=[flag] + ) + if __name__ == '__main__': gtest_test_utils.Main() |