diff options
Diffstat (limited to 'catapult/telemetry/telemetry/internal/results/page_test_results_unittest.py')
| -rw-r--r-- | catapult/telemetry/telemetry/internal/results/page_test_results_unittest.py | 512 |
1 files changed, 512 insertions, 0 deletions
diff --git a/catapult/telemetry/telemetry/internal/results/page_test_results_unittest.py b/catapult/telemetry/telemetry/internal/results/page_test_results_unittest.py new file mode 100644 index 00000000..39ca0416 --- /dev/null +++ b/catapult/telemetry/telemetry/internal/results/page_test_results_unittest.py @@ -0,0 +1,512 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import unittest + +from telemetry import benchmark +from telemetry import story +from telemetry.internal.results import base_test_results_unittest +from telemetry.internal.results import chart_json_output_formatter +from telemetry.internal.results import json_output_formatter +from telemetry.internal.results import page_test_results +from telemetry import page as page_module +from telemetry.testing import stream +from telemetry.timeline import trace_data +from telemetry.value import failure +from telemetry.value import histogram +from telemetry.value import improvement_direction +from telemetry.value import scalar +from telemetry.value import skip +from telemetry.value import trace + + +class PageTestResultsTest(base_test_results_unittest.BaseTestResultsUnittest): + def setUp(self): + story_set = story.StorySet(base_dir=os.path.dirname(__file__)) + story_set.AddStory(page_module.Page("http://www.bar.com/", story_set, story_set.base_dir)) + story_set.AddStory(page_module.Page("http://www.baz.com/", story_set, story_set.base_dir)) + story_set.AddStory(page_module.Page("http://www.foo.com/", story_set, story_set.base_dir)) + self.story_set = story_set + + @property + def pages(self): + return self.story_set.stories + + def testFailures(self): + results = page_test_results.PageTestResults() + results.WillRunPage(self.pages[0]) + results.AddValue( + failure.FailureValue(self.pages[0], self.CreateException())) + results.DidRunPage(self.pages[0]) + + results.WillRunPage(self.pages[1]) + results.DidRunPage(self.pages[1]) + + self.assertEqual(set([self.pages[0]]), results.pages_that_failed) + self.assertEqual(set([self.pages[1]]), results.pages_that_succeeded) + + self.assertEqual(2, len(results.all_page_runs)) + self.assertTrue(results.all_page_runs[0].failed) + self.assertTrue(results.all_page_runs[1].ok) + + def testSkips(self): + results = page_test_results.PageTestResults() + results.WillRunPage(self.pages[0]) + results.AddValue(skip.SkipValue(self.pages[0], 'testing reason')) + results.DidRunPage(self.pages[0]) + + results.WillRunPage(self.pages[1]) + results.DidRunPage(self.pages[1]) + + self.assertTrue(results.all_page_runs[0].skipped) + self.assertEqual(self.pages[0], results.all_page_runs[0].story) + self.assertEqual(set([self.pages[0], self.pages[1]]), + results.pages_that_succeeded) + + self.assertEqual(2, len(results.all_page_runs)) + self.assertTrue(results.all_page_runs[0].skipped) + self.assertTrue(results.all_page_runs[1].ok) + + def testBasic(self): + results = page_test_results.PageTestResults() + results.WillRunPage(self.pages[0]) + results.AddValue(scalar.ScalarValue( + self.pages[0], 'a', 'seconds', 3, + improvement_direction=improvement_direction.UP)) + results.DidRunPage(self.pages[0]) + + results.WillRunPage(self.pages[1]) + results.AddValue(scalar.ScalarValue( + self.pages[1], 'a', 'seconds', 3, + improvement_direction=improvement_direction.UP)) + results.DidRunPage(self.pages[1]) + + results.PrintSummary() + + values = results.FindPageSpecificValuesForPage(self.pages[0], 'a') + self.assertEquals(1, len(values)) + v = values[0] + self.assertEquals(v.name, 'a') + self.assertEquals(v.page, self.pages[0]) + + values = results.FindAllPageSpecificValuesNamed('a') + assert len(values) == 2 + + def testAddValueWithStoryGroupingKeys(self): + results = page_test_results.PageTestResults() + self.pages[0].grouping_keys['foo'] = 'bar' + self.pages[0].grouping_keys['answer'] = '42' + results.WillRunPage(self.pages[0]) + results.AddValue(scalar.ScalarValue( + self.pages[0], 'a', 'seconds', 3, + improvement_direction=improvement_direction.UP)) + results.DidRunPage(self.pages[0]) + + results.PrintSummary() + + values = results.FindPageSpecificValuesForPage(self.pages[0], 'a') + v = values[0] + self.assertEquals(v.grouping_keys['foo'], 'bar') + self.assertEquals(v.grouping_keys['answer'], '42') + self.assertEquals(v.tir_label, '42_bar') + + def testAddValueWithStoryGroupingKeysAndMatchingTirLabel(self): + results = page_test_results.PageTestResults() + self.pages[0].grouping_keys['foo'] = 'bar' + self.pages[0].grouping_keys['answer'] = '42' + results.WillRunPage(self.pages[0]) + results.AddValue(scalar.ScalarValue( + self.pages[0], 'a', 'seconds', 3, + improvement_direction=improvement_direction.UP, + tir_label='42_bar')) + results.DidRunPage(self.pages[0]) + + results.PrintSummary() + + values = results.FindPageSpecificValuesForPage(self.pages[0], 'a') + v = values[0] + self.assertEquals(v.grouping_keys['foo'], 'bar') + self.assertEquals(v.grouping_keys['answer'], '42') + self.assertEquals(v.tir_label, '42_bar') + + def testAddValueWithStoryGroupingKeysAndMismatchingTirLabel(self): + results = page_test_results.PageTestResults() + self.pages[0].grouping_keys['foo'] = 'bar' + self.pages[0].grouping_keys['answer'] = '42' + results.WillRunPage(self.pages[0]) + with self.assertRaises(AssertionError): + results.AddValue(scalar.ScalarValue( + self.pages[0], 'a', 'seconds', 3, + improvement_direction=improvement_direction.UP, + tir_label='another_label')) + + def testAddValueWithDuplicateStoryGroupingKeyFails(self): + results = page_test_results.PageTestResults() + self.pages[0].grouping_keys['foo'] = 'bar' + results.WillRunPage(self.pages[0]) + with self.assertRaises(AssertionError): + results.AddValue(scalar.ScalarValue( + self.pages[0], 'a', 'seconds', 3, + improvement_direction=improvement_direction.UP, + grouping_keys={'foo': 'bar'})) + + def testUrlIsInvalidValue(self): + results = page_test_results.PageTestResults() + results.WillRunPage(self.pages[0]) + self.assertRaises( + AssertionError, + lambda: results.AddValue(scalar.ScalarValue( + self.pages[0], 'url', 'string', 'foo', + improvement_direction=improvement_direction.UP))) + + def testAddSummaryValueWithPageSpecified(self): + results = page_test_results.PageTestResults() + results.WillRunPage(self.pages[0]) + self.assertRaises( + AssertionError, + lambda: results.AddSummaryValue(scalar.ScalarValue( + self.pages[0], 'a', 'units', 3, + improvement_direction=improvement_direction.UP))) + + def testUnitChange(self): + results = page_test_results.PageTestResults() + results.WillRunPage(self.pages[0]) + results.AddValue(scalar.ScalarValue( + self.pages[0], 'a', 'seconds', 3, + improvement_direction=improvement_direction.UP)) + results.DidRunPage(self.pages[0]) + + results.WillRunPage(self.pages[1]) + self.assertRaises( + AssertionError, + lambda: results.AddValue(scalar.ScalarValue( + self.pages[1], 'a', 'foobgrobbers', 3, + improvement_direction=improvement_direction.UP))) + + def testTypeChange(self): + results = page_test_results.PageTestResults() + results.WillRunPage(self.pages[0]) + results.AddValue(scalar.ScalarValue( + self.pages[0], 'a', 'seconds', 3, + improvement_direction=improvement_direction.UP)) + results.DidRunPage(self.pages[0]) + + results.WillRunPage(self.pages[1]) + self.assertRaises( + AssertionError, + lambda: results.AddValue(histogram.HistogramValue( + self.pages[1], 'a', 'seconds', + raw_value_json='{"buckets": [{"low": 1, "high": 2, "count": 1}]}', + improvement_direction=improvement_direction.UP))) + + def testGetPagesThatSucceededAllPagesFail(self): + results = page_test_results.PageTestResults() + results.WillRunPage(self.pages[0]) + results.AddValue(scalar.ScalarValue( + self.pages[0], 'a', 'seconds', 3, + improvement_direction=improvement_direction.UP)) + results.AddValue(failure.FailureValue.FromMessage(self.pages[0], 'message')) + results.DidRunPage(self.pages[0]) + + results.WillRunPage(self.pages[1]) + results.AddValue(scalar.ScalarValue( + self.pages[1], 'a', 'seconds', 7, + improvement_direction=improvement_direction.UP)) + results.AddValue(failure.FailureValue.FromMessage(self.pages[1], 'message')) + results.DidRunPage(self.pages[1]) + + results.PrintSummary() + self.assertEquals(0, len(results.pages_that_succeeded)) + + def testGetSuccessfulPageValuesMergedNoFailures(self): + results = page_test_results.PageTestResults() + results.WillRunPage(self.pages[0]) + results.AddValue(scalar.ScalarValue( + self.pages[0], 'a', 'seconds', 3, + improvement_direction=improvement_direction.UP)) + self.assertEquals(1, len(results.all_page_specific_values)) + results.DidRunPage(self.pages[0]) + + def testGetAllValuesForSuccessfulPages(self): + results = page_test_results.PageTestResults() + results.WillRunPage(self.pages[0]) + value1 = scalar.ScalarValue( + self.pages[0], 'a', 'seconds', 3, + improvement_direction=improvement_direction.UP) + results.AddValue(value1) + results.DidRunPage(self.pages[0]) + + results.WillRunPage(self.pages[1]) + value2 = scalar.ScalarValue( + self.pages[1], 'a', 'seconds', 3, + improvement_direction=improvement_direction.UP) + results.AddValue(value2) + results.DidRunPage(self.pages[1]) + + results.WillRunPage(self.pages[2]) + value3 = scalar.ScalarValue( + self.pages[2], 'a', 'seconds', 3, + improvement_direction=improvement_direction.UP) + results.AddValue(value3) + results.DidRunPage(self.pages[2]) + + self.assertEquals( + [value1, value2, value3], results.all_page_specific_values) + + def testGetAllValuesForSuccessfulPagesOnePageFails(self): + results = page_test_results.PageTestResults() + results.WillRunPage(self.pages[0]) + value1 = scalar.ScalarValue( + self.pages[0], 'a', 'seconds', 3, + improvement_direction=improvement_direction.UP) + results.AddValue(value1) + results.DidRunPage(self.pages[0]) + + results.WillRunPage(self.pages[1]) + value2 = failure.FailureValue.FromMessage(self.pages[1], 'Failure') + results.AddValue(value2) + results.DidRunPage(self.pages[1]) + + results.WillRunPage(self.pages[2]) + value3 = scalar.ScalarValue( + self.pages[2], 'a', 'seconds', 3, + improvement_direction=improvement_direction.UP) + results.AddValue(value3) + results.DidRunPage(self.pages[2]) + + self.assertEquals( + [value1, value2, value3], results.all_page_specific_values) + + def testFindValues(self): + results = page_test_results.PageTestResults() + results.WillRunPage(self.pages[0]) + v0 = scalar.ScalarValue( + self.pages[0], 'a', 'seconds', 3, + improvement_direction=improvement_direction.UP) + results.AddValue(v0) + v1 = scalar.ScalarValue( + self.pages[0], 'a', 'seconds', 4, + improvement_direction=improvement_direction.UP) + results.AddValue(v1) + results.DidRunPage(self.pages[1]) + + values = results.FindValues(lambda v: v.value == 3) + self.assertEquals([v0], values) + + def testValueWithTIRLabel(self): + results = page_test_results.PageTestResults() + results.WillRunPage(self.pages[0]) + v0 = scalar.ScalarValue( + self.pages[0], 'a', 'seconds', 3, tir_label='foo', + improvement_direction=improvement_direction.UP) + results.AddValue(v0) + v1 = scalar.ScalarValue( + self.pages[0], 'a', 'seconds', 3, tir_label='bar', + improvement_direction=improvement_direction.UP) + results.AddValue(v1) + results.DidRunPage(self.pages[0]) + + values = results.FindAllPageSpecificValuesFromIRNamed('foo', 'a') + self.assertEquals([v0], values) + + def testTraceValue(self): + results = page_test_results.PageTestResults() + results.WillRunPage(self.pages[0]) + results.AddValue(trace.TraceValue( + None, trace_data.CreateTraceDataFromRawData([[{'test': 1}]]))) + results.DidRunPage(self.pages[0]) + + results.WillRunPage(self.pages[1]) + results.AddValue(trace.TraceValue( + None, trace_data.CreateTraceDataFromRawData([[{'test': 2}]]))) + results.DidRunPage(self.pages[1]) + + results.PrintSummary() + + values = results.FindAllTraceValues() + self.assertEquals(2, len(values)) + + def testCleanUpCleansUpTraceValues(self): + results = page_test_results.PageTestResults() + v0 = trace.TraceValue( + None, trace_data.CreateTraceDataFromRawData([{'test': 1}])) + v1 = trace.TraceValue( + None, trace_data.CreateTraceDataFromRawData([{'test': 2}])) + + results.WillRunPage(self.pages[0]) + results.AddValue(v0) + results.DidRunPage(self.pages[0]) + + results.WillRunPage(self.pages[1]) + results.AddValue(v1) + results.DidRunPage(self.pages[1]) + + results.CleanUp() + self.assertTrue(v0.cleaned_up) + self.assertTrue(v1.cleaned_up) + + def testNoTracesLeftAfterCleanUp(self): + results = page_test_results.PageTestResults() + v0 = trace.TraceValue(None, + trace_data.CreateTraceDataFromRawData([{'test': 1}])) + v1 = trace.TraceValue(None, + trace_data.CreateTraceDataFromRawData([{'test': 2}])) + + results.WillRunPage(self.pages[0]) + results.AddValue(v0) + results.DidRunPage(self.pages[0]) + + results.WillRunPage(self.pages[1]) + results.AddValue(v1) + results.DidRunPage(self.pages[1]) + + results.CleanUp() + self.assertFalse(results.FindAllTraceValues()) + + def testPrintSummaryDisabledResults(self): + output_stream = stream.TestOutputStream() + output_formatters = [] + benchmark_metadata = benchmark.BenchmarkMetadata( + 'benchmark_name', 'benchmark_description') + output_formatters.append( + chart_json_output_formatter.ChartJsonOutputFormatter( + output_stream, benchmark_metadata)) + output_formatters.append(json_output_formatter.JsonOutputFormatter( + output_stream, benchmark_metadata)) + results = page_test_results.PageTestResults( + output_formatters=output_formatters, benchmark_enabled=False) + results.PrintSummary() + self.assertEquals(output_stream.output_data, + "{\n \"enabled\": false,\n \"benchmark_name\": \"benchmark_name\"\n}\n") + + +class PageTestResultsFilterTest(unittest.TestCase): + def setUp(self): + story_set = story.StorySet(base_dir=os.path.dirname(__file__)) + story_set.AddStory( + page_module.Page('http://www.foo.com/', story_set, story_set.base_dir)) + story_set.AddStory( + page_module.Page('http://www.bar.com/', story_set, story_set.base_dir)) + self.story_set = story_set + + @property + def pages(self): + return self.story_set.stories + + def testFilterValue(self): + def AcceptValueNamed_a(value, _): + return value.name == 'a' + results = page_test_results.PageTestResults( + value_can_be_added_predicate=AcceptValueNamed_a) + results.WillRunPage(self.pages[0]) + results.AddValue(scalar.ScalarValue( + self.pages[0], 'a', 'seconds', 3, + improvement_direction=improvement_direction.UP)) + results.AddValue(scalar.ScalarValue( + self.pages[0], 'b', 'seconds', 3, + improvement_direction=improvement_direction.UP)) + results.DidRunPage(self.pages[0]) + + results.WillRunPage(self.pages[1]) + results.AddValue(scalar.ScalarValue( + self.pages[1], 'a', 'seconds', 3, + improvement_direction=improvement_direction.UP)) + results.AddValue(scalar.ScalarValue( + self.pages[1], 'd', 'seconds', 3, + improvement_direction=improvement_direction.UP)) + results.DidRunPage(self.pages[1]) + results.PrintSummary() + self.assertEquals( + [('a', 'http://www.foo.com/'), ('a', 'http://www.bar.com/')], + [(v.name, v.page.url) for v in results.all_page_specific_values]) + + def testFilterIsFirstResult(self): + def AcceptSecondValues(_, is_first_result): + return not is_first_result + results = page_test_results.PageTestResults( + value_can_be_added_predicate=AcceptSecondValues) + + # First results (filtered out) + results.WillRunPage(self.pages[0]) + results.AddValue(scalar.ScalarValue( + self.pages[0], 'a', 'seconds', 7, + improvement_direction=improvement_direction.UP)) + results.AddValue(scalar.ScalarValue( + self.pages[0], 'b', 'seconds', 8, + improvement_direction=improvement_direction.UP)) + results.DidRunPage(self.pages[0]) + results.WillRunPage(self.pages[1]) + results.AddValue(scalar.ScalarValue( + self.pages[1], 'a', 'seconds', 5, + improvement_direction=improvement_direction.UP)) + results.AddValue(scalar.ScalarValue( + self.pages[1], 'd', 'seconds', 6, + improvement_direction=improvement_direction.UP)) + results.DidRunPage(self.pages[1]) + + # Second results + results.WillRunPage(self.pages[0]) + results.AddValue(scalar.ScalarValue( + self.pages[0], 'a', 'seconds', 3, + improvement_direction=improvement_direction.UP)) + results.AddValue(scalar.ScalarValue( + self.pages[0], 'b', 'seconds', 4, + improvement_direction=improvement_direction.UP)) + results.DidRunPage(self.pages[0]) + results.WillRunPage(self.pages[1]) + results.AddValue(scalar.ScalarValue( + self.pages[1], 'a', 'seconds', 1, + improvement_direction=improvement_direction.UP)) + results.AddValue(scalar.ScalarValue( + self.pages[1], 'd', 'seconds', 2, + improvement_direction=improvement_direction.UP)) + results.DidRunPage(self.pages[1]) + results.PrintSummary() + expected_values = [ + ('a', 'http://www.foo.com/', 3), + ('b', 'http://www.foo.com/', 4), + ('a', 'http://www.bar.com/', 1), + ('d', 'http://www.bar.com/', 2)] + actual_values = [(v.name, v.page.url, v.value) + for v in results.all_page_specific_values] + self.assertEquals(expected_values, actual_values) + + def testFailureValueCannotBeFiltered(self): + def AcceptValueNamed_a(value, _): + return value.name == 'a' + results = page_test_results.PageTestResults( + value_can_be_added_predicate=AcceptValueNamed_a) + results.WillRunPage(self.pages[0]) + results.AddValue(scalar.ScalarValue( + self.pages[0], 'b', 'seconds', 8, + improvement_direction=improvement_direction.UP)) + failure_value = failure.FailureValue.FromMessage(self.pages[0], 'failure') + results.AddValue(failure_value) + results.DidRunPage(self.pages[0]) + results.PrintSummary() + + # Although predicate says only accept values named 'a', the failure value is + # added anyway. + self.assertEquals(len(results.all_page_specific_values), 1) + self.assertIn(failure_value, results.all_page_specific_values) + + def testSkipValueCannotBeFiltered(self): + def AcceptValueNamed_a(value, _): + return value.name == 'a' + results = page_test_results.PageTestResults( + value_can_be_added_predicate=AcceptValueNamed_a) + results.WillRunPage(self.pages[0]) + skip_value = skip.SkipValue(self.pages[0], 'skip for testing') + results.AddValue(scalar.ScalarValue( + self.pages[0], 'b', 'seconds', 8, + improvement_direction=improvement_direction.UP)) + results.AddValue(skip_value) + results.DidRunPage(self.pages[0]) + results.PrintSummary() + + # Although predicate says only accept value with named 'a', skip value is + # added anyway. + self.assertEquals(len(results.all_page_specific_values), 1) + self.assertIn(skip_value, results.all_page_specific_values) |