1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
|
#!/usr/bin/python2
#
# Copyright 2010 Google Inc. All Rights Reserved.
"""Script to compare a baseline results file to a new results file."""
from __future__ import print_function
__author__ = 'raymes@google.com (Raymes Khoury)'
import sys
from cros_utils import logger
from cros_utils import html_tools
PASS = 'pass'
FAIL = 'fail'
NOT_EXECUTED = 'not executed'
class ResultsReport(object):
"""Class for holding report results."""
def __init__(self, report, num_tests_executed, num_passes, num_failures,
num_regressions):
self.report = report
self.num_tests_executed = num_tests_executed
self.num_passes = num_passes
self.num_failures = num_failures
self.num_regressions = num_regressions
def GetReport(self):
return self.report
def GetNumExecuted(self):
return self.num_tests_executed
def GetNumPasses(self):
return self.num_passes
def GetNumFailures(self):
return self.num_failures
def GetNumRegressions(self):
return self.num_regressions
def GetSummary(self):
summary = 'Tests executed: %s\n' % str(self.num_tests_executed)
summary += 'Tests Passing: %s\n' % str(self.num_passes)
summary += 'Tests Failing: %s\n' % str(self.num_failures)
summary += 'Regressions: %s\n' % str(self.num_regressions)
return summary
def Usage():
print('Usage: %s baseline_results new_results' % sys.argv[0])
sys.exit(1)
def ParseResults(results_filename):
results = []
try:
results_file = open(results_filename, 'rb')
for line in results_file:
if line.strip() != '':
results.append(line.strip().split('\t'))
results_file.close()
except IOError:
logger.GetLogger().LogWarning('Could not open results file: ' +
results_filename)
return results
def GenerateResultsReport(baseline_file, new_result_file):
baseline_results = ParseResults(baseline_file)
new_results = ParseResults(new_result_file)
test_status = {}
for new_result in new_results:
new_test_name = new_result[0]
new_test_result = new_result[1]
test_status[new_test_name] = (new_test_result, NOT_EXECUTED)
for baseline_result in baseline_results:
baseline_test_name = baseline_result[0]
baseline_test_result = baseline_result[1]
if baseline_test_name in test_status:
new_test_result = test_status[baseline_test_name][0]
test_status[baseline_test_name] = (new_test_result, baseline_test_result)
else:
test_status[baseline_test_name] = (NOT_EXECUTED, baseline_test_result)
regressions = []
for result in test_status.keys():
if test_status[result][0] != test_status[result][1]:
regressions.append(result)
num_tests_executed = len(new_results)
num_regressions = len(regressions)
num_passes = 0
num_failures = 0
for result in new_results:
if result[1] == PASS:
num_passes += 1
else:
num_failures += 1
report = html_tools.GetPageHeader('Test Summary')
report += html_tools.GetHeader('Test Summary')
report += html_tools.GetListHeader()
report += html_tools.GetListItem('Tests executed: ' + str(num_tests_executed))
report += html_tools.GetListItem('Passes: ' + str(num_passes))
report += html_tools.GetListItem('Failures: ' + str(num_failures))
report += html_tools.GetListItem('Regressions: ' + str(num_regressions))
report += html_tools.GetListFooter()
report += html_tools.GetHeader('Regressions', 2)
report += html_tools.GetTableHeader(['Test name', 'Expected result',
'Actual result'])
for regression in regressions:
report += html_tools.GetTableRow([regression[:150], test_status[regression][
1], test_status[regression][0]])
report += '\n'
report += html_tools.GetTableFooter()
report += html_tools.GetHeader('All Tests', 2)
report += html_tools.GetTableHeader(['Test name', 'Expected result',
'Actual result'])
for result in test_status.keys():
report += html_tools.GetTableRow([result[:150], test_status[result][1],
test_status[result][0]])
report += '\n'
report += html_tools.GetTableFooter()
report += html_tools.GetFooter()
return ResultsReport(report, num_tests_executed, num_passes, num_failures,
num_regressions)
def Main(argv):
if len(argv) < 2:
Usage()
print(GenerateResultsReport(argv[1], argv[2])[0])
if __name__ == '__main__':
Main(sys.argv)
|
