aboutsummaryrefslogtreecommitdiff
path: root/report_generator.py
blob: 6c7467aaca71a63e2022eb86d162d7ab035b84f5 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
#!/usr/bin/python
#
# Copyright 2010 Google Inc. All Rights Reserved.

"""Script to  compare a baseline results file to a new results file."""

__author__ = "raymes@google.com (Raymes Khoury)"

import sys
from utils import logger
from utils import html_tools

PASS = "pass"
FAIL = "fail"
NOT_EXECUTED = "not executed"

class ResultsReport:
  def __init__(self, report, num_tests_executed, num_passes, num_failures, num_regressions):
    self.report = report
    self.num_tests_executed = num_tests_executed
    self.num_passes = num_passes
    self.num_failures = num_failures
    self.num_regressions = num_regressions
  
  def GetReport(self):
    return self.report
  
  def GetNumExecuted(self):
    return self.num_tests_executed
  
  def GetNumPasses(self):
    return self.num_passes
  
  def GetNumFailures(self):
    return self.num_failures
  
  def GetNumRegressions(self):
    return self.num_regressions

  def GetSummary(self):
    summary = "Tests executed: %s\n" % str(self.num_tests_executed)
    summary += "Tests Passing: %s\n" % str(self.num_passes)
    summary += "Tests Failing: %s\n" % str(self.num_failures)
    summary += "Regressions: %s\n" % str(self.num_regressions)
    return summary

def Usage():
  print "Usage: %s baseline_results new_results" % sys.argv[0]
  sys.exit(1)

def ParseResults(results_filename):
  results = []
  try:
    results_file = open(results_filename, 'rb')
    for line in results_file:
      if line.strip() != "":
        results.append(line.strip().split("\t"))
    results_file.close()
  except IOError:
    logger.GetLogger().LogWarning("Could not open results file: " +
                                  results_filename)
  return results

def GenerateResultsReport(baseline_file, new_result_file):
  baseline_results = ParseResults(baseline_file)
  new_results = ParseResults(new_result_file)

  test_status = {}

  for new_result in new_results:
    new_test_name = new_result[0]
    new_test_result = new_result[1]
    test_status[new_test_name] = (new_test_result, NOT_EXECUTED)

  for baseline_result in baseline_results:
    baseline_test_name = baseline_result[0]
    baseline_test_result = baseline_result[1]
    if baseline_test_name in test_status:
      new_test_result = test_status[baseline_test_name][0]
      test_status[baseline_test_name] = (new_test_result, baseline_test_result)
    else:
      test_status[baseline_test_name] = (NOT_EXECUTED, baseline_test_result)

  regressions = []
  for result in test_status.keys():
    if test_status[result][0] != test_status[result][1]:
      regressions.append(result)
  
  num_tests_executed = len(new_results)
  num_regressions = len(regressions)
  num_passes = 0
  num_failures = 0
  for result in new_results:
    if result[1] == PASS:
      num_passes += 1
    else:
      num_failures += 1

  report = html_tools.GetPageHeader("Test Summary")
  report += html_tools.GetHeader("Test Summary")
  report += html_tools.GetListHeader()
  report += html_tools.GetListItem("Tests executed: " + str(num_tests_executed))
  report += html_tools.GetListItem("Passes: " + str(num_passes))
  report += html_tools.GetListItem("Failures: " + str(num_failures))
  report += html_tools.GetListItem("Regressions: " + str(num_regressions))
  report += html_tools.GetListFooter()
  report += html_tools.GetHeader("Regressions", 2)
  report += html_tools.GetTableHeader(["Test name", "Expected result", 
                                       "Actual result"])
  
  for regression in regressions:
    report += html_tools.GetTableRow([regression[:150], test_status[regression][1],
                                      test_status[regression][0]])
    report += "\n"
  report += html_tools.GetTableFooter()
  report += html_tools.GetHeader("All Tests", 2)
  report += html_tools.GetTableHeader(["Test name", "Expected result", "Actual result"])
  for result in test_status.keys():
    report += html_tools.GetTableRow([result[:150], test_status[result][1],
                              test_status[result][0]])
    report += "\n"
  report += html_tools.GetTableFooter()
  report += html_tools.GetFooter()
  return ResultsReport(report, num_tests_executed, num_passes, num_failures, num_regressions)

def Main(argv):
  if len(argv) < 2:
    Usage()

  print GenerateResultsReport(argv[1], argv[2])[0]

if __name__ == "__main__":
  Main(sys.argv)