#!/usr/bin/python2 # # Copyright 2010 Google Inc. All Rights Reserved. """Script to compare a baseline results file to a new results file.""" from __future__ import print_function __author__ = 'raymes@google.com (Raymes Khoury)' import sys from cros_utils import logger from cros_utils import html_tools PASS = 'pass' FAIL = 'fail' NOT_EXECUTED = 'not executed' class ResultsReport(object): """Class for holding report results.""" def __init__(self, report, num_tests_executed, num_passes, num_failures, num_regressions): self.report = report self.num_tests_executed = num_tests_executed self.num_passes = num_passes self.num_failures = num_failures self.num_regressions = num_regressions def GetReport(self): return self.report def GetNumExecuted(self): return self.num_tests_executed def GetNumPasses(self): return self.num_passes def GetNumFailures(self): return self.num_failures def GetNumRegressions(self): return self.num_regressions def GetSummary(self): summary = 'Tests executed: %s\n' % str(self.num_tests_executed) summary += 'Tests Passing: %s\n' % str(self.num_passes) summary += 'Tests Failing: %s\n' % str(self.num_failures) summary += 'Regressions: %s\n' % str(self.num_regressions) return summary def Usage(): print('Usage: %s baseline_results new_results' % sys.argv[0]) sys.exit(1) def ParseResults(results_filename): results = [] try: results_file = open(results_filename, 'rb') for line in results_file: if line.strip() != '': results.append(line.strip().split('\t')) results_file.close() except IOError: logger.GetLogger().LogWarning('Could not open results file: ' + results_filename) return results def GenerateResultsReport(baseline_file, new_result_file): baseline_results = ParseResults(baseline_file) new_results = ParseResults(new_result_file) test_status = {} for new_result in new_results: new_test_name = new_result[0] new_test_result = new_result[1] test_status[new_test_name] = (new_test_result, NOT_EXECUTED) for baseline_result in baseline_results: baseline_test_name = baseline_result[0] baseline_test_result = baseline_result[1] if baseline_test_name in test_status: new_test_result = test_status[baseline_test_name][0] test_status[baseline_test_name] = (new_test_result, baseline_test_result) else: test_status[baseline_test_name] = (NOT_EXECUTED, baseline_test_result) regressions = [] for result in test_status.keys(): if test_status[result][0] != test_status[result][1]: regressions.append(result) num_tests_executed = len(new_results) num_regressions = len(regressions) num_passes = 0 num_failures = 0 for result in new_results: if result[1] == PASS: num_passes += 1 else: num_failures += 1 report = html_tools.GetPageHeader('Test Summary') report += html_tools.GetHeader('Test Summary') report += html_tools.GetListHeader() report += html_tools.GetListItem('Tests executed: ' + str(num_tests_executed)) report += html_tools.GetListItem('Passes: ' + str(num_passes)) report += html_tools.GetListItem('Failures: ' + str(num_failures)) report += html_tools.GetListItem('Regressions: ' + str(num_regressions)) report += html_tools.GetListFooter() report += html_tools.GetHeader('Regressions', 2) report += html_tools.GetTableHeader(['Test name', 'Expected result', 'Actual result']) for regression in regressions: report += html_tools.GetTableRow([regression[:150], test_status[regression][ 1], test_status[regression][0]]) report += '\n' report += html_tools.GetTableFooter() report += html_tools.GetHeader('All Tests', 2) report += html_tools.GetTableHeader(['Test name', 'Expected result', 'Actual result']) for result in test_status.keys(): report += html_tools.GetTableRow([result[:150], test_status[result][1], test_status[result][0]]) report += '\n' report += html_tools.GetTableFooter() report += html_tools.GetFooter() return ResultsReport(report, num_tests_executed, num_passes, num_failures, num_regressions) def Main(argv): if len(argv) < 2: Usage() print(GenerateResultsReport(argv[1], argv[2])[0]) if __name__ == '__main__': Main(sys.argv)