diff options
Diffstat (limited to 'deprecated')
-rwxr-xr-x | deprecated/build_benchmarks.py | 306 | ||||
-rwxr-xr-x | deprecated/compare_benchmarks.py | 176 | ||||
-rw-r--r-- | deprecated/repo_to_repo_files/binutils-master.json.rtr | 22 | ||||
-rw-r--r-- | deprecated/repo_to_repo_files/binutils-mobile_toolchain_v16.json.rtr | 23 | ||||
-rw-r--r-- | deprecated/repo_to_repo_files/crosperf.json.rtr | 38 | ||||
-rw-r--r-- | deprecated/repo_to_repo_files/gcc-branches_google_4_7.json.rtr | 22 | ||||
-rw-r--r-- | deprecated/repo_to_repo_files/gcc-branches_google_main.json.rtr | 22 | ||||
-rw-r--r-- | deprecated/repo_to_repo_files/gcc-master.json.rtr | 23 | ||||
-rw-r--r-- | deprecated/repo_to_repo_files/gdb-master.json.rtr | 20 | ||||
-rw-r--r-- | deprecated/repo_to_repo_files/toolchain-utils.json.rtr | 28 | ||||
-rwxr-xr-x | deprecated/report_generator.py | 145 | ||||
-rwxr-xr-x | deprecated/run_benchmarks.py | 245 | ||||
-rwxr-xr-x | deprecated/sheriff_rotation.py | 130 | ||||
-rwxr-xr-x | deprecated/summarize_results.py | 145 |
14 files changed, 1345 insertions, 0 deletions
diff --git a/deprecated/build_benchmarks.py b/deprecated/build_benchmarks.py new file mode 100755 index 00000000..c10c74d1 --- /dev/null +++ b/deprecated/build_benchmarks.py @@ -0,0 +1,306 @@ +#!/usr/bin/python2 +# +# Copyright 2010 Google Inc. All Rights Reserved. +"""Script to build ChromeOS benchmarks + +Inputs: + chromeos_root + toolchain_root + board + [chromeos/cpu/<benchname> | + chromeos/browser/[pagecycler|sunspider] | + chromeos/startup] + + This script assumes toolchain has already been built in toolchain_root. + + chromeos/cpu/<benchname> + - Execute bench.py script within chroot to build benchmark + - Copy build results to perflab-bin + + chromeos/startup + - Call build_chromeos to build image. + - Copy image to perflab-bin + + chromeos/browser/* + - Call build_chromebrowser to build image with new browser + - Copy image to perflab-bin + +""" + +from __future__ import print_function + +__author__ = 'bjanakiraman@google.com (Bhaskar Janakiraman)' + +import argparse +import os +import sys +import re + +import build_chromeos +from cros_utils import command_executer +from cros_utils import logger + +KNOWN_BENCHMARKS = [ + 'chromeos/startup', 'chromeos/browser/pagecycler', + 'chromeos/browser/sunspider', 'chromeos/browser/v8bench', + 'chromeos/cpu/bikjmp' +] + +# Commands to build CPU benchmarks. + +CPU_BUILDCMD_CLEAN = ('cd /usr/local/toolchain_root/third_party/android_bench/' + 'v2_0/CLOSED_SOURCE/%s; python ../../scripts/bench.py ' + '--toolchain=/usr/bin --action=clean;') + +CPU_BUILDCMD_BUILD = ('cd /usr/local/toolchain_root/third_party/android_bench/' + 'v2_0/CLOSED_SOURCE/%s; python ../../scripts/bench.py ' + '--toolchain=/usr/bin --add_cflags=%s --add_ldflags=%s ' + '--makeopts=%s --action=build') + +# Common initializations +cmd_executer = command_executer.GetCommandExecuter() + + +def Usage(parser, message): + print('ERROR: %s' % message) + parser.print_help() + sys.exit(0) + + +def CreateRunsh(destdir, benchmark): + """Create run.sh script to run benchmark. + + Perflab needs a run.sh that runs the benchmark. + """ + run_cmd = os.path.dirname(os.path.abspath(__file__)) + '/run_benchmarks.py' + contents = '#!/bin/sh\n%s $@ %s\n' % (run_cmd, benchmark) + runshfile = destdir + '/run.sh' + f = open(runshfile, 'w') + f.write(contents) + f.close() + retval = cmd_executer.RunCommand('chmod +x %s' % runshfile) + return retval + + +def CreateBinaryCopy(sourcedir, destdir, copy=None): + """Create links in perflab-bin/destdir/* to sourcedir/*, instead of copies + + Args: + sourcedir: directory from which to copy. + destdir: directory to which to copy. + copy: when none, make soft links to everything under sourcedir, otherwise + copy all to destdir. + TODO: remove this parameter if it's determined that CopyFiles can use + rsync -L. + """ + retval = 0 + # check if sourcedir exists + if not os.path.exists(sourcedir): + logger.GetLogger().LogError('benchmark results %s does not exist.' % + sourcedir) + return 1 + + # Deal with old copies - save off old ones for now. + # Note - if its a link, it doesn't save anything. + if os.path.exists(destdir): + command = 'rm -rf %s.old' % destdir + retval = cmd_executer.RunCommand(command) + if retval != 0: + return retval + command = 'mv %s %s.old' % (destdir, destdir) + retval = cmd_executer.RunCommand(command) + if retval != 0: + return retval + os.makedirs(destdir) + sourcedir = os.path.abspath(sourcedir) + if copy is None: + command = 'ln -s %s/* %s' % (sourcedir, destdir) + else: + command = 'cp -fr %s/* %s' % (sourcedir, destdir) + retval = cmd_executer.RunCommand(command) + return retval + + +def Main(argv): + """Build ChromeOS.""" + # Common initializations + + parser = argparse.ArgumentParser() + parser.add_argument('-c', + '--chromeos_root', + dest='chromeos_root', + help='Target directory for ChromeOS installation.') + parser.add_argument('-t', + '--toolchain_root', + dest='toolchain_root', + help='This is obsolete. Do not use.') + parser.add_argument('-r', + '--third_party', + dest='third_party', + help='The third_party dir containing android ' + 'benchmarks.') + parser.add_argument('-C', + '--clean', + dest='clean', + action='store_true', + default=False, + help='Clean up build.') + parser.add_argument('-B', + '--build', + dest='build', + action='store_true', + default=False, + help='Build benchmark.') + parser.add_argument('-O', + '--only_copy', + dest='only_copy', + action='store_true', + default=False, + help='Only copy to perflab-bin - no builds.') + parser.add_argument('--workdir', + dest='workdir', + default='.', + help='Work directory for perflab outputs.') + parser.add_argument('--clobber_chroot', + dest='clobber_chroot', + action='store_true', + help='Delete the chroot and start fresh', + default=False) + parser.add_argument('--clobber_board', + dest='clobber_board', + action='store_true', + help='Delete the board and start fresh', + default=False) + parser.add_argument('--cflags', + dest='cflags', + default='', + help='CFLAGS for the ChromeOS packages') + parser.add_argument('--cxxflags', + dest='cxxflags', + default='', + help='CXXFLAGS for the ChromeOS packages') + parser.add_argument('--ldflags', + dest='ldflags', + default='', + help='LDFLAGS for the ChromeOS packages') + parser.add_argument('--makeopts', + dest='makeopts', + default='', + help='Make options for the ChromeOS packages') + parser.add_argument('--board', + dest='board', + help='ChromeOS target board, e.g. x86-generic') + # Leftover positional arguments + parser.add_argument('args', nargs='+', help='benchmarks') + + options = parser.parse_args(argv[1:]) + + # validate args + for arg in options.args: + if arg not in KNOWN_BENCHMARKS: + logger.GetLogger().LogFatal('Bad benchmark %s specified' % arg) + + if options.chromeos_root is None: + Usage(parser, '--chromeos_root must be set') + + if options.board is None: + Usage(parser, '--board must be set') + + if options.toolchain_root: + logger.GetLogger().LogWarning('--toolchain_root should not be set') + + options.chromeos_root = os.path.expanduser(options.chromeos_root) + options.workdir = os.path.expanduser(options.workdir) + + retval = 0 + if options.third_party: + third_party = options.third_party + else: + third_party = '%s/../../../third_party' % os.path.dirname(__file__) + third_party = os.path.realpath(third_party) + for arg in options.args: + # CPU benchmarks + if re.match('chromeos/cpu', arg): + comps = re.split('/', arg) + benchname = comps[2] + + tec_options = [] + if third_party: + tec_options.append('--third_party=%s' % third_party) + if options.clean: + retval = cmd_executer.ChrootRunCommand(options.chromeos_root, + CPU_BUILDCMD_CLEAN % benchname, + tec_options=tec_options) + logger.GetLogger().LogErrorIf(retval, + 'clean of benchmark %s failed.' % arg) + if options.build: + retval = cmd_executer.ChrootRunCommand( + options.chromeos_root, + CPU_BUILDCMD_BUILD % (benchname, options.cflags, options.ldflags, + options.makeopts), + tec_options=tec_options) + logger.GetLogger().LogErrorIf(retval, + 'Build of benchmark %s failed.' % arg) + if retval == 0 and (options.build or options.only_copy): + benchdir = ('%s/android_bench/v2_0/CLOSED_SOURCE/%s' % + (third_party, benchname)) + linkdir = '%s/perflab-bin/%s' % (options.workdir, arg) + + # For cpu/*, we need to copy (not symlinks) of all the contents, + # because they are part of the test fixutre. + retval = CreateBinaryCopy(benchdir, linkdir, True) + if retval != 0: + return retval + retval = CreateRunsh(linkdir, arg) + if retval != 0: + return retval + elif re.match('chromeos/startup', arg): + if options.build: + # Clean for chromeos/browser and chromeos/startup is a Nop + # since builds are always from scratch. + build_args = [ + os.path.dirname(os.path.abspath(__file__)) + '/build_chromeos.py', + '--chromeos_root=' + options.chromeos_root, + '--board=' + options.board, '--cflags=' + options.cflags, + '--cxxflags=' + options.cxxflags, '--ldflags=' + options.ldflags, + '--clobber_board' + ] + retval = build_chromeos.Main(build_args) + logger.GetLogger().LogErrorIf(retval, 'Build of ChromeOS failed.') + if retval == 0 and (options.build or options.only_copy): + benchdir = '%s/src/build/images/%s/latest' % (options.chromeos_root, + options.board) + linkdir = '%s/perflab-bin/%s' % (options.workdir, arg) + retval = CreateBinaryCopy(benchdir, linkdir) + if retval != 0: + return retval + CreateRunsh(linkdir, arg) + if retval != 0: + return retval + elif re.match('chromeos/browser', arg): + if options.build: + # For now, re-build os. TBD: Change to call build_browser + build_args = [os.path.dirname(os.path.abspath(__file__)) + + '/build_chrome_browser.py', + '--chromeos_root=' + options.chromeos_root, + '--board=' + options.board, '--cflags=' + options.cflags, + '--cxxflags=' + options.cxxflags, + '--ldflags=' + options.ldflags] + retval = build_chromeos.Main(build_args) + logger.GetLogger().LogErrorIf(retval, 'Build of ChromeOS failed.') + if retval == 0 and (options.build or options.only_copy): + benchdir = '%s/src/build/images/%s/latest' % (options.chromeos_root, + options.board) + linkdir = '%s/perflab-bin/%s' % (options.workdir, arg) + retval = CreateBinaryCopy(benchdir, linkdir) + if retval != 0: + return retval + retval = CreateRunsh(linkdir, arg) + if retval != 0: + return retval + + return 0 + + +if __name__ == '__main__': + sys.exit(Main(sys.argv)) diff --git a/deprecated/compare_benchmarks.py b/deprecated/compare_benchmarks.py new file mode 100755 index 00000000..b6a30cad --- /dev/null +++ b/deprecated/compare_benchmarks.py @@ -0,0 +1,176 @@ +#!/usr/bin/python2 +# +# Copyright 2010 Google Inc. All Rights Reserved. +"""Script to compare ChromeOS benchmarks + +Inputs: + <perflab-output directory 1 - baseline> + <perflab-output directory 2 - results> + --csv - comma separated results + +This script doesn't really know much about benchmarks. It simply looks for +similarly names directories and a results.txt file, and compares +the results and presents it, along with a geometric mean. + +""" + +from __future__ import print_function + +__author__ = 'bjanakiraman@google.com (Bhaskar Janakiraman)' + +import glob +import math +import argparse +import re +import sys + +from cros_utils import command_executer + +BENCHDIRS = ('%s/default/default/*/gcc-4.4.3-glibc-2.11.1-grte-k8-opt/ref/*' + '/results.txt') + +# Common initializations +cmd_executer = command_executer.GetCommandExecuter() + + +def Usage(parser, message): + print('ERROR: %s' % message) + parser.print_help() + sys.exit(0) + + +def GetStats(in_file): + """Return stats from file""" + f = open(in_file, 'r') + pairs = [] + for l in f: + line = l.strip() + # Look for match lines like the following: + # METRIC isolated TotalTime_ms (down, scalar) trial_run_0: ['1524.4'] + # METRIC isolated isolated_walltime (down, scalar) trial_run_0: \ + # ['167.407445192'] + m = re.match(r"METRIC\s+isolated\s+(\S+).*\['(\d+(?:\.\d+)?)'\]", line) + if not m: + continue + metric = m.group(1) + if re.match(r'isolated_walltime', metric): + continue + + value = float(m.group(2)) + pairs.append((metric, value)) + + return dict(pairs) + + +def PrintDash(n): + tmpstr = '' + for _ in range(n): + tmpstr += '-' + print(tmpstr) + + +def PrintHeaderCSV(hdr): + tmpstr = '' + for i in range(len(hdr)): + if tmpstr != '': + tmpstr += ',' + tmpstr += hdr[i] + print(tmpstr) + + +def PrintHeader(hdr): + tot_len = len(hdr) + PrintDash(tot_len * 15) + + tmpstr = '' + for i in range(len(hdr)): + tmpstr += '%15.15s' % hdr[i] + + print(tmpstr) + PrintDash(tot_len * 15) + + +def Main(argv): + """Compare Benchmarks.""" + # Common initializations + + parser = argparse.ArgumentParser() + parser.add_argument('-c', + '--csv', + dest='csv_output', + action='store_true', + default=False, + help='Output in csv form.') + parser.add_argument('args', nargs='+', help='positional arguments: ' + '<baseline-output-dir> <results-output-dir>') + + options = parser.parse_args(argv[1:]) + + # validate args + if len(options.args) != 2: + Usage(parser, 'Needs <baseline output dir> <results output dir>') + + base_dir = options.args[0] + res_dir = options.args[1] + + # find res benchmarks that have results + resbenches_glob = BENCHDIRS % res_dir + resbenches = glob.glob(resbenches_glob) + + basebenches_glob = BENCHDIRS % base_dir + basebenches = glob.glob(basebenches_glob) + + to_compare = [] + for resbench in resbenches: + tmp = resbench.replace(res_dir, base_dir, 1) + if tmp in basebenches: + to_compare.append((resbench, tmp)) + + for (resfile, basefile) in to_compare: + stats = GetStats(resfile) + basestats = GetStats(basefile) + # Print a header + # benchname (remove results.txt), basetime, restime, %speed-up + hdr = [] + benchname = re.split('/', resfile)[-2:-1][0] + benchname = benchname.replace('chromeos__', '', 1) + hdr.append(benchname) + hdr.append('basetime') + hdr.append('restime') + hdr.append('%speed up') + if options.csv_output: + PrintHeaderCSV(hdr) + else: + PrintHeader(hdr) + + # For geomean computations + prod = 1.0 + count = 0 + for key in stats.keys(): + if key in basestats.keys(): + # ignore very small values. + if stats[key] < 0.01: + continue + count = count + 1 + prod = prod * (stats[key] / basestats[key]) + speedup = (basestats[key] - stats[key]) / basestats[key] + speedup = speedup * 100.0 + if options.csv_output: + print('%s,%f,%f,%f' % (key, basestats[key], stats[key], speedup)) + else: + print('%15.15s%15.2f%15.2f%14.2f%%' % (key, basestats[key], + stats[key], speedup)) + + prod = math.exp(1.0 / count * math.log(prod)) + prod = (1.0 - prod) * 100 + if options.csv_output: + print('%s,,,%f' % ('Geomean', prod)) + else: + print('%15.15s%15.15s%15.15s%14.2f%%' % ('Geomean', '', '', prod)) + print('') + return 0 + + +if __name__ == '__main__': + retval = Main(sys.argv) + sys.exit(retval) diff --git a/deprecated/repo_to_repo_files/binutils-master.json.rtr b/deprecated/repo_to_repo_files/binutils-master.json.rtr new file mode 100644 index 00000000..e9baf804 --- /dev/null +++ b/deprecated/repo_to_repo_files/binutils-master.json.rtr @@ -0,0 +1,22 @@ +{ + "input": [ + { + "type": "p4", + "mappings": [ + "//depot2/branches/mobile_toolchain_v15_release_branch/gcctools/google_vendor_src_branch/binutils/binutils-2.21/... ..." + ], + "address": "perforce2:2666" + } + ], + "output": [ + { + "type": "git", + "gerrit": true, + "ignores": [ + ".svn", + ".git" + ], + "address": "https://chromium.googlesource.com/chromiumos/third_party/binutils.git" + } + ] +}
\ No newline at end of file diff --git a/deprecated/repo_to_repo_files/binutils-mobile_toolchain_v16.json.rtr b/deprecated/repo_to_repo_files/binutils-mobile_toolchain_v16.json.rtr new file mode 100644 index 00000000..927a9a84 --- /dev/null +++ b/deprecated/repo_to_repo_files/binutils-mobile_toolchain_v16.json.rtr @@ -0,0 +1,23 @@ +{ + "input": [ + { + "type": "p4", + "mappings": [ + "//depot2/branches/mobile_toolchain_v16_release_branch/gcctools/google_vendor_src_branch/binutils/binutils-2.22/... ..." + ], + "address": "perforce2:2666" + } + ], + "output": [ + { + "gerrit": true, + "type": "git", + "branch": "mobile_toolchain_v16_release_branch", + "ignores": [ + ".svn", + ".git" + ], + "address": "https://chromium.googlesource.com/chromiumos/third_party/binutils.git" + } + ] +}
\ No newline at end of file diff --git a/deprecated/repo_to_repo_files/crosperf.json.rtr b/deprecated/repo_to_repo_files/crosperf.json.rtr new file mode 100644 index 00000000..95c8f624 --- /dev/null +++ b/deprecated/repo_to_repo_files/crosperf.json.rtr @@ -0,0 +1,38 @@ +{ + "input": [ + { + "type": "p4", + "mappings": [ + "//depot2/gcctools/chromeos/v14/crosperf/... crosperf/...", + "//depot2/gcctools/chromeos/v14/utils/__init__.py utils/__init__.py", + "//depot2/gcctools/chromeos/v14/utils/colortrans.py utils/colortrans.py", + "//depot2/gcctools/chromeos/v14/utils/command_executer.py utils/command_executer.py", + "//depot2/gcctools/chromeos/v14/utils/email_sender.py utils/email_sender.py", + "//depot2/gcctools/chromeos/v14/utils/file_utils.py utils/file_utils.py", + "//depot2/gcctools/chromeos/v14/utils/logger.py utils/logger.py", + "//depot2/gcctools/chromeos/v14/utils/misc.py utils/misc.py", + "//depot2/gcctools/chromeos/v14/utils/misc_test.py utils/misc_test.py", + "//depot2/gcctools/chromeos/v14/utils/stats.py utils/stats.py", + "//depot2/gcctools/chromeos/v14/utils/tabulator.py utils/tabulator.py", + "//depot2/gcctools/chromeos/v14/utils/tabulator_test.py utils/tabulator_test.py", + "//depot2/gcctools/chromeos/v14/utils/timeline.py utils/timeline.py", + "//depot2/gcctools/chromeos/v14/utils/timeline_test.py utils/timeline_test.py", + "//depot2/gcctools/chromeos/v14/image_chromeos.py image_chromeos.py", + "//depot2/gcctools/chromeos/v14/lock_machine.py lock_machine.py", + "//depot2/gcctools/chromeos/v14/README README" + ], + "address": "perforce2:2666" + } + ], + "output": [ + { + "type": "git", + "gerrit": true, + "ignores": [ + ".svn", + ".git" + ], + "address": "https://chromium.googlesource.com/chromiumos/third_party/toolchain-utils.git" + } + ] +}
\ No newline at end of file diff --git a/deprecated/repo_to_repo_files/gcc-branches_google_4_7.json.rtr b/deprecated/repo_to_repo_files/gcc-branches_google_4_7.json.rtr new file mode 100644 index 00000000..bc4dadf0 --- /dev/null +++ b/deprecated/repo_to_repo_files/gcc-branches_google_4_7.json.rtr @@ -0,0 +1,22 @@ +{ + "input": [ + { + "type": "svn", + "mappings": [ + "branches/google/gcc-4_7-mobile" + ], + "address": "svn://gcc.gnu.org/svn/gcc" + } + ], + "output": [ + { + "type": "git", + "ignores": [ + ".svn", + ".git" + ], + "branch": "gcc.gnu.org/branches/google/gcc-4_7-mobile", + "address": "https://chromium.googlesource.com/chromiumos/third_party/gcc.git" + } + ] +}
\ No newline at end of file diff --git a/deprecated/repo_to_repo_files/gcc-branches_google_main.json.rtr b/deprecated/repo_to_repo_files/gcc-branches_google_main.json.rtr new file mode 100644 index 00000000..f34063d2 --- /dev/null +++ b/deprecated/repo_to_repo_files/gcc-branches_google_main.json.rtr @@ -0,0 +1,22 @@ +{ + "input": [ + { + "type": "svn", + "mappings": [ + "branches/google/main" + ], + "address": "svn://gcc.gnu.org/svn/gcc" + } + ], + "output": [ + { + "type": "git", + "branch": "gcc.gnu.org/branches/google/main", + "ignores": [ + ".svn", + ".git" + ], + "address": "https://chromium.googlesource.com/chromiumos/third_party/gcc.git" + } + ] +}
\ No newline at end of file diff --git a/deprecated/repo_to_repo_files/gcc-master.json.rtr b/deprecated/repo_to_repo_files/gcc-master.json.rtr new file mode 100644 index 00000000..692ae7c9 --- /dev/null +++ b/deprecated/repo_to_repo_files/gcc-master.json.rtr @@ -0,0 +1,23 @@ +{ + "input": [ + { + "type": "svn", + "mappings": [ + "branches/google/gcc-4_7-mobile" + ], + "address": "svn://gcc.gnu.org/svn/gcc" + } + ], + "output": [ + { + "gerrit": true, + "type": "git", + "ignores": [ + ".svn", + ".git" + ], + "branch": "master", + "address": "https://chromium.googlesource.com/chromiumos/third_party/gcc.git" + } + ] +}
\ No newline at end of file diff --git a/deprecated/repo_to_repo_files/gdb-master.json.rtr b/deprecated/repo_to_repo_files/gdb-master.json.rtr new file mode 100644 index 00000000..b67ec6f7 --- /dev/null +++ b/deprecated/repo_to_repo_files/gdb-master.json.rtr @@ -0,0 +1,20 @@ +{ + "input": [ + { + "type": "git", + "branch": "202befe3cb3ad0eec07eaaf3f0706e42926b6bbb", + "address": "git://sourceware.org/git/gdb.git" + } + ], + "output": [ + { + "type": "git", + "ignores": [ + ".svn", + ".git" + ], + "branch": "chromeos_master", + "address": "https://chromium.googlesource.com/chromiumos/third_party/gdb.git" + } + ] +}
\ No newline at end of file diff --git a/deprecated/repo_to_repo_files/toolchain-utils.json.rtr b/deprecated/repo_to_repo_files/toolchain-utils.json.rtr new file mode 100644 index 00000000..1975d6b5 --- /dev/null +++ b/deprecated/repo_to_repo_files/toolchain-utils.json.rtr @@ -0,0 +1,28 @@ +{ + "input": [ + { + "type": "p4", + "mappings": [ + "//depot2/gcctools/chromeos/v14/crosperf/... crosperf/...", + "//depot2/gcctools/chromeos/v14/utils/... utils/...", + "//depot2/gcctools/chromeos/v14/image_chromeos.py image_chromeos.py", + "//depot2/gcctools/chromeos/v14/lock_machine.py lock_machine.py", + "//depot2/gcctools/chromeos/v14/README README", + "//depot2/gcctools/chromeos/v14/.gitignore .gitignore" + ], + "address": "perforce2:2666" + } + ], + "output": [ + { + "type": "git", + "ignores": [ + ".svn", + ".git", + ".pyc", + "logs" + ], + "address": "https://chrome-internal.googlesource.com/chromeos/toolchain-utils.git" + } + ] +}
\ No newline at end of file diff --git a/deprecated/report_generator.py b/deprecated/report_generator.py new file mode 100755 index 00000000..c94b51c2 --- /dev/null +++ b/deprecated/report_generator.py @@ -0,0 +1,145 @@ +#!/usr/bin/python2 +# +# Copyright 2010 Google Inc. All Rights Reserved. +"""Script to compare a baseline results file to a new results file.""" + +from __future__ import print_function + +__author__ = 'raymes@google.com (Raymes Khoury)' + +import sys +from cros_utils import logger +from cros_utils import html_tools + +PASS = 'pass' +FAIL = 'fail' +NOT_EXECUTED = 'not executed' + + +class ResultsReport(object): + """Class for holding report results.""" + + def __init__(self, report, num_tests_executed, num_passes, num_failures, + num_regressions): + self.report = report + self.num_tests_executed = num_tests_executed + self.num_passes = num_passes + self.num_failures = num_failures + self.num_regressions = num_regressions + + def GetReport(self): + return self.report + + def GetNumExecuted(self): + return self.num_tests_executed + + def GetNumPasses(self): + return self.num_passes + + def GetNumFailures(self): + return self.num_failures + + def GetNumRegressions(self): + return self.num_regressions + + def GetSummary(self): + summary = 'Tests executed: %s\n' % str(self.num_tests_executed) + summary += 'Tests Passing: %s\n' % str(self.num_passes) + summary += 'Tests Failing: %s\n' % str(self.num_failures) + summary += 'Regressions: %s\n' % str(self.num_regressions) + return summary + + +def Usage(): + print('Usage: %s baseline_results new_results' % sys.argv[0]) + sys.exit(1) + + +def ParseResults(results_filename): + results = [] + try: + results_file = open(results_filename, 'rb') + for line in results_file: + if line.strip() != '': + results.append(line.strip().split('\t')) + results_file.close() + except IOError: + logger.GetLogger().LogWarning('Could not open results file: ' + + results_filename) + return results + + +def GenerateResultsReport(baseline_file, new_result_file): + baseline_results = ParseResults(baseline_file) + new_results = ParseResults(new_result_file) + + test_status = {} + + for new_result in new_results: + new_test_name = new_result[0] + new_test_result = new_result[1] + test_status[new_test_name] = (new_test_result, NOT_EXECUTED) + + for baseline_result in baseline_results: + baseline_test_name = baseline_result[0] + baseline_test_result = baseline_result[1] + if baseline_test_name in test_status: + new_test_result = test_status[baseline_test_name][0] + test_status[baseline_test_name] = (new_test_result, baseline_test_result) + else: + test_status[baseline_test_name] = (NOT_EXECUTED, baseline_test_result) + + regressions = [] + for result in test_status.keys(): + if test_status[result][0] != test_status[result][1]: + regressions.append(result) + + num_tests_executed = len(new_results) + num_regressions = len(regressions) + num_passes = 0 + num_failures = 0 + for result in new_results: + if result[1] == PASS: + num_passes += 1 + else: + num_failures += 1 + + report = html_tools.GetPageHeader('Test Summary') + report += html_tools.GetHeader('Test Summary') + report += html_tools.GetListHeader() + report += html_tools.GetListItem('Tests executed: ' + str(num_tests_executed)) + report += html_tools.GetListItem('Passes: ' + str(num_passes)) + report += html_tools.GetListItem('Failures: ' + str(num_failures)) + report += html_tools.GetListItem('Regressions: ' + str(num_regressions)) + report += html_tools.GetListFooter() + report += html_tools.GetHeader('Regressions', 2) + report += html_tools.GetTableHeader(['Test name', 'Expected result', + 'Actual result']) + + for regression in regressions: + report += html_tools.GetTableRow([regression[:150], test_status[regression][ + 1], test_status[regression][0]]) + report += '\n' + report += html_tools.GetTableFooter() + report += html_tools.GetHeader('All Tests', 2) + report += html_tools.GetTableHeader(['Test name', 'Expected result', + 'Actual result']) + for result in test_status.keys(): + report += html_tools.GetTableRow([result[:150], test_status[result][1], + test_status[result][0]]) + report += '\n' + report += html_tools.GetTableFooter() + report += html_tools.GetFooter() + return ResultsReport(report, num_tests_executed, num_passes, num_failures, + num_regressions) + + +def Main(argv): + if len(argv) < 2: + Usage() + + print(GenerateResultsReport(argv[1], argv[2])[0]) + + +if __name__ == '__main__': + Main(sys.argv) diff --git a/deprecated/run_benchmarks.py b/deprecated/run_benchmarks.py new file mode 100755 index 00000000..4818750a --- /dev/null +++ b/deprecated/run_benchmarks.py @@ -0,0 +1,245 @@ +#!/usr/bin/python2 +# +# Copyright 2010 Google Inc. All Rights Reserved. +"""Script to run ChromeOS benchmarks + +Inputs: + chromeos_root + board + [chromeos/cpu/<benchname>| + chromeos/browser/[pagecycler|sunspider]| + chromeos/startup] + hostname/IP of Chromeos machine + + chromeos/cpu/<benchname> + - Read run script rules from bench.mk perflab-bin, copy benchmark to + host, run + and return results. + + chromeos/startup + - Re-image host with image in perflab-bin + - Call run_tests to run startup test, gather results. + - Restore host back to what it was. + + chromeos/browser/* + - Call build_chromebrowser to build image with new browser + - Copy image to perflab-bin + +""" + +from __future__ import print_function + +__author__ = 'bjanakiraman@google.com (Bhaskar Janakiraman)' + +import argparse +import os +import re +import sys + +import image_chromeos +import run_tests +from cros_utils import command_executer +from cros_utils import logger + +# pylint: disable=anomalous-backslash-in-string + +KNOWN_BENCHMARKS = [ + 'chromeos/startup', 'chromeos/browser/pagecycler', + 'chromeos/browser/sunspider', 'chromeos/browser/v8bench', + 'chromeos/cpu/bikjmp' +] + +name_map = { + 'pagecycler': 'Page', + 'sunspider': 'SunSpider', + 'v8bench': 'V8Bench', + 'startup': 'BootPerfServer' +} + +# Run command template + +# Common initializations +cmd_executer = command_executer.GetCommandExecuter() + + +def Usage(parser, message): + print('ERROR: %s' % message) + parser.print_help() + sys.exit(0) + + +def RunBrowserBenchmark(chromeos_root, board, bench, machine): + """Run browser benchmarks. + + Args: + chromeos_root: ChromeOS src dir + board: Board being tested + bench: Name of benchmark (chromeos/browser/*) + machine: name of chromeos machine + """ + benchname = re.split('/', bench)[2] + benchname = name_map[benchname] + ret = run_tests.RunRemoteTests(chromeos_root, machine, board, benchname) + return ret + + +def RunStartupBenchmark(chromeos_root, board, machine): + """Run browser benchmarks. + + Args: + chromeos_root: ChromeOS src dir + board: Board being tested + machine: name of chromeos machine + """ + benchname = 'startup' + benchname = name_map[benchname] + ret = run_tests.RunRemoteTests(chromeos_root, machine, board, benchname) + return ret + + +def RunCpuBenchmark(chromeos_root, bench, workdir, machine): + """Run CPU benchmark. + + Args: + chromeos_root: ChromeOS src dir + bench: Name of benchmark + workdir: directory containing benchmark directory + machine: name of chromeos machine + + Returns: + status: 0 on success + """ + + benchname = re.split('/', bench)[2] + benchdir = '%s/%s' % (workdir, benchname) + + # Delete any existing run directories on machine. + # Since this has exclusive access to the machine, + # we do not worry about duplicates. + args = 'rm -rf /tmp/%s' % benchname + retv = cmd_executer.CrosRunCommand(args, + chromeos_root=chromeos_root, + machine=machine) + if retv: + return retv + + # Copy benchmark directory. + retv = cmd_executer.CopyFiles(benchdir, + '/tmp/' + benchname, + chromeos_root=chromeos_root, + dest_machine=machine, + dest_cros=True) + if retv: + return retv + + # Parse bench.mk to extract run flags. + + benchmk_file = open('%s/bench.mk' % benchdir, 'r') + for line in benchmk_file: + line.rstrip() + if re.match('^run_cmd', line): + line = re.sub('^run_cmd.*\${PERFLAB_PATH}', './out', line) + line = re.sub('\${PERFLAB_INPUT}', './data', line) + run_cmd = line + break + + # Execute on remote machine + # Capture output and process it. + sshargs = "'cd /tmp/%s;" % benchname + sshargs += "time -p %s'" % run_cmd + cmd_executer.CrosRunCommand(sshargs, + chromeos_root=chromeos_root, + machine=machine) + + return retv + + +def Main(argv): + """Build ChromeOS.""" + # Common initializations + + parser = argparse.ArgumentParser() + parser.add_argument('-c', + '--chromeos_root', + dest='chromeos_root', + help='Target directory for ChromeOS installation.') + parser.add_argument('-m', + '--machine', + dest='machine', + help='The chromeos host machine.') + parser.add_argument('--workdir', + dest='workdir', + default='./perflab-bin', + help='Work directory for perflab outputs.') + parser.add_argument('--board', + dest='board', + help='ChromeOS target board, e.g. x86-generic') + parser.add_argumen('args', margs='+', help='Benchmarks to run.') + + options = parser.parse_args(argv[1:]) + + # validate args + for arg in options.args: + if arg not in KNOWN_BENCHMARKS: + logger.GetLogger().LogFatal('Bad benchmark %s specified' % arg) + + if options.chromeos_root is None: + Usage(parser, '--chromeos_root must be set') + + if options.board is None: + Usage(parser, '--board must be set') + + if options.machine is None: + Usage(parser, '--machine must be set') + + found_err = 0 + retv = 0 + for arg in options.args: + # CPU benchmarks + comps = re.split('/', arg) + if re.match('chromeos/cpu', arg): + benchname = comps[2] + print('RUNNING %s' % benchname) + retv = RunCpuBenchmark(options.chromeos_root, arg, options.workdir, + options.machine) + if not found_err: + found_err = retv + elif re.match('chromeos/startup', arg): + benchname = comps[1] + image_args = [ + os.path.dirname(os.path.abspath(__file__)) + '/image_chromeos.py', + '--chromeos_root=' + options.chromeos_root, + '--remote=' + options.machine, '--image=' + options.workdir + '/' + + benchname + '/chromiumos_image.bin' + ] + logger.GetLogger().LogOutput('Reimaging machine %s' % options.machine) + image_chromeos.Main(image_args) + + logger.GetLogger().LogOutput('Running %s' % arg) + retv = RunStartupBenchmark(options.chromeos_root, options.board, + options.machine) + if not found_err: + found_err = retv + elif re.match('chromeos/browser', arg): + benchname = comps[2] + image_args = [ + os.path.dirname(os.path.abspath(__file__)) + '/image_chromeos.py', + '--chromeos_root=' + options.chromeos_root, + '--remote=' + options.machine, '--image=' + options.workdir + '/' + + benchname + '/chromiumos_image.bin' + ] + logger.GetLogger().LogOutput('Reimaging machine %s' % options.machine) + image_chromeos.Main(image_args) + + logger.GetLogger().LogOutput('Running %s' % arg) + retv = RunBrowserBenchmark(options.chromeos_root, options.board, arg, + options.machine) + if not found_err: + found_err = retv + + return found_err + + +if __name__ == '__main__': + retval = Main(sys.argv) + sys.exit(retval) diff --git a/deprecated/sheriff_rotation.py b/deprecated/sheriff_rotation.py new file mode 100755 index 00000000..f6873077 --- /dev/null +++ b/deprecated/sheriff_rotation.py @@ -0,0 +1,130 @@ +#!/usr/bin/python2 +# +# Copyright 2010 Google Inc. All Rights Reserved. +"""Script to rotate the weekly team sheriff. + +This script determines who the next sheriff is, updates the file +appropriately and sends out email notifying the team. +""" + +from __future__ import print_function + +__author__ = 'asharif@google.com (Ahmad Sharif)' + +import argparse +import datetime +import os +import sys + +from cros_utils import constants +from cros_utils import email_sender + + +class SheriffHandler(object): + """Main class for handling sheriff rotations.""" + + SHERIFF_FILE = os.path.join(constants.CROSTC_WORKSPACE, 'sheriffs.txt') + SUBJECT = 'You (%s) are the sheriff for the week: %s - %s' + BODY = ('Please see instructions here: ' + 'https://sites.google.com/a/google.com/chromeos-toolchain-team-home2' + '/home/sheriff-s-corner/sheriff-duties') + + def GetWeekInfo(self, day=datetime.datetime.today()): + """Return week_start, week_end.""" + + epoch = datetime.datetime.utcfromtimestamp(0) + delta_since_epoch = day - epoch + + abs_days = abs(delta_since_epoch.days) - 2 # To get it to start from Sat. + day_of_week = abs_days % 7 + + week_begin = day - datetime.timedelta(days=day_of_week) + week_end = day + datetime.timedelta(days=(6 - day_of_week)) + + strftime_format = '%A, %B %d %Y' + + return (week_begin.strftime(strftime_format), + week_end.strftime(strftime_format)) + + def GetCurrentSheriff(self): + """Return the current sheriff.""" + return self.ReadSheriffsAsList()[0] + + def ReadSheriffsAsList(self): + """Return the sheriff file contents.""" + contents = '' + with open(self.SHERIFF_FILE, 'r') as f: + contents = f.read() + return contents.splitlines() + + def WriteSheriffsAsList(self, to_write): + with open(self.SHERIFF_FILE, 'w') as f: + f.write('\n'.join(to_write)) + + def GetRotatedSheriffs(self, num_rotations=1): + """Return the sheriff file contents.""" + sheriff_list = self.ReadSheriffsAsList() + + new_sheriff_list = [] + num_rotations = num_rotations % len(sheriff_list) + new_sheriff_list = ( + sheriff_list[num_rotations:] + sheriff_list[:num_rotations]) + return new_sheriff_list + + def Email(self): + es = email_sender.EmailSender() + current_sheriff = self.GetCurrentSheriff() + week_start, week_end = self.GetWeekInfo() + subject = self.SUBJECT % (current_sheriff, week_start, week_end) + es.SendEmail([current_sheriff], + subject, + self.BODY, + email_from=os.path.basename(__file__), + email_cc=['c-compiler-chrome']) + + +def Main(argv): + parser = argparse.ArgumentParser() + parser.add_argument('-e', + '--email', + dest='email', + action='store_true', + help='Email the sheriff.') + parser.add_argument('-r', + '--rotate', + dest='rotate', + help='Print sheriffs after n rotations.') + parser.add_argument('-w', + '--write', + dest='write', + action='store_true', + default=False, + help='Wrote rotated contents to the sheriff file.') + + options = parser.parse_args(argv) + + sheriff_handler = SheriffHandler() + + current_sheriff = sheriff_handler.GetCurrentSheriff() + week_start, week_end = sheriff_handler.GetWeekInfo() + + print('Current sheriff: %s (%s - %s)' % (current_sheriff, week_start, + week_end)) + + if options.email: + sheriff_handler.Email() + + if options.rotate: + rotated_sheriffs = sheriff_handler.GetRotatedSheriffs(int(options.rotate)) + print('Rotated sheriffs (after %s rotations)' % options.rotate) + print('\n'.join(rotated_sheriffs)) + if options.write: + sheriff_handler.WriteSheriffsAsList(rotated_sheriffs) + print('Rotated sheriffs written to file.') + + return 0 + + +if __name__ == '__main__': + retval = Main(sys.argv[1:]) + sys.exit(retval) diff --git a/deprecated/summarize_results.py b/deprecated/summarize_results.py new file mode 100755 index 00000000..67d7e9a4 --- /dev/null +++ b/deprecated/summarize_results.py @@ -0,0 +1,145 @@ +#!/usr/bin/python2 +# +# Copyright 2010 Google Inc. All Rights Reserved. +"""Script to summarize the results of various log files.""" + +from __future__ import print_function + +__author__ = 'raymes@google.com (Raymes Khoury)' + +from cros_utils import command_executer +import os +import sys +import re + +RESULTS_DIR = 'results' +RESULTS_FILE = RESULTS_DIR + '/results.csv' + +# pylint: disable=anomalous-backslash-in-string + +class DejaGNUSummarizer(object): + """DejaGNU Summarizer Class""" + + def __int__(self): + pass + + def Matches(self, log_file): + for log_line in log_file: + if log_line.find("""tests ===""") > -1: + return True + return False + + def Summarize(self, log_file, filename): + result = '' + pass_statuses = ['PASS', 'XPASS'] + fail_statuses = ['FAIL', 'XFAIL', 'UNSUPPORTED'] + name_count = {} + for line in log_file: + line = line.strip().split(':') + if len(line) > 1 and (line[0] in pass_statuses or + line[0] in fail_statuses): + test_name = (':'.join(line[1:])).replace('\t', ' ').strip() + count = name_count.get(test_name, 0) + 1 + name_count[test_name] = count + test_name = '%s (%s)' % (test_name, str(count)) + if line[0] in pass_statuses: + test_result = 'pass' + else: + test_result = 'fail' + result += '%s\t%s\t%s\n' % (test_name, test_result, filename) + return result + + +class PerflabSummarizer(object): + """Perflab Summarizer class""" + + def __init__(self): + pass + + def Matches(self, log_file): + p = re.compile('METRIC isolated \w+') + for log_line in log_file: + if p.search(log_line): + return True + return False + + def Summarize(self, log_file, filename): + result = '' + p = re.compile("METRIC isolated (\w+) .*\['(.*?)'\]") + log_file_lines = '\n'.join(log_file) + matches = p.findall(log_file_lines) + for match in matches: + if len(match) != 2: + continue + result += '%s\t%s\t%s\n' % (match[0], match[1], filename) + return result + + +class AutoTestSummarizer(object): + """AutoTest Summarizer class""" + + def __init__(self): + pass + + def Matches(self, log_file): + for log_line in log_file: + if log_line.find("""Installing autotest on""") > -1: + return True + return False + + def Summarize(self, log_file, filename): + result = '' + pass_statuses = ['PASS'] + fail_statuses = ['FAIL'] + for line in log_file: + line = line.strip().split(' ') + if len(line) > 1 and (line[-1].strip() in pass_statuses or + line[-1].strip() in fail_statuses): + test_name = (line[0].strip()) + if line[-1].strip() in pass_statuses: + test_result = 'pass' + else: + test_result = 'fail' + result += '%s\t%s\t%s\n' % (test_name, test_result, filename) + return result + + +def Usage(): + print('Usage: %s log_file' % sys.argv[0]) + sys.exit(1) + + +def SummarizeFile(filename): + summarizers = [DejaGNUSummarizer(), AutoTestSummarizer(), PerflabSummarizer()] + inp = open(filename, 'rb') + executer = command_executer.GetCommandExecuter() + for summarizer in summarizers: + inp.seek(0) + if summarizer.Matches(inp): + executer.CopyFiles(filename, RESULTS_DIR, recursive=False) + inp.seek(0) + result = summarizer.Summarize(inp, os.path.basename(filename)) + inp.close() + return result + inp.close() + return None + + +def Main(argv): + if len(argv) != 2: + Usage() + filename = argv[1] + + executer = command_executer.GetCommandExecuter() + executer.RunCommand('mkdir -p %s' % RESULTS_DIR) + summary = SummarizeFile(filename) + if summary is not None: + output = open(RESULTS_FILE, 'a') + output.write(summary.strip() + '\n') + output.close() + return 0 + + +if __name__ == '__main__': + retval = Main(sys.argv) + sys.exit(retval) |