From b3bd95d9a5f4c30969c85f70f6acf08fcf312281 Mon Sep 17 00:00:00 2001 From: Alexandre Rames Date: Mon, 25 Apr 2016 15:17:48 +0100 Subject: Improve the filter option. - When no filters match, retry with the original filters prefixed and suffixed with `*`. This allows not specifying wildcards for simple runs. For example `./tools/benchmarks/run.py --filter Sort` would not run anything. - Do not crash while computing statistics if there are no results. - Add support for filtering to the top-level `compare.py` script. Change-Id: I7d279f6ecaf9dfdd5c76abba035da2315578ff2f --- compare.py | 4 ++ test/test.py | 3 +- tools/benchmarks/compare.py | 28 +---------- tools/benchmarks/run.py | 29 +++++------- tools/compilation_statistics/run.py | 2 +- tools/utils.py | 94 +++++++++++++++++++++++++++++++++---- tools/utils_stats.py | 3 ++ 7 files changed, 109 insertions(+), 54 deletions(-) diff --git a/compare.py b/compare.py index 2b8b2bf..017b033 100755 --- a/compare.py +++ b/compare.py @@ -73,4 +73,8 @@ if __name__ == "__main__": res_2 = json.load(file_2, object_pairs_hook=OrderedDict) file_1.close() file_2.close() + + res_1 = utils.Filter(res_1, args.filter, args.filter_out) + res_2 = utils.Filter(res_2, args.filter, args.filter_out) + PrintDiff(res_1, res_2) diff --git a/test/test.py b/test/test.py index e87c2c7..1694872 100755 --- a/test/test.py +++ b/test/test.py @@ -24,7 +24,7 @@ import sys dir_test = os.path.dirname(os.path.realpath(__file__)) dir_root = os.path.realpath(os.path.join(dir_test, '..')) dir_tools = os.path.join(dir_root,'tools') -sys.path.insert(0, dir_tools) +sys.path.append(dir_tools) import lint import utils @@ -141,6 +141,7 @@ def TestTopLevelWrapperScripts(): rc |= TestCommand(["./run.py", "--output-json=/tmp/res1"], _cwd=utils.dir_root) rc |= TestCommand(["./run.py", "--output-json=/tmp/res2"], _cwd=utils.dir_root) rc |= TestCommand(["./compare.py", "/tmp/res1", "/tmp/res2"], _cwd=utils.dir_root) + rc |= TestCommand(["./compare.py", "--filter", "benchmarks", "/tmp/res1", "/tmp/res2"], _cwd=utils.dir_root) return rc diff --git a/tools/benchmarks/compare.py b/tools/benchmarks/compare.py index 05dfa9f..d73468c 100755 --- a/tools/benchmarks/compare.py +++ b/tools/benchmarks/compare.py @@ -55,11 +55,6 @@ def BuildOptions(): help = '''Results with a deviation higher than this threshold (in %%) will be included in the significant results even if the difference threshold is not met.''') - parser.add_argument('-f', '--filter', action = 'append', - help='Quoted (benchmark name) filter pattern.') - parser.add_argument('-F', '--filter-out', action = 'append', - help='''Filter out the benchmarks matching this patern - from the results.''') return parser.parse_args() @@ -114,33 +109,14 @@ def OrderResultsByDifference(in_1, in_2): return (regressions_1, regressions_2), (improvements_1, improvements_2) -def FilterBenchmarks(benchmarks, filters, filters_out): - # We cannot use dictionary comprehension because need to preserve the order - # of keys. - res = benchmarks - if filters: - tmp = OrderedDict({}) - for b in res: - if utils.NameMatchesAnyFilter(b, filters): - tmp[b] = benchmarks[b] - res = tmp - if filters_out: - tmp = OrderedDict({}) - for b in res: - if not utils.NameMatchesAnyFilter(b, filters_out): - tmp[b] = res[b] - res = tmp - return res - - if __name__ == "__main__": args = BuildOptions() file_1 = open(args.res_1, 'r') file_2 = open(args.res_2, 'r') res_1 = json.load(file_1, object_pairs_hook=OrderedDict) res_2 = json.load(file_2, object_pairs_hook=OrderedDict) - res_1 = FilterBenchmarks(res_1, args.filter, args.filter_out) - res_2 = FilterBenchmarks(res_2, args.filter, args.filter_out) + res_1 = utils.Filter(res_1, args.filter, args.filter_out) + res_2 = utils.Filter(res_2, args.filter, args.filter_out) if args.significant_changes: res_1, res_2 = \ diff --git a/tools/benchmarks/run.py b/tools/benchmarks/run.py index b5d7256..acb5bd2 100755 --- a/tools/benchmarks/run.py +++ b/tools/benchmarks/run.py @@ -45,17 +45,20 @@ def BuildOptions(): utils.AddOutputFormatOptions(parser, utils.default_output_formats + ['csv']) parser.add_argument('--dont-auto-calibrate', action='store_true', default = False, - dest = 'no_auto_calibrate', help='''Do not auto-calibrate - the benchmarks. Instead, run each benchmark's `main()` - function directly.''') + dest = 'no_auto_calibrate', + help='''Do not auto-calibrate the benchmarks. Instead, + run each benchmark's `main()` function directly.''') parser.add_argument('-n', '--norun', action='store_true', - help='Build and configure everything, but do not run the benchmarks.') + help='''Build and configure everything, but do not run + the benchmarks.''') parser.add_argument('-f', '--filter', action = 'append', - help='Quoted (benchmark name) filter pattern.') + help='''Quoted (benchmark name) filter pattern. If no + filters match, filtering will be attempted with all the + patterns prefixed and suffixed with `*`.''') parser.add_argument('-F', '--filter-out', action = 'append', - help='''Filter out the benchmarks matching this patern. - Defaults to \'benchmarks/deprecated/*\' if no other filter is - specified.''') + help='''Filter out the benchmarks matching this pattern. + Defaults to \'benchmarks/deprecated/*\' if no other + filter is specified.''') args = parser.parse_args() @@ -200,14 +203,6 @@ def ListAllBenchmarks(): return benchs -def FilterBenchmarks(benchmarks, filters, filters_out): - res = benchmarks - if filters: - res = [b for b in res if utils.NameMatchesAnyFilter(b, filters)] - if filters_out: - res = [b for b in res if not utils.NameMatchesAnyFilter(b, filters_out)] - return res - def GetBenchmarkResults(args): if getattr(args, 'filter', []) == []: setattr(args, 'filter', None) @@ -245,7 +240,7 @@ def GetBenchmarkResults(args): filter_out = args.filter_out else: filter_out = ['benchmarks/deprecated/*'] - benchmarks = FilterBenchmarks(benchmarks, args.filter, filter_out) + benchmarks = utils.FilterList(benchmarks, args.filter, filter_out) rc = RunBenchs(remote_apk, benchmarks, diff --git a/tools/compilation_statistics/run.py b/tools/compilation_statistics/run.py index eb100b0..5c3d9ff 100755 --- a/tools/compilation_statistics/run.py +++ b/tools/compilation_statistics/run.py @@ -133,7 +133,7 @@ def GetStats(apk, # Only the output of the first command is necessary; execute in a subshell # to guarantee PID value; only one thread is used for compilation to reduce # measurement noise. - command = '(echo $BASHPID && exec dex2oat -j1 ' + \ + command = '(echo $BASHPID && exec dex2oat -j1 --runtime-arg -Xnorelocate' + \ ' '.join(dex2oat_options) + \ ' --dex-file=' + apk_path + ' --oat-file=' + oat command += ' --instruction-set=' + isa + ') | head -n1' diff --git a/tools/utils.py b/tools/utils.py index e5b2ddc..0c105f7 100644 --- a/tools/utils.py +++ b/tools/utils.py @@ -54,6 +54,9 @@ default_mode = '' default_compiler_mode = None default_n_iterations = 1 +# TODO: Use python's logging and warning capabilities instead! +def Info(message): + print('INFO: ' + message) def Warning(message, exc=None): print(utils_print.COLOUR_ORANGE + 'WARNING: ' + message, file=sys.stderr) @@ -102,15 +105,6 @@ def PrettySIFactor(value): return si_factor, si_prefix -def NameMatchesAnyFilter(name, filters): - # Ensure we have a list of filters. This lets the function work if only one - # filter is passed as a string. - filters = list(filters) - for f in filters: - if fnmatch.fnmatch(name, f): - return True - return False - # Wrapper around `subprocess.Popen` returning the output of the given command. def Command(command, command_string=None, exit_on_error=True, cwd=None): if not command_string: @@ -241,6 +235,14 @@ def AddOutputFormatOptions(parser, formats=default_output_formats): def AddCommonCompareOptions(parser): parser.add_argument('res_1', metavar = 'res_1.pkl') parser.add_argument('res_2', metavar = 'res_2.pkl') + parser.add_argument('-f', '--filter', action = 'append', + help='''Quoted (benchmark name) filter pattern. If no + filters match, filtering will be attempted with all the + patterns prefixed and suffixed with `*`.''') + parser.add_argument('-F', '--filter-out', action = 'append', + help='''Filter out the benchmarks matching this pattern + from the results. Filters failing are **not** retried + with added wildcards.''') def CheckDependencies(dependencies): for d in dependencies: @@ -267,3 +269,77 @@ def PrintData(data, key=None, indentation=''): print('') elif isinstance(data, list): return [key] + list(utils_stats.ComputeStats(data)) + + +def NameMatchesAnyFilter(name, filters): + assert(isinstance(name, str)) + if filters is None: + return False + # Ensure we have a list of filters. This lets the function work if only one + # filter is passed as a string. + filters = list(filters) + for f in filters: + if fnmatch.fnmatch(name, f): + return True + return False + + +def FilterListHelper(data, filters, negative_filter=False): + assert(isinstance(data, list)) + return [x for x in data \ + if NameMatchesAnyFilter(x, filters) != negative_filter] + + +def FilterList(data, filters, filters_out): + assert(isinstance(data, list)) + + res = data + + if filters: + res = FilterListHelper(data, filters) + if not res: + # Try again with all patterns prefixed and suffixed with `*`. + extended_filters = list(map(lambda f: '*' + f + '*', filters)) + Info('The filters ' + str(filters) + ' did not match any ' + \ + 'data. Retrying with ' + str(extended_filters) + '.') + res = FilterListHelper(data, extended_filters) + if filters_out: + res = FilterListHelper(res, filters_out, negative_filter=True) + + return res + + +def FilterHelper(data, filters, negative_filter=False): + if (not isinstance(data, dict) and not isinstance(data, OrderedDict)): + return data if negative_filter else None + res = OrderedDict() + for key in data: + name_matches_any_filter = NameMatchesAnyFilter(key, filters) + if not name_matches_any_filter: + # Filter the sub-data and keep it if it is not empty. + subres = FilterHelper(data[key], filters, negative_filter) + if subres: + res[key] = subres + elif not negative_filter: + res[key] = data[key] + return res + + +def Filter(data, filters, filters_out): + if not isinstance(data, dict) and not isinstance(data, OrderedDict): + return data + + res = data + + if filters: + res = FilterHelper(data, filters) + if not res: + # Try again with all patterns prefixed and suffixed with `*`. + extended_filters = list(map(lambda f: '*' + f + '*', filters)) + Info('The filters ' + str(filters) + ' did not match any ' + \ + 'data. Retrying with ' + str(extended_filters) + '.') + res = FilterHelper(data, extended_filters) + if filters_out: + res = FilterHelper(res, filters_out, negative_filter=True) + + return res diff --git a/tools/utils_stats.py b/tools/utils_stats.py index 86e9596..398f316 100644 --- a/tools/utils_stats.py +++ b/tools/utils_stats.py @@ -59,6 +59,7 @@ def GetSuiteName(benchmark): return benchmark.split("/", 2)[1] def ComputeGeomean(dict_results): + if not dict_results: return stats_dict = {} for benchmark in dict_results: @@ -95,6 +96,7 @@ def ComputeGeomean(dict_results): return results def ComputeAndPrintGeomean(dict_results): + if not dict_results: return results = ComputeGeomean(dict_results) print("GEOMEANS:") headers = ['suite', 'geomean', 'error', 'error (% of geomean)'] @@ -105,6 +107,7 @@ def PrintDiff(res_1, res_2, title = ''): # Only print results for benchmarks present in both sets of results. # Pay attention to maintain the order of the keys. benchmarks = [b for b in res_1.keys() if b in res_2.keys()] + if not benchmarks: return headers = [title] + stats_diff_headers results = [] stats_dict = {} -- cgit v1.2.3