Improve the filter option.

- When no filters match, retry with the original filters prefixed and
suffixed with `*`. This allows not specifying wildcards for simple
runs. For example `./tools/benchmarks/run.py --filter Sort` would not run
anything.
- Do not crash while computing statistics if there are no results.
- Add support for filtering to the top-level `compare.py` script.

Change-Id: I7d279f6ecaf9dfdd5c76abba035da2315578ff2f

7 files changed, 109 insertions, 54 deletions

diff --git a/compare.py b/compare.py
index 2b8b2bf..017b033 100755
--- a/compare.py
+++ b/compare.py
@@ -73,4 +73,8 @@ if __name__ == "__main__":
     res_2 = json.load(file_2, object_pairs_hook=OrderedDict)
     file_1.close()
     file_2.close()
+
+    res_1 = utils.Filter(res_1, args.filter, args.filter_out)
+    res_2 = utils.Filter(res_2, args.filter, args.filter_out)
+
     PrintDiff(res_1, res_2)
diff --git a/test/test.py b/test/test.py
index e87c2c7..1694872 100755
--- a/test/test.py
+++ b/test/test.py
@@ -24,7 +24,7 @@ import sys
 dir_test = os.path.dirname(os.path.realpath(__file__))
 dir_root = os.path.realpath(os.path.join(dir_test, '..'))
 dir_tools = os.path.join(dir_root,'tools')
-sys.path.insert(0, dir_tools)
+sys.path.append(dir_tools)
 import lint
 import utils
 
@@ -141,6 +141,7 @@ def TestTopLevelWrapperScripts():
     rc |= TestCommand(["./run.py", "--output-json=/tmp/res1"], _cwd=utils.dir_root)
     rc |= TestCommand(["./run.py", "--output-json=/tmp/res2"], _cwd=utils.dir_root)
     rc |= TestCommand(["./compare.py", "/tmp/res1", "/tmp/res2"], _cwd=utils.dir_root)
+    rc |= TestCommand(["./compare.py", "--filter", "benchmarks", "/tmp/res1", "/tmp/res2"], _cwd=utils.dir_root)
     return rc
 
 
diff --git a/tools/benchmarks/compare.py b/tools/benchmarks/compare.py
index 05dfa9f..d73468c 100755
--- a/tools/benchmarks/compare.py
+++ b/tools/benchmarks/compare.py
@@ -55,11 +55,6 @@ def BuildOptions():
                         help = '''Results with a deviation higher than this
                         threshold (in %%) will be included in the significant
                         results even if the difference threshold is not met.''')
-    parser.add_argument('-f', '--filter', action = 'append',
-                        help='Quoted (benchmark name) filter pattern.')
-    parser.add_argument('-F', '--filter-out', action = 'append',
-                        help='''Filter out the benchmarks matching this patern
-                        from the results.''')
     return parser.parse_args()
 
 
@@ -114,33 +109,14 @@ def OrderResultsByDifference(in_1, in_2):
     return (regressions_1, regressions_2), (improvements_1, improvements_2)
 
 
-def FilterBenchmarks(benchmarks, filters, filters_out):
-    # We cannot use dictionary comprehension because need to preserve the order
-    # of keys.
-    res = benchmarks
-    if filters:
-        tmp = OrderedDict({})
-        for b in res:
-            if utils.NameMatchesAnyFilter(b, filters):
-                tmp[b] = benchmarks[b]
-        res = tmp
-    if filters_out:
-        tmp = OrderedDict({})
-        for b in res:
-            if not utils.NameMatchesAnyFilter(b, filters_out):
-                tmp[b] = res[b]
-        res = tmp
-    return res
-
-
 if __name__ == "__main__":
     args = BuildOptions()
     file_1 = open(args.res_1, 'r')
     file_2 = open(args.res_2, 'r')
     res_1 = json.load(file_1, object_pairs_hook=OrderedDict)
     res_2 = json.load(file_2, object_pairs_hook=OrderedDict)
-    res_1 = FilterBenchmarks(res_1, args.filter, args.filter_out)
-    res_2 = FilterBenchmarks(res_2, args.filter, args.filter_out)
+    res_1 = utils.Filter(res_1, args.filter, args.filter_out)
+    res_2 = utils.Filter(res_2, args.filter, args.filter_out)
 
     if args.significant_changes:
         res_1, res_2 = \
diff --git a/tools/benchmarks/run.py b/tools/benchmarks/run.py
index b5d7256..acb5bd2 100755
--- a/tools/benchmarks/run.py
+++ b/tools/benchmarks/run.py
@@ -45,17 +45,20 @@ def BuildOptions():
     utils.AddOutputFormatOptions(parser, utils.default_output_formats + ['csv'])
     parser.add_argument('--dont-auto-calibrate',
                         action='store_true', default = False,
-                        dest = 'no_auto_calibrate', help='''Do not auto-calibrate
-                        the benchmarks. Instead, run each benchmark's `main()`
-                        function directly.''')
+                        dest = 'no_auto_calibrate',
+                        help='''Do not auto-calibrate the benchmarks. Instead,
+                        run each benchmark's `main()` function directly.''')
     parser.add_argument('-n', '--norun', action='store_true',
-                        help='Build and configure everything, but do not run the benchmarks.')
+                        help='''Build and configure everything, but do not run
+                        the benchmarks.''')
     parser.add_argument('-f', '--filter', action = 'append',
-                        help='Quoted (benchmark name) filter pattern.')
+                        help='''Quoted (benchmark name) filter pattern. If no
+                        filters match, filtering will be attempted with all the
+                        patterns prefixed and suffixed with `*`.''')
     parser.add_argument('-F', '--filter-out', action = 'append',
-                        help='''Filter out the benchmarks matching this patern.
-                             Defaults to \'benchmarks/deprecated/*\' if no other filter is
-                             specified.''')
+                        help='''Filter out the benchmarks matching this pattern.
+                        Defaults to \'benchmarks/deprecated/*\' if no other
+                        filter is specified.''')
 
     args = parser.parse_args()
 
@@ -200,14 +203,6 @@ def ListAllBenchmarks():
     return benchs
 
 
-def FilterBenchmarks(benchmarks, filters, filters_out):
-    res = benchmarks
-    if filters:
-        res = [b for b in res if utils.NameMatchesAnyFilter(b, filters)]
-    if filters_out:
-        res = [b for b in res if not utils.NameMatchesAnyFilter(b, filters_out)]
-    return res
-
 def GetBenchmarkResults(args):
     if getattr(args, 'filter', []) == []:
         setattr(args, 'filter', None)
@@ -245,7 +240,7 @@ def GetBenchmarkResults(args):
         filter_out = args.filter_out
     else:
         filter_out = ['benchmarks/deprecated/*']
-    benchmarks = FilterBenchmarks(benchmarks, args.filter, filter_out)
+    benchmarks = utils.FilterList(benchmarks, args.filter, filter_out)
 
     rc = RunBenchs(remote_apk,
                    benchmarks,
diff --git a/tools/compilation_statistics/run.py b/tools/compilation_statistics/run.py
index eb100b0..5c3d9ff 100755
--- a/tools/compilation_statistics/run.py
+++ b/tools/compilation_statistics/run.py
@@ -133,7 +133,7 @@ def GetStats(apk,
         # Only the output of the first command is necessary; execute in a subshell
         # to guarantee PID value; only one thread is used for compilation to reduce
         # measurement noise.
-        command = '(echo $BASHPID && exec dex2oat -j1 ' + \
+        command = '(echo $BASHPID && exec dex2oat -j1 --runtime-arg -Xnorelocate' + \
                   ' '.join(dex2oat_options) + \
                   ' --dex-file=' + apk_path + ' --oat-file=' + oat
         command += ' --instruction-set=' + isa + ') | head -n1'
diff --git a/tools/utils.py b/tools/utils.py
index e5b2ddc..0c105f7 100644
--- a/tools/utils.py
+++ b/tools/utils.py
@@ -54,6 +54,9 @@ default_mode = ''
 default_compiler_mode = None
 default_n_iterations = 1
 
+# TODO: Use python's logging and warning capabilities instead!
+def Info(message):
+    print('INFO: ' + message)
 
 def Warning(message, exc=None):
     print(utils_print.COLOUR_ORANGE + 'WARNING: ' + message, file=sys.stderr)
@@ -102,15 +105,6 @@ def PrettySIFactor(value):
 
     return si_factor, si_prefix
 
-def NameMatchesAnyFilter(name, filters):
-    # Ensure we have a list of filters. This lets the function work if only one
-    # filter is passed as a string.
-    filters = list(filters)
-    for f in filters:
-        if fnmatch.fnmatch(name, f):
-            return True
-    return False
-
 # Wrapper around `subprocess.Popen` returning the output of the given command.
 def Command(command, command_string=None, exit_on_error=True, cwd=None):
     if not command_string:
@@ -241,6 +235,14 @@ def AddOutputFormatOptions(parser, formats=default_output_formats):
 def AddCommonCompareOptions(parser):
     parser.add_argument('res_1', metavar = 'res_1.pkl')
     parser.add_argument('res_2', metavar = 'res_2.pkl')
+    parser.add_argument('-f', '--filter', action = 'append',
+                        help='''Quoted (benchmark name) filter pattern. If no
+                        filters match, filtering will be attempted with all the
+                        patterns prefixed and suffixed with `*`.''')
+    parser.add_argument('-F', '--filter-out', action = 'append',
+                        help='''Filter out the benchmarks matching this pattern
+                        from the results. Filters failing are **not** retried
+                        with added wildcards.''')
 
 def CheckDependencies(dependencies):
     for d in dependencies:
@@ -267,3 +269,77 @@ def PrintData(data, key=None, indentation=''):
             print('')
     elif isinstance(data, list):
         return [key] + list(utils_stats.ComputeStats(data))
+
+
+def NameMatchesAnyFilter(name, filters):
+    assert(isinstance(name, str))
+    if filters is None:
+        return False
+    # Ensure we have a list of filters. This lets the function work if only one
+    # filter is passed as a string.
+    filters = list(filters)
+    for f in filters:
+        if fnmatch.fnmatch(name, f):
+            return True
+    return False
+
+
+def FilterListHelper(data, filters, negative_filter=False):
+    assert(isinstance(data, list))
+    return [x for x in data \
+            if NameMatchesAnyFilter(x, filters) != negative_filter]
+
+
+def FilterList(data, filters, filters_out):
+    assert(isinstance(data, list))
+
+    res = data
+
+    if filters:
+        res = FilterListHelper(data, filters)
+        if not res:
+            # Try again with all patterns prefixed and suffixed with `*`.
+            extended_filters = list(map(lambda f: '*' + f + '*', filters))
+            Info('The filters ' + str(filters) + ' did not match any ' + \
+                 'data. Retrying with ' + str(extended_filters) + '.')
+            res = FilterListHelper(data, extended_filters)
+    if filters_out:
+        res = FilterListHelper(res, filters_out, negative_filter=True)
+
+    return res
+
+
+def FilterHelper(data, filters, negative_filter=False):
+    if (not isinstance(data, dict) and not isinstance(data, OrderedDict)):
+        return data if negative_filter else None
+    res = OrderedDict()
+    for key in data:
+        name_matches_any_filter = NameMatchesAnyFilter(key, filters)
+        if not name_matches_any_filter:
+            # Filter the sub-data and keep it if it is not empty.
+            subres = FilterHelper(data[key], filters, negative_filter)
+            if subres:
+                res[key] = subres
+        elif not negative_filter:
+            res[key] = data[key]
+    return res
+
+
+def Filter(data, filters, filters_out):
+    if not isinstance(data, dict) and not isinstance(data, OrderedDict):
+        return data
+
+    res = data
+
+    if filters:
+        res = FilterHelper(data, filters)
+        if not res:
+            # Try again with all patterns prefixed and suffixed with `*`.
+            extended_filters = list(map(lambda f: '*' + f + '*', filters))
+            Info('The filters ' + str(filters) + ' did not match any ' + \
+                 'data. Retrying with ' + str(extended_filters) + '.')
+            res = FilterHelper(data, extended_filters)
+    if filters_out:
+        res = FilterHelper(res, filters_out, negative_filter=True)
+
+    return res
diff --git a/tools/utils_stats.py b/tools/utils_stats.py
index 86e9596..398f316 100644
--- a/tools/utils_stats.py
+++ b/tools/utils_stats.py
@@ -59,6 +59,7 @@ def GetSuiteName(benchmark):
     return benchmark.split("/", 2)[1]
 
 def ComputeGeomean(dict_results):
+    if not dict_results: return
     stats_dict = {}
 
     for benchmark in dict_results:
@@ -95,6 +96,7 @@ def ComputeGeomean(dict_results):
     return results
 
 def ComputeAndPrintGeomean(dict_results):
+    if not dict_results: return
     results = ComputeGeomean(dict_results)
     print("GEOMEANS:")
     headers = ['suite', 'geomean', 'error', 'error (% of geomean)']
@@ -105,6 +107,7 @@ def PrintDiff(res_1, res_2, title = ''):
     # Only print results for benchmarks present in both sets of results.
     # Pay attention to maintain the order of the keys.
     benchmarks = [b for b in res_1.keys() if b in res_2.keys()]
+    if not benchmarks: return
     headers = [title] + stats_diff_headers
     results = []
     stats_dict = {}