diff options
author | Artem Kotsiuba <artem.kotsiuba@linaro.org> | 2021-06-30 15:19:00 +0100 |
---|---|---|
committer | Artem Kotsiuba <artem.kotsiuba@linaro.org> | 2021-09-15 12:43:35 +0100 |
commit | 6c186d961882448fe0cda42476e4619323b88274 (patch) | |
tree | 4c030c5fee422fb5bd1113f8a4ce1891d11ee1cf | |
parent | 7455e3bc48366cd8d0ebcb16a53561e5d6a668cd (diff) | |
download | art-testing-6c186d961882448fe0cda42476e4619323b88274.tar.gz |
ART: Replace old script (run.py) to get compilation metrics
(time and size) with new compilation_stats.py script
compilation_stats.py is a new Python script that compiles APK to oat
using chroot mechanism (similar to other benchmark scripts) and
gets metrics like compilation time and executable size. The old script
did not use chroot mechanism and used dex2oat tool located on the device
for compilation which did not take into account local changes to ART.
The new script iteates over all specified APKs, compiles them and
saves compilation statistics to the file.
This patch also changes the way compilation statistics is
collected for performance benchmarks. Compilation duration
is computed based on the difference between the two timestamps
emitted before and after compilation.
Also added more utility functions used by compile statistics
functionality for JSON reading and output parsing.
Test: ./scripts/benchmarks/compilation_stats_target.sh
./benchmarks/apks/another.music.player_5870.apk --iterations 1
Change-Id: Ic35194cd22198fc75cb2b321d5c96b4805735d1e
-rwxr-xr-x | compilation_stats.py | 101 | ||||
-rwxr-xr-x | run.py | 5 | ||||
-rwxr-xr-x | tools/benchmarks/run.py | 45 | ||||
-rwxr-xr-x | tools/compilation_statistics/run.py | 315 | ||||
-rw-r--r-- | tools/utils.py | 43 |
5 files changed, 179 insertions, 330 deletions
diff --git a/compilation_stats.py b/compilation_stats.py new file mode 100755 index 0000000..78dea5f --- /dev/null +++ b/compilation_stats.py @@ -0,0 +1,101 @@ +#! /usr/bin/env python3 + +# Copyright (C) 2021 Linaro Limited. All rights received. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import sys +import json +import tempfile +import shutil + +from collections import OrderedDict + +dir_benchs = os.path.dirname(os.path.realpath(__file__)) +dir_tools = os.path.join(dir_benchs, '..') +sys.path.insert(0, dir_tools) + +from tools import utils, utils_adb, utils_print, utils_stats + +def BuildOptions(): + parser = argparse.ArgumentParser( + description = "Collect compilation statistics.", + # Print default values. + formatter_class = argparse.ArgumentDefaultsHelpFormatter) + utils.AddCommonRunOptions(parser) + utils.AddOutputFormatOptions(parser, utils.default_output_formats) + args = parser.parse_args() + return args + +def SaveAndPrintResults(apk, + compilation_times, + section_sizes, + output_json_filename): + output_obj = utils.ReadJSON(output_json_filename) + + apk_basename = os.path.basename(apk) + output_obj[apk_basename] = dict() + + output_obj[apk_basename].update(compilation_times) + print("Compilation times (seconds):") + utils.PrintData(compilation_times) + + output_obj[apk_basename]["Executable size"] = section_sizes + print("Executable sizes (bytes):") + utils.PrintData(section_sizes) + + with open(output_json_filename, "w") as fp: + json.dump(output_obj, fp, indent=2) + +if __name__ == "__main__": + # create temp directory to pull executables from the device + # to measure their size + work_dir = tempfile.mkdtemp() + try: + args = BuildOptions() + apk = args.add_pathname[0] + apk_name = utils.TargetPathJoin(args.target_copy_path, apk) + # command is used to call a shell script using chroot + # this script calls dex2oat on a given APK and prints + # before/after timestamps + # after command is executed we pull the executable from the device + # and measure its size + if 'ART_COMMAND' in os.environ: + command = os.getenv('ART_COMMAND') + else: + utils.Error("ART_COMMAND is not set.") + format_data = {'workdir': os.path.dirname(apk_name)} + command = command.format(**format_data) + + compilation_times = [] + for i in range(args.iterations): + print("Compiling APK") + results = utils_adb.shell(command, args.target, exit_on_error=False) + lines = results[1] + compilation_time = utils.ExtractCompilationTimeFromOutput(lines) + compilation_times += [compilation_time] + print("Compilation took {:.2f}s\n".format(compilation_time)) + + # Pull the executable and get its size + local_oat = os.path.join(work_dir, apk + '.oat') + utils_adb.pull(args.output_oat, local_oat, args.target) + section_sizes = utils.GetSectionSizes(local_oat) + + compile_time_dict = OrderedDict([("Time", compilation_times)]) + SaveAndPrintResults(apk, compile_time_dict, section_sizes, args.output_json) + + finally: + shutil.rmtree(work_dir) + @@ -23,7 +23,6 @@ from collections import OrderedDict from tools import utils from tools import utils_stats from tools.benchmarks.run import GetAndPrintBenchmarkResults -from tools.compilation_statistics.run import GetAndPrintCompilationStatisticsResults def BuildOptions(): parser = argparse.ArgumentParser( @@ -52,9 +51,5 @@ if __name__ == "__main__": result = OrderedDict() result[utils.benchmarks_label] = GetAndPrintBenchmarkResults(args) - if args.target: - result[utils.compilation_statistics_label] = \ - GetAndPrintCompilationStatisticsResults(args) - utils.OutputObject(result, 'pkl', args.output_pkl) utils.OutputObject(result, 'json', args.output_json) diff --git a/tools/benchmarks/run.py b/tools/benchmarks/run.py index edfb3a6..844d484 100755 --- a/tools/benchmarks/run.py +++ b/tools/benchmarks/run.py @@ -18,9 +18,11 @@ import argparse import csv import os +import shutil import subprocess import sys import time +import tempfile from collections import OrderedDict @@ -34,8 +36,6 @@ import utils_stats bench_runner_main = 'org.linaro.bench.RunBench' -# Options - def BuildOptions(): parser = argparse.ArgumentParser( description = "Run java benchmarks.", @@ -160,13 +160,18 @@ def RunBench(apk, classname, try: for line in outerr.rstrip().splitlines(): - if not line.startswith('benchmarks/'): - continue - name = line.split(":")[0].rstrip() - score = float(line.split(":")[1].strip().split(" ")[0].strip()) - if name not in result: - result[name] = list() - result[name].append(score) + if line.startswith('benchmarks/'): + name = line.split(":")[0].rstrip() + score = float(line.split(":")[1].strip().split(" ")[0].strip()) + if name not in result: + result[name] = list() + result[name].append(score) + duration = utils.ExtractCompilationTimeFromOutput(outerr) + if duration > 0: + if utils.compilation_times_label not in result: + result[utils.compilation_times_label] = list() + result[utils.compilation_times_label].append(duration) + except Exception as e: utils.Warning(str(e) + "\n \-> Error parsing output from %s", e) rc += 1 @@ -178,6 +183,7 @@ def RunBench(apk, classname, def RunBenchs(apk, bench_names, target, + output_oat, auto_calibrate, iterations=utils.default_n_iterations, mode=utils.default_mode, @@ -198,6 +204,16 @@ def RunBenchs(apk, bench_names, android_root = android_root, target = target, cpuset = cpuset) + if output_oat is not None: + try: + work_dir = tempfile.mkdtemp() + local_oat = os.path.join(work_dir, 'bench.oat') + utils_adb.pull(output_oat, local_oat, target) + section_sizes = utils.GetSectionSizes(local_oat) + result[utils.compilation_statistics_label] = section_sizes + finally: + shutil.rmtree(work_dir) + return rc @@ -253,6 +269,7 @@ def GetBenchmarkResults(args): rc = RunBenchs(remote_apk, benchmarks, args.target, + args.output_oat, not args.no_auto_calibrate, args.iterations, args.mode, @@ -269,7 +286,15 @@ def GetBenchmarkResults(args): def GetAndPrintBenchmarkResults(args): results = GetBenchmarkResults(args) utils.PrintData(results) - unflattened_results = utils.Unflatten(results) + + # remove compilation statistics from the results, it was + # already printed + filtered_results = dict() + for (k,v) in results.items(): + if utils.compilation_statistics_label not in k: + filtered_results[k] = v + + unflattened_results = utils.Unflatten(filtered_results) utils_stats.ComputeAndPrintGeomeanWithRelativeDiff(unflattened_results) print('') return results diff --git a/tools/compilation_statistics/run.py b/tools/compilation_statistics/run.py deleted file mode 100755 index 23fa91b..0000000 --- a/tools/compilation_statistics/run.py +++ /dev/null @@ -1,315 +0,0 @@ -#! /usr/bin/env python3 - -# Copyright (C) 2015 Linaro Limited. All rights received. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import glob -import json -import os -import pickle -import re -import shutil -import subprocess -import sys -import tempfile -import time - -from collections import OrderedDict - -dir_compilation_statistics = os.path.dirname(os.path.realpath(__file__)) -dir_tools = os.path.join(dir_compilation_statistics, '..') -sys.path.insert(0, dir_tools) - -import utils -import utils_adb -import utils_stats - -memory_unit_prefixes = {'' : 1, 'G' : 2 ** 30, 'K' : 2 ** 10, 'M' : 2 ** 20} -sections = set(['.bss', '.rodata', '.text', 'Total']) - -def BuildOptions(): - parser = argparse.ArgumentParser( - description = '''Collect statistics about the APK compilation process on a target - adb device: Compilation time, memory usage by the compiler - (arena, Java, and native allocations, and free native memory), - and size of the generated executable (total, .bss, .rodata, and - .text section sizes).''', - # Print default values. - formatter_class = argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('pathnames', - nargs = '+', - help='''Path containing APK files or a file name for which - compilation statistics should be collected.''') - utils.AddCommonRunOptions(parser) - utils.AddOutputFormatOptions(parser, utils.default_output_formats) - - # TODO: Support running on host? - # For now override the default value for the `--target`. - parser.set_defaults(target=utils.adb_default_target_string) - - args = parser.parse_args() - - # This cannot fire for now since this script always runs on target, but - # eventually we may want to run on host as well. - utils.ValidateCommonRunOptions(args) - - return args - -def GetStats(apk, - target, - isa, - compiler_mode, - android_root, - target_copy_path, - iterations, - cpuset, - work_dir, - boot_oat_file): - path, env, runtime_param = utils.GetAndroidRootConfiguration(android_root, isa.endswith('64')) - dex2oat = utils.TargetPathJoin(path, 'dex2oat') - - if boot_oat_file: - oat = utils.TargetPathJoin(target_copy_path, 'boot.' + isa + '.oat') - art = utils.TargetPathJoin(target_copy_path, 'boot.' + isa + '.art') - - # Check if dump file exists. - dump_oat_file_location = utils.TargetPathJoin(target_copy_path, 'boot.oat.' + isa + '.txt') - dump_exists_command = "if [ -f %s ] ; then echo found; fi; exit 0" \ - % (dump_oat_file_location) - # Since we are interested in whether the dump file exists or not, we can't simply execute - # [ -f file_name ] since newer versions of adb return the error code of the command that's - # being executed. Therefore, if the file is found we output a string and at the end we - # always return error code 0, so that we can get an error only if the failure was due - # to adb not executing properly. - rc, out = utils_adb.shell(dump_exists_command, target) - # The command prints an extra new line as well. - if out.strip() != "found": - # Dump the oat file the first time, keeping only the parts we are interested in. - dump_command = 'oatdump --oat-file=%s | grep "dex2oat-" > %s' % (boot_oat_file, \ - dump_oat_file_location) - utils_adb.shell(dump_command, target) - # Read dex2oat-host from dump file. - dex2oat_host_command = 'grep "dex2oat-host" %s' % (dump_oat_file_location) - rc, out = utils_adb.shell(dex2oat_host_command, target) - if rc: - utils.Error("Dump file doesn't contain dex2oat-host.") - if out.strip() == 'dex2oat-host = x86-64': - utils.Error("boot.oat was built on a x84-64 machine, which is most likely the" \ - " host: %s \nWe want it to be built on the target instead. Have you" \ - " configured the device with WITH_DEXPREOPT=false ?" % out) - - # Read command. - dex2oat_cmdline_command = 'grep "dex2oat-cmdline" %s' % (dump_oat_file_location) - rc, out = utils_adb.shell(dex2oat_cmdline_command, target) - if rc: - utils.Error("Dump file doesn't contain dex2oat-cmldine.") - command = out.strip() - # Replace destination: --oat-file, fix beginning of command. - command = re.sub("--oat-file=(.+?) --", "--oat-file=%s --" % oat, command) - command = re.sub("--image=(.+?) --", "--image=%s --" % art, command) - command = re.sub("dex2oat-cmdline +=", dex2oat, command) - # Force 1 thread only - we want compilation times to be as stable as possible and we are - # interested in single thread performance, not multi-thread (throughput). - command = re.sub(" -j\d+ ", " -j1 ", command) - # Remove newline at end. - command = re.sub("\n$", "", command) - command = '(echo $BASHPID && ' - - if cpuset: - command += 'echo $BASHPID > /dev/cpuset/' + cpuset + '/tasks && ' - - command += env + ' exec ' + command + ') | head -n1' - else: - runtime_arguments = ' --runtime-arg -Xnorelocate ' - - for param in runtime_param: - runtime_arguments += '--runtime-arg ' + param + ' ' - - apk_path = utils.TargetPathJoin(target_copy_path, apk) - oat = apk_path + '.' + isa + '.oat' - dex2oat_options = utils.GetDex2oatOptions(compiler_mode) - # Only the output of the first command is necessary; execute in a subshell - # to guarantee PID value; only one thread is used for compilation to reduce - # measurement noise. - command = '(echo $BASHPID && ' - - if cpuset: - command += 'echo $BASHPID > /dev/cpuset/' + cpuset + '/tasks && ' - - command += env + ' exec ' + dex2oat + \ - ' -j1' + runtime_arguments + ' '.join(dex2oat_options) + \ - ' --dex-file=' + apk_path + ' --oat-file=' + oat - command += ' --instruction-set=' + isa + ') | head -n1' - - linux_target = os.getenv('ART_TARGET_LINUX', 'false') == 'true' - dex2oat_time_regex = '.*?took (?P<value>.*?)(?P<unit>[mnu]{,1})s.*?\)' - compilation_times = [] - for i in range(iterations): - rc, stdout = utils_adb.shell(command, target) - if linux_target: - # On Linux, dex2oat writes to stdout, and output of compilation time is likely last - for out in reversed(stdout.splitlines()): - compile_time = re.match(dex2oat_time_regex, out) - if compile_time: - break - else: - # To simplify parsing, assume that PID values are rarely recycled by the system. - stats_command = 'logcat -dsv process dex2oat | grep "^I([[:space:]]*' + \ - stdout.rstrip() + ').*took" | tail -n1' - rc, out = utils_adb.shell(stats_command, target) - compile_time = re.match(dex2oat_time_regex, out) - - if not compile_time: - utils.Error('dex2oat failed; check adb logcat.') - - value = float(compile_time.group('value')) * \ - utils.si_unit_prefixes[compile_time.group('unit')] - compilation_times.append(value) - - # The rest of the statistics are deterministic, so there is no need to run several - # iterations; just get the values from the last run. - out = out[compile_time.end():] - # Newer versions of dex2oat also have number of threads output, that we need to get rid of - out = re.sub('\(threads:\s+[0-9]+\) ', '', out) - memory_stats = OrderedDict() - byte_size = True - - for m in re.findall(' (.*?)=([0-9]+)([GKM]?)B( \(([0-9]+)B\))?', out): - # Old versions of dex2oat do not show the exact memory usage values in bytes, so - # try to parse the output in the new format first, and if that fails, fall back - # to the legacy one. - if m[4]: - value = int(m[4]) - else: - value = int(m[1]) * memory_unit_prefixes[m[2]] - - if m[2]: - byte_size = False - - memory_stats[m[0]] = [value] - - if not byte_size: - utils.Warning('Memory usage values have been rounded down, so they might be ' - 'inaccurate.') - - if boot_oat_file: - local_oat = os.path.join(utils.dir_root, work_dir, "boot.%s.oat" % isa) - else: - local_oat = os.path.join(utils.dir_root, work_dir, apk + '.oat') - utils_adb.pull(oat, local_oat, target) - command = ['size', '-A', '-d', local_oat] - rc, outerr = utils.Command(command) - section_sizes = OrderedDict((s[0], [int(s[1])]) for s - in re.findall('(\S+)\s+([0-9]+).*', outerr) - if s[0] in sections) - return OrderedDict([(utils.compilation_times_label, compilation_times), - (utils.memory_stats_label, memory_stats), - (utils.oat_size_label, section_sizes)]) - - -def GetCompilationStatisticsResults(args): - utils.CheckDependencies(['adb', 'size']) - isa = utils_adb.GetISA(args.target, args.mode) - res = OrderedDict() - work_dir = tempfile.mkdtemp() - apk_list = set() - boot_oat_file = None - - for pathname in args.pathnames: - if pathname == "boot.oat": - # Check if multiple boot.oat parameters have been passed. - if boot_oat_file: - continue - - # Get ISA list to check that the environment is in a good state. - isa_list = utils_adb.GetISAList(args.target) - # The oat cache is accessible only to root. - utils_adb.root(args.target) - # Find oat file on device. - find_command = 'find / -type d \( -name proc -o -name sys \) -prune -o ' \ - '-name "*boot.oat" -print 2>/dev/null' - rc, out = utils_adb.shell(find_command, args.target) - boot_oat_files = out.splitlines()[:-1] - - if len(boot_oat_files) != len(isa_list): - utils.Error("Number of architectures different from number of boot.oat files. " \ - "The list of boot.oat files is here:\n\n %s\n\nMake sure there are " \ - "no stale boot.oat files in %s or some other directory. " \ - "Another possibility is that you didn't build Android with " \ - "`WITH_DEXPREOPT=false`. Do a `lunch` and then `WITH_DEXPREOPT=false " \ - "make -j$(nproc)`." % (boot_oat_files, args.target_copy_path)) - # Order both lists. Now, as long as both oat files have the same parent dir, order - # should match. - isa_list.sort() - boot_oat_files.sort() - # Remove leading dot and trailing whitespace. - boot_oat_file = boot_oat_files[isa_list.index(isa)][1:].strip() - apk_list.add("boot.oat " + isa) - elif os.path.isfile(pathname): - apk_list.add(pathname) - else: - dentries = [dentry for dentry in glob.glob(os.path.join(pathname, '*.apk')) - if os.path.isfile(dentry)] - - for d in dentries: - apk_list.add(d) - - for apk in sorted(apk_list): - # pathname just contains boot.oat. - if apk[:8] == "boot.oat": - res[apk] = GetStats(apk, args.target, isa, args.compiler_mode, args.android_root, - args.target_copy_path, args.iterations, args.cpuset, work_dir, - boot_oat_file) - # This is a local path for boot.oat. We get stats locally without compiling on target. - elif apk[-8:] == "boot.oat": - res["boot.oat"] = GetLocalOatSizeStats(apk) - else: - utils_adb.push(apk, args.target_copy_path, args.target) - apk_name = os.path.basename(apk) - res[apk_name] = GetStats(apk_name, args.target, isa, args.compiler_mode, - args.android_root, args.target_copy_path, - args.iterations, args.cpuset, work_dir, None) - - shutil.rmtree(work_dir) - return res - -def GetAndPrintCompilationStatisticsResults(args): - results = GetCompilationStatisticsResults(args) - utils.PrintData(results) - print('') - return results - -def GetLocalOatSizeStats(oat_path): - command = ['size', '-A', '-d', oat_path] - rc, outerr = utils.Command(command) - section_sizes = OrderedDict((s[0], [int(s[1])]) for s - in re.findall('(\S+)\s+([0-9]+).*', outerr) - if s[0] in sections) - # Add total file size in bytes. - command = ['stat', '-c', '%s', oat_path] - rc, outerr = utils.Command(command) - section_sizes['FileSize'] = [int(outerr)] - return OrderedDict([(utils.oat_size_label, section_sizes)]) - -if __name__ == "__main__": - # TODO: Mac OS support - if os.uname().sysname != 'Linux': - utils.Error('Running this script is supported only on Linux.') - - args = BuildOptions() - stats = GetAndPrintCompilationStatisticsResults(args) - - utils.OutputObject(stats, 'pkl', args.output_pkl) - utils.OutputObject(stats, 'json', args.output_json) diff --git a/tools/utils.py b/tools/utils.py index 6a0ab5f..b2e5e3d 100644 --- a/tools/utils.py +++ b/tools/utils.py @@ -18,6 +18,7 @@ import fnmatch import json import os import pickle +import re import subprocess import sys import time @@ -55,6 +56,13 @@ default_mode = '' default_compiler_mode = None default_n_iterations = 1 +# used in shell scripts for compile stats +before_timestamp_key = "Before: " +after_timestamp_key = "After: " + +sections = set(['.bss', '.rodata', '.text', 'Total']) + + # TODO: Use python's logging and warning capabilities instead! def Info(message): print('INFO: ' + message) @@ -181,6 +189,9 @@ def AddCommonRunOptions(parser): nargs='+', default = None, help = 'Add pathnames to be considered for compilation statistics.') + opts.add_argument('--output-oat', + default=None, + help = 'Full name of the compiled executable file') def ValidateCommonRunOptions(args): options_requiring_target_mode = ['mode', 'compiler-mode'] @@ -469,3 +480,35 @@ def TargetPathJoin(path, *paths): path = path.replace(os.sep, '/') return path + +def ReadJSON(json_filename): + output_obj = dict() + try: + with open(json_filename, "r") as fp: + output_obj = json.load(fp) + except IOError: + pass + return output_obj + +def ExtractCompilationTimeFromOutput(lines): + timestamp_before = 0 + timestamp_after = 0 + # Extracting the size and timestamps (before and after compilation) + # cmdline.sh script is expected to print lines with + # two timestamps: before (before_timestamp_key) + # and after (after_timestamp_key) compilation + for line in lines.rstrip().splitlines(): + if (line.startswith(after_timestamp_key)): + timestamp_after = float(line[len(after_timestamp_key):]) + if (line.startswith(before_timestamp_key)): + timestamp_before = float(line[len(before_timestamp_key):]) + return timestamp_after - timestamp_before + +def GetSectionSizes(path_to_executable): + command = ['size', '-A', '-d', path_to_executable] + rc, outerr = Command(command) + section_sizes = OrderedDict( + (section_line[0], [int(section_line[1])]) for section_line + in re.findall('(\S+)\s+([0-9]+).*', outerr) + if section_line[0] in sections) + return section_sizes |