diff options
Diffstat (limited to 'afdo_redaction')
| -rwxr-xr-x | afdo_redaction/redact_profile.py | 306 | ||||
| -rwxr-xr-x | afdo_redaction/redact_profile_test.py | 225 | ||||
| -rwxr-xr-x | afdo_redaction/remove_cold_functions.py | 293 | ||||
| -rwxr-xr-x | afdo_redaction/remove_cold_functions_test.py | 162 | ||||
| -rwxr-xr-x | afdo_redaction/remove_indirect_calls.py | 87 | ||||
| -rwxr-xr-x | afdo_redaction/remove_indirect_calls_test.py | 33 |
6 files changed, 582 insertions, 524 deletions
diff --git a/afdo_redaction/redact_profile.py b/afdo_redaction/redact_profile.py index 02bae928..0779d2ac 100755 --- a/afdo_redaction/redact_profile.py +++ b/afdo_redaction/redact_profile.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2018 The Chromium OS Authors. All rights reserved. +# Copyright 2018 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -24,7 +24,6 @@ It reads a textual AFDO profile from stdin, and prints a 'fixed' version of it to stdout. A summary of what the script actually did is printed to stderr. """ -from __future__ import division, print_function import collections import re @@ -32,23 +31,23 @@ import sys def _count_samples(samples): - """Count the total number of samples in a function.""" - line_re = re.compile(r'^(\s*)\d+(?:\.\d+)?: (\d+)\s*$') + """Count the total number of samples in a function.""" + line_re = re.compile(r"^(\s*)\d+(?:\.\d+)?: (\d+)\s*$") - top_level_samples = 0 - all_samples = 0 - for line in samples: - m = line_re.match(line) - if not m: - continue + top_level_samples = 0 + all_samples = 0 + for line in samples: + m = line_re.match(line) + if not m: + continue - spaces, n = m.groups() - n = int(n) - all_samples += n - if len(spaces) == 1: - top_level_samples += n + spaces, n = m.groups() + n = int(n) + all_samples += n + if len(spaces) == 1: + top_level_samples += n - return top_level_samples, all_samples + return top_level_samples, all_samples # A ProfileRecord is a set of samples for a top-level symbol in a textual AFDO @@ -80,70 +79,75 @@ def _count_samples(samples): # And samples look like one of: # arbitrary_number: sample_count # arbitrary_number: inlined_function_symbol:inlined_entry_count -ProfileRecord = collections.namedtuple('ProfileRecord', - ['function_line', 'samples']) +ProfileRecord = collections.namedtuple( + "ProfileRecord", ["function_line", "samples"] +) def _normalize_samples(samples): - """Normalizes the samples in the given function body. - - Normalization just means that we redact inlined function names. This is - done so that a bit of templating doesn't make two function bodies look - distinct. Namely: - - template <typename T> - __attribute__((noinline)) - int getNumber() { return 1; } - - template <typename T> - __attribute__((noinline)) - int getNumberIndirectly() { return getNumber<T>(); } - - int main() { - return getNumber<int>() + getNumber<float>(); - } - - If the profile has the mangled name for getNumber<float> in - getNumberIndirectly<float> (and similar for <int>), we'll consider them to - be distinct when they're not. - """ - - # I'm not actually sure if this ends up being an issue in practice, but it's - # simple enough to guard against. - inlined_re = re.compile(r'(^\s*\d+): [^:]+:(\s*\d+)\s*$') - result = [] - for s in samples: - m = inlined_re.match(s) - if m: - result.append('%s: __REDACTED__:%s' % m.groups()) - else: - result.append(s) - return tuple(result) + """Normalizes the samples in the given function body. + + Normalization just means that we redact inlined function names. This is + done so that a bit of templating doesn't make two function bodies look + distinct. Namely: + + template <typename T> + __attribute__((noinline)) + int getNumber() { return 1; } + + template <typename T> + __attribute__((noinline)) + int getNumberIndirectly() { return getNumber<T>(); } + + int main() { + return getNumber<int>() + getNumber<float>(); + } + + If the profile has the mangled name for getNumber<float> in + getNumberIndirectly<float> (and similar for <int>), we'll consider them to + be distinct when they're not. + """ + + # I'm not actually sure if this ends up being an issue in practice, but it's + # simple enough to guard against. + inlined_re = re.compile(r"(^\s*\d+): [^:]+:(\s*\d+)\s*$") + result = [] + for s in samples: + m = inlined_re.match(s) + if m: + result.append("%s: __REDACTED__:%s" % m.groups()) + else: + result.append(s) + return tuple(result) def _read_textual_afdo_profile(stream): - """Parses an AFDO profile from a line stream into ProfileRecords.""" - # ProfileRecords are actually nested, due to inlining. For the purpose of - # this script, that doesn't matter. - lines = (line.rstrip() for line in stream) - function_line = None - samples = [] - for line in lines: - if not line: - continue - - if line[0].isspace(): - assert function_line is not None, 'sample exists outside of a function?' - samples.append(line) - continue - - if function_line is not None: - yield ProfileRecord(function_line=function_line, samples=tuple(samples)) - function_line = line + """Parses an AFDO profile from a line stream into ProfileRecords.""" + # ProfileRecords are actually nested, due to inlining. For the purpose of + # this script, that doesn't matter. + lines = (line.rstrip() for line in stream) + function_line = None samples = [] + for line in lines: + if not line: + continue + + if line[0].isspace(): + assert ( + function_line is not None + ), "sample exists outside of a function?" + samples.append(line) + continue + + if function_line is not None: + yield ProfileRecord( + function_line=function_line, samples=tuple(samples) + ) + function_line = line + samples = [] - if function_line is not None: - yield ProfileRecord(function_line=function_line, samples=tuple(samples)) + if function_line is not None: + yield ProfileRecord(function_line=function_line, samples=tuple(samples)) # The default of 100 is arbitrarily selected, but it does make the overwhelming @@ -157,86 +161,96 @@ def _read_textual_afdo_profile(stream): # Non-nm based approaches are superior because they don't require any prior # build artifacts; just an AFDO profile. def dedup_records(profile_records, summary_file, max_repeats=100): - """Removes heavily duplicated records from profile_records. - - profile_records is expected to be an iterable of ProfileRecord. - max_repeats ia how many functions must share identical bodies for us to - consider it 'heavily duplicated' and remove the results. - """ - - # Build a mapping of function structure -> list of functions with identical - # structure and sample counts - counts = collections.defaultdict(list) - for record in profile_records: - counts[_normalize_samples(record.samples)].append(record) - - # Be sure that we didn't see any duplicate functions, since that's bad... - total_functions_recorded = sum(len(records) for records in counts.values()) - - unique_function_names = { - record.function_line.split(':')[0] - for records in counts.values() - for record in records - } - - assert len(unique_function_names) == total_functions_recorded, \ - 'duplicate function names?' - - num_kept = 0 - num_samples_kept = 0 - num_top_samples_kept = 0 - num_total = 0 - num_samples_total = 0 - num_top_samples_total = 0 - - for normalized_samples, records in counts.items(): - top_sample_count, all_sample_count = _count_samples(normalized_samples) - top_sample_count *= len(records) - all_sample_count *= len(records) - - num_total += len(records) - num_samples_total += all_sample_count - num_top_samples_total += top_sample_count - - if len(records) >= max_repeats: - continue - - num_kept += len(records) - num_samples_kept += all_sample_count - num_top_samples_kept += top_sample_count - for record in records: - yield record - - print( - 'Retained {:,}/{:,} functions'.format(num_kept, num_total), - file=summary_file) - print( - 'Retained {:,}/{:,} samples, total'.format(num_samples_kept, - num_samples_total), - file=summary_file) - print('Retained {:,}/{:,} top-level samples' \ - .format(num_top_samples_kept, num_top_samples_total), - file=summary_file) + """Removes heavily duplicated records from profile_records. + + profile_records is expected to be an iterable of ProfileRecord. + max_repeats ia how many functions must share identical bodies for us to + consider it 'heavily duplicated' and remove the results. + """ + + # Build a mapping of function structure -> list of functions with identical + # structure and sample counts + counts = collections.defaultdict(list) + for record in profile_records: + counts[_normalize_samples(record.samples)].append(record) + + # Be sure that we didn't see any duplicate functions, since that's bad... + total_functions_recorded = sum(len(records) for records in counts.values()) + + unique_function_names = { + record.function_line.split(":")[0] + for records in counts.values() + for record in records + } + + assert ( + len(unique_function_names) == total_functions_recorded + ), "duplicate function names?" + + num_kept = 0 + num_samples_kept = 0 + num_top_samples_kept = 0 + num_total = 0 + num_samples_total = 0 + num_top_samples_total = 0 + + for normalized_samples, records in counts.items(): + top_sample_count, all_sample_count = _count_samples(normalized_samples) + top_sample_count *= len(records) + all_sample_count *= len(records) + + num_total += len(records) + num_samples_total += all_sample_count + num_top_samples_total += top_sample_count + + if len(records) >= max_repeats: + continue + + num_kept += len(records) + num_samples_kept += all_sample_count + num_top_samples_kept += top_sample_count + for record in records: + yield record + + print( + "Retained {:,}/{:,} functions".format(num_kept, num_total), + file=summary_file, + ) + print( + "Retained {:,}/{:,} samples, total".format( + num_samples_kept, num_samples_total + ), + file=summary_file, + ) + print( + "Retained {:,}/{:,} top-level samples".format( + num_top_samples_kept, num_top_samples_total + ), + file=summary_file, + ) def run(profile_input_file, summary_output_file, profile_output_file): - profile_records = _read_textual_afdo_profile(profile_input_file) + profile_records = _read_textual_afdo_profile(profile_input_file) - # Sort this so we get deterministic output. AFDO doesn't care what order it's - # in. - deduped = sorted( - dedup_records(profile_records, summary_output_file), - key=lambda r: r.function_line) - for function_line, samples in deduped: - print(function_line, file=profile_output_file) - print('\n'.join(samples), file=profile_output_file) + # Sort this so we get deterministic output. AFDO doesn't care what order it's + # in. + deduped = sorted( + dedup_records(profile_records, summary_output_file), + key=lambda r: r.function_line, + ) + for function_line, samples in deduped: + print(function_line, file=profile_output_file) + print("\n".join(samples), file=profile_output_file) def _main(): - run(profile_input_file=sys.stdin, - summary_output_file=sys.stderr, - profile_output_file=sys.stdout) + run( + profile_input_file=sys.stdin, + summary_output_file=sys.stderr, + profile_output_file=sys.stdout, + ) -if __name__ == '__main__': - _main() +if __name__ == "__main__": + _main() diff --git a/afdo_redaction/redact_profile_test.py b/afdo_redaction/redact_profile_test.py index e2438972..93c65510 100755 --- a/afdo_redaction/redact_profile_test.py +++ b/afdo_redaction/redact_profile_test.py @@ -1,136 +1,139 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2018 The Chromium OS Authors. All rights reserved. +# Copyright 2018 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Tests for redact_profile.py.""" -from __future__ import division, print_function import io import unittest from afdo_redaction import redact_profile + _redact_limit = redact_profile.dedup_records.__defaults__[0] def _redact(input_lines, summary_to=None): - if isinstance(input_lines, str): - input_lines = input_lines.splitlines() + if isinstance(input_lines, str): + input_lines = input_lines.splitlines() - if summary_to is None: - summary_to = io.StringIO() + if summary_to is None: + summary_to = io.StringIO() - output_to = io.StringIO() - redact_profile.run( - profile_input_file=input_lines, - summary_output_file=summary_to, - profile_output_file=output_to) - return output_to.getvalue() + output_to = io.StringIO() + redact_profile.run( + profile_input_file=input_lines, + summary_output_file=summary_to, + profile_output_file=output_to, + ) + return output_to.getvalue() def _redact_with_summary(input_lines): - summary = io.StringIO() - result = _redact(input_lines, summary_to=summary) - return result, summary.getvalue() - - -def _generate_repeated_function_body(repeats, fn_name='_some_name'): - # Arbitrary function body ripped from a textual AFDO profile. - function_header = fn_name + ':1234:185' - function_body = [ - ' 6: 83', - ' 15: 126', - ' 62832: 126', - ' 6: _ZNK5blink10PaintLayer14GroupedMappingEv:2349', - ' 1: 206', - ' 1: _ZNK5blink10PaintLayer14GroupedMappersEv:2060', - ' 1: 206', - ' 11: _ZNK5blink10PaintLayer25GetCompositedLayerMappingEv:800', - ' 2.1: 80', - ] - - # Be sure to zfill this, so the functions are output in sorted order. - num_width = len(str(repeats)) - - lines = [] - for i in range(repeats): - num = str(i).zfill(num_width) - lines.append(num + function_header) - lines.extend(function_body) - return lines + summary = io.StringIO() + result = _redact(input_lines, summary_to=summary) + return result, summary.getvalue() + + +def _generate_repeated_function_body(repeats, fn_name="_some_name"): + # Arbitrary function body ripped from a textual AFDO profile. + function_header = fn_name + ":1234:185" + function_body = [ + " 6: 83", + " 15: 126", + " 62832: 126", + " 6: _ZNK5blink10PaintLayer14GroupedMappingEv:2349", + " 1: 206", + " 1: _ZNK5blink10PaintLayer14GroupedMappersEv:2060", + " 1: 206", + " 11: _ZNK5blink10PaintLayer25GetCompositedLayerMappingEv:800", + " 2.1: 80", + ] + + # Be sure to zfill this, so the functions are output in sorted order. + num_width = len(str(repeats)) + + lines = [] + for i in range(repeats): + num = str(i).zfill(num_width) + lines.append(num + function_header) + lines.extend(function_body) + return lines class Tests(unittest.TestCase): - """All of our tests for redact_profile.""" - - def test_no_input_works(self): - self.assertEqual(_redact(''), '') - - def test_single_function_works(self): - lines = _generate_repeated_function_body(1) - result_file = '\n'.join(lines) + '\n' - self.assertEqual(_redact(lines), result_file) - - def test_duplicate_of_single_function_works(self): - lines = _generate_repeated_function_body(2) - result_file = '\n'.join(lines) + '\n' - self.assertEqual(_redact(lines), result_file) - - def test_not_too_many_duplicates_of_single_function_redacts_none(self): - lines = _generate_repeated_function_body(_redact_limit - 1) - result_file = '\n'.join(lines) + '\n' - self.assertEqual(_redact(lines), result_file) - - def test_many_duplicates_of_single_function_redacts_them_all(self): - lines = _generate_repeated_function_body(_redact_limit) - self.assertEqual(_redact(lines), '') - - def test_many_duplicates_of_single_function_leaves_other_functions(self): - kept_lines = _generate_repeated_function_body(1, fn_name='_keep_me') - # Something to distinguish us from the rest. Just bump a random counter. - kept_lines[1] += '1' - - result_file = '\n'.join(kept_lines) + '\n' - - lines = _generate_repeated_function_body( - _redact_limit, fn_name='_discard_me') - self.assertEqual(_redact(kept_lines + lines), result_file) - self.assertEqual(_redact(lines + kept_lines), result_file) - - more_lines = _generate_repeated_function_body( - _redact_limit, fn_name='_and_discard_me') - self.assertEqual(_redact(lines + kept_lines + more_lines), result_file) - self.assertEqual(_redact(lines + more_lines), '') - - def test_correct_summary_is_printed_when_nothing_is_redacted(self): - lines = _generate_repeated_function_body(1) - _, summary = _redact_with_summary(lines) - self.assertIn('Retained 1/1 functions', summary) - self.assertIn('Retained 827/827 samples, total', summary) - # Note that top-level samples == "samples without inlining taken into - # account," not "sum(entry_counts)" - self.assertIn('Retained 335/335 top-level samples', summary) - - def test_correct_summary_is_printed_when_everything_is_redacted(self): - lines = _generate_repeated_function_body(_redact_limit) - _, summary = _redact_with_summary(lines) - self.assertIn('Retained 0/100 functions', summary) - self.assertIn('Retained 0/82,700 samples, total', summary) - self.assertIn('Retained 0/33,500 top-level samples', summary) - - def test_correct_summary_is_printed_when_most_everything_is_redacted(self): - kept_lines = _generate_repeated_function_body(1, fn_name='_keep_me') - kept_lines[1] += '1' - - lines = _generate_repeated_function_body(_redact_limit) - _, summary = _redact_with_summary(kept_lines + lines) - self.assertIn('Retained 1/101 functions', summary) - self.assertIn('Retained 1,575/84,275 samples, total', summary) - self.assertIn('Retained 1,083/34,583 top-level samples', summary) - - -if __name__ == '__main__': - unittest.main() + """All of our tests for redact_profile.""" + + def test_no_input_works(self): + self.assertEqual(_redact(""), "") + + def test_single_function_works(self): + lines = _generate_repeated_function_body(1) + result_file = "\n".join(lines) + "\n" + self.assertEqual(_redact(lines), result_file) + + def test_duplicate_of_single_function_works(self): + lines = _generate_repeated_function_body(2) + result_file = "\n".join(lines) + "\n" + self.assertEqual(_redact(lines), result_file) + + def test_not_too_many_duplicates_of_single_function_redacts_none(self): + lines = _generate_repeated_function_body(_redact_limit - 1) + result_file = "\n".join(lines) + "\n" + self.assertEqual(_redact(lines), result_file) + + def test_many_duplicates_of_single_function_redacts_them_all(self): + lines = _generate_repeated_function_body(_redact_limit) + self.assertEqual(_redact(lines), "") + + def test_many_duplicates_of_single_function_leaves_other_functions(self): + kept_lines = _generate_repeated_function_body(1, fn_name="_keep_me") + # Something to distinguish us from the rest. Just bump a random counter. + kept_lines[1] += "1" + + result_file = "\n".join(kept_lines) + "\n" + + lines = _generate_repeated_function_body( + _redact_limit, fn_name="_discard_me" + ) + self.assertEqual(_redact(kept_lines + lines), result_file) + self.assertEqual(_redact(lines + kept_lines), result_file) + + more_lines = _generate_repeated_function_body( + _redact_limit, fn_name="_and_discard_me" + ) + self.assertEqual(_redact(lines + kept_lines + more_lines), result_file) + self.assertEqual(_redact(lines + more_lines), "") + + def test_correct_summary_is_printed_when_nothing_is_redacted(self): + lines = _generate_repeated_function_body(1) + _, summary = _redact_with_summary(lines) + self.assertIn("Retained 1/1 functions", summary) + self.assertIn("Retained 827/827 samples, total", summary) + # Note that top-level samples == "samples without inlining taken into + # account," not "sum(entry_counts)" + self.assertIn("Retained 335/335 top-level samples", summary) + + def test_correct_summary_is_printed_when_everything_is_redacted(self): + lines = _generate_repeated_function_body(_redact_limit) + _, summary = _redact_with_summary(lines) + self.assertIn("Retained 0/100 functions", summary) + self.assertIn("Retained 0/82,700 samples, total", summary) + self.assertIn("Retained 0/33,500 top-level samples", summary) + + def test_correct_summary_is_printed_when_most_everything_is_redacted(self): + kept_lines = _generate_repeated_function_body(1, fn_name="_keep_me") + kept_lines[1] += "1" + + lines = _generate_repeated_function_body(_redact_limit) + _, summary = _redact_with_summary(kept_lines + lines) + self.assertIn("Retained 1/101 functions", summary) + self.assertIn("Retained 1,575/84,275 samples, total", summary) + self.assertIn("Retained 1,083/34,583 top-level samples", summary) + + +if __name__ == "__main__": + unittest.main() diff --git a/afdo_redaction/remove_cold_functions.py b/afdo_redaction/remove_cold_functions.py index 097085db..c6043bc0 100755 --- a/afdo_redaction/remove_cold_functions.py +++ b/afdo_redaction/remove_cold_functions.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Copyright 2020 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -14,7 +14,7 @@ same sample count, we need to remove all of them in order to meet the target, so the result profile will always have less than or equal to the given number of functions. -The script is intended to be used on production Chrome OS profiles, after +The script is intended to be used on production ChromeOS profiles, after other redaction/trimming scripts. It can be used with given textual CWP and benchmark profiles, in order to analyze how many removed functions are from which profile (or both), which can be used an indicator of fairness @@ -24,160 +24,191 @@ This is part of the effort to stablize the impact of AFDO profile on Chrome binary size. See crbug.com/1062014 for more context. """ -from __future__ import division, print_function import argparse import collections import re import sys -_function_line_re = re.compile(r'^([\w\$\.@]+):(\d+)(?::\d+)?$') + +_function_line_re = re.compile(r"^([\w\$\.@]+):(\d+)(?::\d+)?$") ProfileRecord = collections.namedtuple( - 'ProfileRecord', ['function_count', 'function_body', 'function_name']) + "ProfileRecord", ["function_count", "function_body", "function_name"] +) def _read_sample_count(line): - m = _function_line_re.match(line) - assert m, 'Failed to interpret function line %s' % line - return m.group(1), int(m.group(2)) + m = _function_line_re.match(line) + assert m, "Failed to interpret function line %s" % line + return m.group(1), int(m.group(2)) def _read_textual_afdo_profile(stream): - """Parses an AFDO profile from a line stream into ProfileRecords.""" - # ProfileRecords are actually nested, due to inlining. For the purpose of - # this script, that doesn't matter. - lines = (line.rstrip() for line in stream) - function_line = None - samples = [] - ret = [] - for line in lines: - if not line: - continue - - if line[0].isspace(): - assert function_line is not None, 'sample exists outside of a function?' - samples.append(line) - continue - - if function_line is not None: - name, count = _read_sample_count(function_line) - body = [function_line] + samples - ret.append( - ProfileRecord( - function_count=count, function_body=body, function_name=name)) - function_line = line + """Parses an AFDO profile from a line stream into ProfileRecords.""" + # ProfileRecords are actually nested, due to inlining. For the purpose of + # this script, that doesn't matter. + lines = (line.rstrip() for line in stream) + function_line = None samples = [] + ret = [] + for line in lines: + if not line: + continue + + if line[0].isspace(): + assert ( + function_line is not None + ), "sample exists outside of a function?" + samples.append(line) + continue + + if function_line is not None: + name, count = _read_sample_count(function_line) + body = [function_line] + samples + ret.append( + ProfileRecord( + function_count=count, function_body=body, function_name=name + ) + ) + function_line = line + samples = [] - if function_line is not None: - name, count = _read_sample_count(function_line) - body = [function_line] + samples - ret.append( - ProfileRecord( - function_count=count, function_body=body, function_name=name)) - return ret + if function_line is not None: + name, count = _read_sample_count(function_line) + body = [function_line] + samples + ret.append( + ProfileRecord( + function_count=count, function_body=body, function_name=name + ) + ) + return ret def write_textual_afdo_profile(stream, records): - for r in records: - print('\n'.join(r.function_body), file=stream) + for r in records: + print("\n".join(r.function_body), file=stream) def analyze_functions(records, cwp, benchmark): - cwp_functions = {x.function_name for x in cwp} - benchmark_functions = {x.function_name for x in benchmark} - all_functions = {x.function_name for x in records} - cwp_only_functions = len((all_functions & cwp_functions) - - benchmark_functions) - benchmark_only_functions = len((all_functions & benchmark_functions) - - cwp_functions) - common_functions = len(all_functions & benchmark_functions & cwp_functions) - none_functions = len(all_functions - benchmark_functions - cwp_functions) - - assert not none_functions - return cwp_only_functions, benchmark_only_functions, common_functions + cwp_functions = {x.function_name for x in cwp} + benchmark_functions = {x.function_name for x in benchmark} + all_functions = {x.function_name for x in records} + cwp_only_functions = len( + (all_functions & cwp_functions) - benchmark_functions + ) + benchmark_only_functions = len( + (all_functions & benchmark_functions) - cwp_functions + ) + common_functions = len(all_functions & benchmark_functions & cwp_functions) + none_functions = len(all_functions - benchmark_functions - cwp_functions) + + assert not none_functions + return cwp_only_functions, benchmark_only_functions, common_functions def run(input_stream, output_stream, goal, cwp=None, benchmark=None): - records = _read_textual_afdo_profile(input_stream) - num_functions = len(records) - if not num_functions: - return - assert goal, "It's invalid to remove all functions in the profile" - - if cwp and benchmark: - cwp_records = _read_textual_afdo_profile(cwp) - benchmark_records = _read_textual_afdo_profile(benchmark) - cwp_num, benchmark_num, common_num = analyze_functions( - records, cwp_records, benchmark_records) - - records.sort(key=lambda x: (-x.function_count, x.function_name)) - records = records[:goal] - - print( - 'Retained %d/%d (%.1f%%) functions in the profile' % - (len(records), num_functions, 100.0 * len(records) / num_functions), - file=sys.stderr) - write_textual_afdo_profile(output_stream, records) - - if cwp and benchmark: - cwp_num_after, benchmark_num_after, common_num_after = analyze_functions( - records, cwp_records, benchmark_records) - print( - 'Retained %d/%d (%.1f%%) functions only appear in the CWP profile' % - (cwp_num_after, cwp_num, 100.0 * cwp_num_after / cwp_num), - file=sys.stderr) - print( - 'Retained %d/%d (%.1f%%) functions only appear in the benchmark profile' - % (benchmark_num_after, benchmark_num, - 100.0 * benchmark_num_after / benchmark_num), - file=sys.stderr) + records = _read_textual_afdo_profile(input_stream) + num_functions = len(records) + if not num_functions: + return + assert goal, "It's invalid to remove all functions in the profile" + + if cwp and benchmark: + cwp_records = _read_textual_afdo_profile(cwp) + benchmark_records = _read_textual_afdo_profile(benchmark) + cwp_num, benchmark_num, common_num = analyze_functions( + records, cwp_records, benchmark_records + ) + + records.sort(key=lambda x: (-x.function_count, x.function_name)) + records = records[:goal] + print( - 'Retained %d/%d (%.1f%%) functions appear in both CWP and benchmark' - ' profiles' % (common_num_after, common_num, - 100.0 * common_num_after / common_num), - file=sys.stderr) + "Retained %d/%d (%.1f%%) functions in the profile" + % (len(records), num_functions, 100.0 * len(records) / num_functions), + file=sys.stderr, + ) + write_textual_afdo_profile(output_stream, records) + + if cwp and benchmark: + ( + cwp_num_after, + benchmark_num_after, + common_num_after, + ) = analyze_functions(records, cwp_records, benchmark_records) + print( + "Retained %d/%d (%.1f%%) functions only appear in the CWP profile" + % (cwp_num_after, cwp_num, 100.0 * cwp_num_after / cwp_num), + file=sys.stderr, + ) + print( + "Retained %d/%d (%.1f%%) functions only appear in the benchmark profile" + % ( + benchmark_num_after, + benchmark_num, + 100.0 * benchmark_num_after / benchmark_num, + ), + file=sys.stderr, + ) + print( + "Retained %d/%d (%.1f%%) functions appear in both CWP and benchmark" + " profiles" + % ( + common_num_after, + common_num, + 100.0 * common_num_after / common_num, + ), + file=sys.stderr, + ) def main(): - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument( - '--input', - default='/dev/stdin', - help='File to read from. Defaults to stdin.') - parser.add_argument( - '--output', - default='/dev/stdout', - help='File to write to. Defaults to stdout.') - parser.add_argument( - '--number', - type=int, - required=True, - help='Number of functions to retain in the profile.') - parser.add_argument( - '--cwp', help='Textualized CWP profiles, used for further analysis') - parser.add_argument( - '--benchmark', - help='Textualized benchmark profile, used for further analysis') - args = parser.parse_args() - - if not args.number: - parser.error("It's invalid to remove the number of functions to 0.") - - if (args.cwp and not args.benchmark) or (not args.cwp and args.benchmark): - parser.error('Please specify both --cwp and --benchmark') - - with open(args.input) as stdin: - with open(args.output, 'w') as stdout: - # When user specify textualized cwp and benchmark profiles, perform - # the analysis. Otherwise, just trim the cold functions from profile. - if args.cwp and args.benchmark: - with open(args.cwp) as cwp: - with open(args.benchmark) as benchmark: - run(stdin, stdout, args.number, cwp, benchmark) - else: - run(stdin, stdout, args.number) - - -if __name__ == '__main__': - main() + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--input", + default="/dev/stdin", + help="File to read from. Defaults to stdin.", + ) + parser.add_argument( + "--output", + default="/dev/stdout", + help="File to write to. Defaults to stdout.", + ) + parser.add_argument( + "--number", + type=int, + required=True, + help="Number of functions to retain in the profile.", + ) + parser.add_argument( + "--cwp", help="Textualized CWP profiles, used for further analysis" + ) + parser.add_argument( + "--benchmark", + help="Textualized benchmark profile, used for further analysis", + ) + args = parser.parse_args() + + if not args.number: + parser.error("It's invalid to remove the number of functions to 0.") + + if (args.cwp and not args.benchmark) or (not args.cwp and args.benchmark): + parser.error("Please specify both --cwp and --benchmark") + + with open(args.input) as stdin: + with open(args.output, "w") as stdout: + # When user specify textualized cwp and benchmark profiles, perform + # the analysis. Otherwise, just trim the cold functions from profile. + if args.cwp and args.benchmark: + with open(args.cwp) as cwp: + with open(args.benchmark) as benchmark: + run(stdin, stdout, args.number, cwp, benchmark) + else: + run(stdin, stdout, args.number) + + +if __name__ == "__main__": + main() diff --git a/afdo_redaction/remove_cold_functions_test.py b/afdo_redaction/remove_cold_functions_test.py index 14f946b0..89a87f82 100755 --- a/afdo_redaction/remove_cold_functions_test.py +++ b/afdo_redaction/remove_cold_functions_test.py @@ -1,28 +1,27 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Copyright 2020 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Tests for remove_cold_functions.""" -from __future__ import print_function import io -from unittest.mock import patch import unittest +from unittest.mock import patch from afdo_redaction import remove_cold_functions def _construct_profile(indices=None): - real_world_profile_functions = [ - """SomeFunction1:24150:300 + real_world_profile_functions = [ + """SomeFunction1:24150:300 2: 75 3: 23850 39: 225 """, - """SomeFunction2:8925:225 + """SomeFunction2:8925:225 0: 225 0.2: 150 0.1: SomeFunction2:6300 @@ -30,7 +29,7 @@ def _construct_profile(indices=None): 0.2: SomeFunction2:150 3: 75 """, - """SomeFunction3:7500:75 + """SomeFunction3:7500:75 0: 75 0.2: 75 0.1: SomeFunction3:6600 @@ -38,7 +37,7 @@ def _construct_profile(indices=None): 0.2: SomeFunction3:75 1: 75 """, - """LargerFunction4:51450:0 + """LargerFunction4:51450:0 1: 0 3: 0 3.1: 7350 @@ -59,7 +58,7 @@ def _construct_profile(indices=None): 8: 0 9: 0 """, - """SomeFakeFunction5:7500:75 + """SomeFakeFunction5:7500:75 0: 75 0.2: 75 0.1: SomeFakeFunction5:6600 @@ -67,80 +66,87 @@ def _construct_profile(indices=None): 0.2: SomeFakeFunction5:75 1: 75 """, - ] + ] - ret = [] - if not indices: - for x in real_world_profile_functions: - ret += x.strip().splitlines() - return ret + ret = [] + if not indices: + for x in real_world_profile_functions: + ret += x.strip().splitlines() + return ret - ret = [] - for i in indices: - ret += real_world_profile_functions[i].strip().splitlines() - return ret + ret = [] + for i in indices: + ret += real_world_profile_functions[i].strip().splitlines() + return ret def _run_test(input_lines, goal, cwp_file=None, benchmark_file=None): - input_buf = io.StringIO('\n'.join(input_lines)) - output_buf = io.StringIO() - remove_cold_functions.run(input_buf, output_buf, goal, cwp_file, - benchmark_file) - return output_buf.getvalue().splitlines() + input_buf = io.StringIO("\n".join(input_lines)) + output_buf = io.StringIO() + remove_cold_functions.run( + input_buf, output_buf, goal, cwp_file, benchmark_file + ) + return output_buf.getvalue().splitlines() class Test(unittest.TestCase): - """Test functions in remove_cold_functions.py""" - - def test_empty_profile(self): - self.assertEqual(_run_test([], 0), []) - - def test_remove_all_functions_fail(self): - input_profile_lines = _construct_profile() - with self.assertRaises(Exception) as context: - _run_test(input_profile_lines, 0) - self.assertEqual( - str(context.exception), - "It's invalid to remove all functions in the profile") - - def test_remove_cold_functions_work(self): - input_profile_lines = _construct_profile() - # To make sure the cold functions are removed in order - expected_profile_lines = { - 5: input_profile_lines, - # Entry 4 wins the tie breaker because the name is smaller - # alphabetically. - 4: _construct_profile([0, 1, 3, 4]), - 3: _construct_profile([0, 1, 3]), - 2: _construct_profile([0, 3]), - 1: _construct_profile([3]), - } - - for num in expected_profile_lines: - self.assertCountEqual( - _run_test(input_profile_lines, num), expected_profile_lines[num]) - - def test_analyze_cwp_and_benchmark_work(self): - input_profile_lines = _construct_profile() - cwp_profile = _construct_profile([0, 1, 3, 4]) - benchmark_profile = _construct_profile([1, 2, 3, 4]) - cwp_buf = io.StringIO('\n'.join(cwp_profile)) - benchmark_buf = io.StringIO('\n'.join(benchmark_profile)) - with patch('sys.stderr', new=io.StringIO()) as fake_output: - _run_test(input_profile_lines, 3, cwp_buf, benchmark_buf) - - output = fake_output.getvalue() - self.assertIn('Retained 3/5 (60.0%) functions in the profile', output) - self.assertIn( - 'Retained 1/1 (100.0%) functions only appear in the CWP profile', - output) - self.assertIn( - 'Retained 0/1 (0.0%) functions only appear in the benchmark profile', - output) - self.assertIn( - 'Retained 2/3 (66.7%) functions appear in both CWP and benchmark' - ' profiles', output) - - -if __name__ == '__main__': - unittest.main() + """Test functions in remove_cold_functions.py""" + + def test_empty_profile(self): + self.assertEqual(_run_test([], 0), []) + + def test_remove_all_functions_fail(self): + input_profile_lines = _construct_profile() + with self.assertRaises(Exception) as context: + _run_test(input_profile_lines, 0) + self.assertEqual( + str(context.exception), + "It's invalid to remove all functions in the profile", + ) + + def test_remove_cold_functions_work(self): + input_profile_lines = _construct_profile() + # To make sure the cold functions are removed in order + expected_profile_lines = { + 5: input_profile_lines, + # Entry 4 wins the tie breaker because the name is smaller + # alphabetically. + 4: _construct_profile([0, 1, 3, 4]), + 3: _construct_profile([0, 1, 3]), + 2: _construct_profile([0, 3]), + 1: _construct_profile([3]), + } + + for num in expected_profile_lines: + self.assertCountEqual( + _run_test(input_profile_lines, num), expected_profile_lines[num] + ) + + def test_analyze_cwp_and_benchmark_work(self): + input_profile_lines = _construct_profile() + cwp_profile = _construct_profile([0, 1, 3, 4]) + benchmark_profile = _construct_profile([1, 2, 3, 4]) + cwp_buf = io.StringIO("\n".join(cwp_profile)) + benchmark_buf = io.StringIO("\n".join(benchmark_profile)) + with patch("sys.stderr", new=io.StringIO()) as fake_output: + _run_test(input_profile_lines, 3, cwp_buf, benchmark_buf) + + output = fake_output.getvalue() + self.assertIn("Retained 3/5 (60.0%) functions in the profile", output) + self.assertIn( + "Retained 1/1 (100.0%) functions only appear in the CWP profile", + output, + ) + self.assertIn( + "Retained 0/1 (0.0%) functions only appear in the benchmark profile", + output, + ) + self.assertIn( + "Retained 2/3 (66.7%) functions appear in both CWP and benchmark" + " profiles", + output, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/afdo_redaction/remove_indirect_calls.py b/afdo_redaction/remove_indirect_calls.py index 0dc15077..32dab3f4 100755 --- a/afdo_redaction/remove_indirect_calls.py +++ b/afdo_redaction/remove_indirect_calls.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2019 The Chromium OS Authors. All rights reserved. +# Copyright 2019 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -17,21 +17,20 @@ objects as Chrome, this can become problematic, and lead to NaCl doubling in size (or worse). See crbug.com/1005023 and crbug.com/916130. """ -from __future__ import division, print_function import argparse import re def _remove_indirect_call_targets(lines): - # Lines with indirect call targets look like: - # 1.1: 1234 foo:111 bar:122 - # - # Where 1.1 is the line info/discriminator, 1234 is the total number of - # samples seen for that line/discriminator, foo:111 is "111 of the calls here - # went to foo," and bar:122 is "122 of the calls here went to bar." - call_target_re = re.compile( - r""" + # Lines with indirect call targets look like: + # 1.1: 1234 foo:111 bar:122 + # + # Where 1.1 is the line info/discriminator, 1234 is the total number of + # samples seen for that line/discriminator, foo:111 is "111 of the calls here + # went to foo," and bar:122 is "122 of the calls here went to bar." + call_target_re = re.compile( + r""" ^\s+ # Top-level lines are function records. \d+(?:\.\d+)?: # Line info/discriminator \s+ @@ -39,42 +38,48 @@ def _remove_indirect_call_targets(lines): \s+ ((?:[^\s:]+:\d+\s*)+) # Indirect call target(s) $ - """, re.VERBOSE) - for line in lines: - line = line.rstrip() + """, + re.VERBOSE, + ) + for line in lines: + line = line.rstrip() - match = call_target_re.match(line) - if not match: - yield line + '\n' - continue + match = call_target_re.match(line) + if not match: + yield line + "\n" + continue - group_start, group_end = match.span(1) - assert group_end == len(line) - yield line[:group_start].rstrip() + '\n' + group_start, group_end = match.span(1) + assert group_end == len(line) + yield line[:group_start].rstrip() + "\n" def run(input_stream, output_stream): - for line in _remove_indirect_call_targets(input_stream): - output_stream.write(line) + for line in _remove_indirect_call_targets(input_stream): + output_stream.write(line) def main(): - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument( - '--input', - default='/dev/stdin', - help='File to read from. Defaults to stdin.') - parser.add_argument( - '--output', - default='/dev/stdout', - help='File to write to. Defaults to stdout.') - args = parser.parse_args() - - with open(args.input) as stdin: - with open(args.output, 'w') as stdout: - run(stdin, stdout) - - -if __name__ == '__main__': - main() + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--input", + default="/dev/stdin", + help="File to read from. Defaults to stdin.", + ) + parser.add_argument( + "--output", + default="/dev/stdout", + help="File to write to. Defaults to stdout.", + ) + args = parser.parse_args() + + with open(args.input) as stdin: + with open(args.output, "w") as stdout: + run(stdin, stdout) + + +if __name__ == "__main__": + main() diff --git a/afdo_redaction/remove_indirect_calls_test.py b/afdo_redaction/remove_indirect_calls_test.py index 164b284f..640b747f 100755 --- a/afdo_redaction/remove_indirect_calls_test.py +++ b/afdo_redaction/remove_indirect_calls_test.py @@ -1,12 +1,11 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2019 The Chromium OS Authors. All rights reserved. +# Copyright 2019 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Tests for remove_indirect_calls""" -from __future__ import print_function import io import unittest @@ -15,22 +14,22 @@ from afdo_redaction import remove_indirect_calls def _run_test(input_lines): - input_buf = io.StringIO('\n'.join(input_lines)) - output_buf = io.StringIO() - remove_indirect_calls.run(input_buf, output_buf) - return output_buf.getvalue().splitlines() + input_buf = io.StringIO("\n".join(input_lines)) + output_buf = io.StringIO() + remove_indirect_calls.run(input_buf, output_buf) + return output_buf.getvalue().splitlines() class Test(unittest.TestCase): - """Tests""" + """Tests""" - def test_empty_profile(self): - self.assertEqual(_run_test([]), []) + def test_empty_profile(self): + self.assertEqual(_run_test([]), []) - def test_removal_on_real_world_code(self): - # These are copied from an actual textual AFDO profile, but the names made - # lints unhappy due to their length, so I had to be creative. - profile_lines = """_ZLongSymbolName:52862:1766 + def test_removal_on_real_world_code(self): + # These are copied from an actual textual AFDO profile, but the names made + # lints unhappy due to their length, so I had to be creative. + profile_lines = """_ZLongSymbolName:52862:1766 14: 2483 8.1: _SomeInlinedSym:45413 11: _AndAnother:35481 @@ -45,7 +44,7 @@ class Test(unittest.TestCase): 0: 2483 """.strip().splitlines() - expected_lines = """_ZLongSymbolName:52862:1766 + expected_lines = """_ZLongSymbolName:52862:1766 14: 2483 8.1: _SomeInlinedSym:45413 11: _AndAnother:35481 @@ -60,8 +59,8 @@ class Test(unittest.TestCase): 0: 2483 """.strip().splitlines() - self.assertEqual(_run_test(profile_lines), expected_lines) + self.assertEqual(_run_test(profile_lines), expected_lines) -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() |