diff options
Diffstat (limited to 'heatmaps')
| -rwxr-xr-x | heatmaps/heat_map.py | 341 | ||||
| -rwxr-xr-x | heatmaps/heat_map_test.py | 281 | ||||
| -rw-r--r-- | heatmaps/heatmap_generator.py | 929 | ||||
| -rwxr-xr-x | heatmaps/heatmap_generator_test.py | 572 | ||||
| -rwxr-xr-x | heatmaps/perf-to-inst-page.sh | 2 |
5 files changed, 1117 insertions, 1008 deletions
diff --git a/heatmaps/heat_map.py b/heatmaps/heat_map.py index a989ab70..a3c52369 100755 --- a/heatmaps/heat_map.py +++ b/heatmaps/heat_map.py @@ -1,12 +1,11 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2015 The Chromium OS Authors. All rights reserved. +# Copyright 2015 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Wrapper to generate heat maps for chrome.""" -from __future__ import print_function import argparse import os @@ -19,167 +18,189 @@ from heatmaps import heatmap_generator def IsARepoRoot(directory): - """Returns True if directory is the root of a repo checkout.""" - return os.path.exists( - os.path.join(os.path.realpath(os.path.expanduser(directory)), '.repo')) + """Returns True if directory is the root of a repo checkout.""" + return os.path.exists( + os.path.join(os.path.realpath(os.path.expanduser(directory)), ".repo") + ) class HeatMapProducer(object): - """Class to produce heat map.""" - - def __init__(self, - chromeos_root, - perf_data, - hugepage, - binary, - title, - logger=None): - self.chromeos_root = os.path.realpath(os.path.expanduser(chromeos_root)) - self.perf_data = os.path.realpath(os.path.expanduser(perf_data)) - self.hugepage = hugepage - self.dir = os.path.dirname(os.path.realpath(__file__)) - self.binary = binary - self.ce = command_executer.GetCommandExecuter() - self.temp_dir = '' - self.temp_perf_inchroot = '' - self.temp_dir_created = False - self.perf_report = '' - self.title = title - self.logger = logger - - def _EnsureFileInChroot(self): - chroot_prefix = os.path.join(self.chromeos_root, 'chroot') - if self.perf_data.startswith(chroot_prefix): - # If the path to perf_data starts with the same chromeos_root, assume - # it's in the chromeos_root so no need for temporary directory and copy. - self.temp_dir = self.perf_data.replace('perf.data', '') - self.temp_perf_inchroot = self.temp_dir.replace(chroot_prefix, '') - - else: - # Otherwise, create a temporary directory and copy perf.data into chroot. - self.temp_dir = tempfile.mkdtemp( - prefix=os.path.join(self.chromeos_root, 'src/')) - temp_perf = os.path.join(self.temp_dir, 'perf.data') - shutil.copy2(self.perf_data, temp_perf) - self.temp_perf_inchroot = os.path.join('~/trunk/src', - os.path.basename(self.temp_dir)) - self.temp_dir_created = True - - def _GeneratePerfReport(self): - cmd = ('cd %s && perf report -D -i perf.data > perf_report.txt' % - self.temp_perf_inchroot) - retval = self.ce.ChrootRunCommand(self.chromeos_root, cmd) - if retval: - raise RuntimeError('Failed to generate perf report') - self.perf_report = os.path.join(self.temp_dir, 'perf_report.txt') - - def _GetHeatMap(self, top_n_pages): - generator = heatmap_generator.HeatmapGenerator( - perf_report=self.perf_report, - page_size=4096, - hugepage=self.hugepage, - title=self.title) - generator.draw() - # Analyze top N hottest symbols with the binary, if provided - if self.binary: - generator.analyze(self.binary, top_n_pages) - - def _RemoveFiles(self): - files = [ - 'out.txt', 'inst-histo.txt', 'inst-histo-hp.txt', 'inst-histo-sp.txt' - ] - for f in files: - if os.path.exists(f): - os.remove(f) - - def Run(self, top_n_pages): - try: - self._EnsureFileInChroot() - self._GeneratePerfReport() - self._GetHeatMap(top_n_pages) - finally: - self._RemoveFiles() - msg = ('heat map and time histogram genereated in the current ' - 'directory with name heat_map.png and timeline.png ' - 'accordingly.') - if self.binary: - msg += ('\nThe hottest %d pages inside and outside hugepage ' - 'is symbolized and saved to addr2symbol.txt' % top_n_pages) - if self.logger: - self.logger.LogOutput(msg) - else: - print(msg) + """Class to produce heat map.""" + + def __init__( + self, chromeos_root, perf_data, hugepage, binary, title, logger=None + ): + self.chromeos_root = os.path.realpath(os.path.expanduser(chromeos_root)) + self.perf_data = os.path.realpath(os.path.expanduser(perf_data)) + self.hugepage = hugepage + self.dir = os.path.dirname(os.path.realpath(__file__)) + self.binary = binary + self.ce = command_executer.GetCommandExecuter() + self.temp_dir = "" + self.temp_perf_inchroot = "" + self.temp_dir_created = False + self.perf_report = "" + self.title = title + self.logger = logger + + def _EnsureFileInChroot(self): + chroot_prefix = os.path.join(self.chromeos_root, "chroot") + if self.perf_data.startswith(chroot_prefix): + # If the path to perf_data starts with the same chromeos_root, assume + # it's in the chromeos_root so no need for temporary directory and copy. + self.temp_dir = self.perf_data.replace("perf.data", "") + self.temp_perf_inchroot = self.temp_dir.replace(chroot_prefix, "") + + else: + # Otherwise, create a temporary directory and copy perf.data into chroot. + self.temp_dir = tempfile.mkdtemp( + prefix=os.path.join(self.chromeos_root, "src/") + ) + temp_perf = os.path.join(self.temp_dir, "perf.data") + shutil.copy2(self.perf_data, temp_perf) + self.temp_perf_inchroot = os.path.join( + "~/trunk/src", os.path.basename(self.temp_dir) + ) + self.temp_dir_created = True + + def _GeneratePerfReport(self): + cmd = ( + "cd %s && perf report -D -i perf.data > perf_report.txt" + % self.temp_perf_inchroot + ) + retval = self.ce.ChrootRunCommand(self.chromeos_root, cmd) + if retval: + raise RuntimeError("Failed to generate perf report") + self.perf_report = os.path.join(self.temp_dir, "perf_report.txt") + + def _GetHeatMap(self, top_n_pages): + generator = heatmap_generator.HeatmapGenerator( + perf_report=self.perf_report, + page_size=4096, + hugepage=self.hugepage, + title=self.title, + ) + generator.draw() + # Analyze top N hottest symbols with the binary, if provided + if self.binary: + generator.analyze(self.binary, top_n_pages) + + def _RemoveFiles(self): + files = [ + "out.txt", + "inst-histo.txt", + "inst-histo-hp.txt", + "inst-histo-sp.txt", + ] + for f in files: + if os.path.exists(f): + os.remove(f) + + def Run(self, top_n_pages): + try: + self._EnsureFileInChroot() + self._GeneratePerfReport() + self._GetHeatMap(top_n_pages) + finally: + self._RemoveFiles() + msg = ( + "heat map and time histogram genereated in the current " + "directory with name heat_map.png and timeline.png " + "accordingly." + ) + if self.binary: + msg += ( + "\nThe hottest %d pages inside and outside hugepage " + "is symbolized and saved to addr2symbol.txt" % top_n_pages + ) + if self.logger: + self.logger.LogOutput(msg) + else: + print(msg) def main(argv): - """Parse the options. - - Args: - argv: The options with which this script was invoked. - - Returns: - 0 unless an exception is raised. - """ - parser = argparse.ArgumentParser() - - parser.add_argument( - '--chromeos_root', - dest='chromeos_root', - required=True, - help='ChromeOS root to use for generate heatmaps.') - parser.add_argument( - '--perf_data', - dest='perf_data', - required=True, - help='The raw perf data. Must be collected with -e instructions while ' - 'disabling ASLR.') - parser.add_argument( - '--binary', - dest='binary', - help='The path to the Chrome binary. Only useful if want to print ' - 'symbols on hottest pages', - default=None) - parser.add_argument( - '--top_n', - dest='top_n', - type=int, - default=10, - help='Print out top N hottest pages within/outside huge page range. ' - 'Must be used with --hugepage and --binary. (Default: %(default)s)') - parser.add_argument( - '--title', dest='title', help='Title of the heatmap', default='') - parser.add_argument( - '--hugepage', - dest='hugepage', - help='A range of addresses (start,end) where huge page starts and ends' - ' in text section, separated by a comma.' - ' Used to differentiate regions in heatmap.' - ' Example: --hugepage=0,4096' - ' If not specified, no effect on the heatmap.', - default=None) - - options = parser.parse_args(argv) - - if not IsARepoRoot(options.chromeos_root): - parser.error('%s does not contain .repo dir.' % options.chromeos_root) - - if not os.path.isfile(options.perf_data): - parser.error('Cannot find perf_data: %s.' % options.perf_data) - - hugepage_range = None - if options.hugepage: - hugepage_range = options.hugepage.split(',') - if len(hugepage_range) != 2 or \ - int(hugepage_range[0]) > int(hugepage_range[1]): - parser.error('Wrong format of hugepage range: %s' % options.hugepage) - hugepage_range = [int(x) for x in hugepage_range] - - heatmap_producer = HeatMapProducer(options.chromeos_root, options.perf_data, - hugepage_range, options.binary, - options.title) - - heatmap_producer.Run(options.top_n) - - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) + """Parse the options. + + Args: + argv: The options with which this script was invoked. + + Returns: + 0 unless an exception is raised. + """ + parser = argparse.ArgumentParser() + + parser.add_argument( + "--chromeos_root", + dest="chromeos_root", + required=True, + help="ChromeOS root to use for generate heatmaps.", + ) + parser.add_argument( + "--perf_data", + dest="perf_data", + required=True, + help="The raw perf data. Must be collected with -e instructions while " + "disabling ASLR.", + ) + parser.add_argument( + "--binary", + dest="binary", + help="The path to the Chrome binary. Only useful if want to print " + "symbols on hottest pages", + default=None, + ) + parser.add_argument( + "--top_n", + dest="top_n", + type=int, + default=10, + help="Print out top N hottest pages within/outside huge page range. " + "Must be used with --hugepage and --binary. (Default: %(default)s)", + ) + parser.add_argument( + "--title", dest="title", help="Title of the heatmap", default="" + ) + parser.add_argument( + "--hugepage", + dest="hugepage", + help="A range of addresses (start,end) where huge page starts and ends" + " in text section, separated by a comma." + " Used to differentiate regions in heatmap." + " Example: --hugepage=0,4096" + " If not specified, no effect on the heatmap.", + default=None, + ) + + options = parser.parse_args(argv) + + if not IsARepoRoot(options.chromeos_root): + parser.error("%s does not contain .repo dir." % options.chromeos_root) + + if not os.path.isfile(options.perf_data): + parser.error("Cannot find perf_data: %s." % options.perf_data) + + hugepage_range = None + if options.hugepage: + hugepage_range = options.hugepage.split(",") + if len(hugepage_range) != 2 or int(hugepage_range[0]) > int( + hugepage_range[1] + ): + parser.error( + "Wrong format of hugepage range: %s" % options.hugepage + ) + hugepage_range = [int(x) for x in hugepage_range] + + heatmap_producer = HeatMapProducer( + options.chromeos_root, + options.perf_data, + hugepage_range, + options.binary, + options.title, + ) + + heatmap_producer.Run(options.top_n) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/heatmaps/heat_map_test.py b/heatmaps/heat_map_test.py index ad62cd91..96300bb4 100755 --- a/heatmaps/heat_map_test.py +++ b/heatmaps/heat_map_test.py @@ -1,158 +1,179 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # -# Copyright 2019 The Chromium OS Authors. All rights reserved. +# Copyright 2019 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Tests for heat_map.py.""" -from __future__ import print_function - -import unittest.mock as mock -import unittest import os +import unittest +import unittest.mock as mock from cros_utils import command_executer - from heatmaps import heat_map from heatmaps import heatmap_generator -def make_heatmap(chromeos_root='/path/to/fake/chromeos_root/', - perf_data='/any_path/perf.data'): - return heat_map.HeatMapProducer(chromeos_root, perf_data, None, None, '') +def make_heatmap( + chromeos_root="/path/to/fake/chromeos_root/", + perf_data="/any_path/perf.data", +): + return heat_map.HeatMapProducer(chromeos_root, perf_data, None, None, "") def fake_mkdtemp(prefix): - """Mock tempfile.mkdtemp() by just create a pathname.""" - return prefix + 'random_dir' + """Mock tempfile.mkdtemp() by just create a pathname.""" + return prefix + "random_dir" def fake_parser_error(_, msg): - """Redirect parser.error() to exception.""" - raise Exception(msg) + """Redirect parser.error() to exception.""" + raise Exception(msg) def fake_generate_perf_report_exception(_): - raise Exception + raise Exception class HeatmapTest(unittest.TestCase): - """All of our tests for heat_map.""" - - # pylint: disable=protected-access - @mock.patch('shutil.copy2') - @mock.patch('tempfile.mkdtemp') - def test_EnsureFileInChrootAlreadyInside(self, mock_mkdtemp, mock_copy): - perf_data_inchroot = ( - '/path/to/fake/chromeos_root/chroot/inchroot_path/perf.data') - heatmap = make_heatmap(perf_data=perf_data_inchroot) - heatmap._EnsureFileInChroot() - self.assertFalse(heatmap.temp_dir_created) - self.assertEqual(heatmap.temp_dir, - '/path/to/fake/chromeos_root/chroot/inchroot_path/') - self.assertEqual(heatmap.temp_perf_inchroot, '/inchroot_path/') - mock_mkdtemp.assert_not_called() - mock_copy.assert_not_called() - - @mock.patch('shutil.copy2') - @mock.patch('tempfile.mkdtemp', fake_mkdtemp) - def test_EnsureFileInChrootOutsideNeedCopy(self, mock_copy): - heatmap = make_heatmap() - heatmap._EnsureFileInChroot() - self.assertTrue(heatmap.temp_dir_created) - self.assertEqual(mock_copy.call_count, 1) - self.assertEqual(heatmap.temp_dir, - '/path/to/fake/chromeos_root/src/random_dir') - self.assertEqual(heatmap.temp_perf_inchroot, '~/trunk/src/random_dir') - - @mock.patch.object(command_executer.CommandExecuter, 'ChrootRunCommand') - def test_GeneratePerfReport(self, mock_ChrootRunCommand): - heatmap = make_heatmap() - heatmap.temp_dir = '/fake/chroot/inchroot_path/' - heatmap.temp_perf_inchroot = '/inchroot_path/' - mock_ChrootRunCommand.return_value = 0 - heatmap._GeneratePerfReport() - cmd = ('cd %s && perf report -D -i perf.data > perf_report.txt' % - heatmap.temp_perf_inchroot) - mock_ChrootRunCommand.assert_called_with(heatmap.chromeos_root, cmd) - self.assertEqual(mock_ChrootRunCommand.call_count, 1) - self.assertEqual(heatmap.perf_report, - '/fake/chroot/inchroot_path/perf_report.txt') - - @mock.patch.object(heatmap_generator, 'HeatmapGenerator') - def test_GetHeatMap(self, mock_heatmap_generator): - heatmap = make_heatmap() - heatmap._GetHeatMap(10) - self.assertTrue(mock_heatmap_generator.called) - - @mock.patch.object(heat_map.HeatMapProducer, '_EnsureFileInChroot') - @mock.patch.object(heat_map.HeatMapProducer, '_GeneratePerfReport') - @mock.patch.object(heat_map.HeatMapProducer, '_GetHeatMap') - @mock.patch.object(heat_map.HeatMapProducer, '_RemoveFiles') - def test_Run(self, mock_remove_files, mock_get_heatmap, - mock_generate_perf_report, mock_ensure_file_in_chroot): - heatmap = make_heatmap() - heatmap.Run(10) - mock_ensure_file_in_chroot.assert_called_once_with() - mock_generate_perf_report.assert_called_once_with() - mock_get_heatmap.assert_called_once_with(10) - mock_remove_files.assert_called_once_with() - - @mock.patch.object(heat_map.HeatMapProducer, '_EnsureFileInChroot') - @mock.patch.object( - heat_map.HeatMapProducer, - '_GeneratePerfReport', - new=fake_generate_perf_report_exception) - @mock.patch.object(heat_map.HeatMapProducer, '_GetHeatMap') - @mock.patch.object(heat_map.HeatMapProducer, '_RemoveFiles') - @mock.patch('builtins.print') - def test_Run_with_exception(self, mock_print, mock_remove_files, - mock_get_heatmap, mock_ensure_file_in_chroot): - heatmap = make_heatmap() - with self.assertRaises(Exception): - heatmap.Run(10) - mock_ensure_file_in_chroot.assert_called_once_with() - mock_get_heatmap.assert_not_called() - mock_remove_files.assert_called_once_with() - mock_print.assert_not_called() - - @mock.patch('argparse.ArgumentParser.error', fake_parser_error) - @mock.patch.object(os.path, 'isfile') - @mock.patch.object(heat_map, 'IsARepoRoot') - def test_main_arg_format(self, mock_IsARepoRoot, mock_isfile): - """Test wrong arg format are detected.""" - args = ['--chromeos_root=/fake/chroot/', '--perf_data=/path/to/perf.data'] - - # Test --chromeos_root format - mock_IsARepoRoot.return_value = False - with self.assertRaises(Exception) as msg: - heat_map.main(args) - self.assertIn('does not contain .repo dir.', str(msg.exception)) - - # Test --perf_data format - mock_IsARepoRoot.return_value = True - mock_isfile.return_value = False - with self.assertRaises(Exception) as msg: - heat_map.main(args) - self.assertIn('Cannot find perf_data', str(msg.exception)) - - # Test --hugepage format - mock_isfile.return_value = True - args.append('--hugepage=0') - with self.assertRaises(Exception) as msg: - heat_map.main(args) - self.assertIn('Wrong format of hugepage range', str(msg.exception)) - - # Test --hugepage parse - args[-1] = '--hugepage=0,4096' - heat_map.HeatMapProducer = mock.MagicMock() - heat_map.main(args) - heat_map.HeatMapProducer.assert_called_with( - '/fake/chroot/', '/path/to/perf.data', [0, 4096], None, '') - - -if __name__ == '__main__': - unittest.main() + """All of our tests for heat_map.""" + + # pylint: disable=protected-access + @mock.patch("shutil.copy2") + @mock.patch("tempfile.mkdtemp") + def test_EnsureFileInChrootAlreadyInside(self, mock_mkdtemp, mock_copy): + perf_data_inchroot = ( + "/path/to/fake/chromeos_root/chroot/inchroot_path/perf.data" + ) + heatmap = make_heatmap(perf_data=perf_data_inchroot) + heatmap._EnsureFileInChroot() + self.assertFalse(heatmap.temp_dir_created) + self.assertEqual( + heatmap.temp_dir, + "/path/to/fake/chromeos_root/chroot/inchroot_path/", + ) + self.assertEqual(heatmap.temp_perf_inchroot, "/inchroot_path/") + mock_mkdtemp.assert_not_called() + mock_copy.assert_not_called() + + @mock.patch("shutil.copy2") + @mock.patch("tempfile.mkdtemp", fake_mkdtemp) + def test_EnsureFileInChrootOutsideNeedCopy(self, mock_copy): + heatmap = make_heatmap() + heatmap._EnsureFileInChroot() + self.assertTrue(heatmap.temp_dir_created) + self.assertEqual(mock_copy.call_count, 1) + self.assertEqual( + heatmap.temp_dir, "/path/to/fake/chromeos_root/src/random_dir" + ) + self.assertEqual(heatmap.temp_perf_inchroot, "~/trunk/src/random_dir") + + @mock.patch.object(command_executer.CommandExecuter, "ChrootRunCommand") + def test_GeneratePerfReport(self, mock_ChrootRunCommand): + heatmap = make_heatmap() + heatmap.temp_dir = "/fake/chroot/inchroot_path/" + heatmap.temp_perf_inchroot = "/inchroot_path/" + mock_ChrootRunCommand.return_value = 0 + heatmap._GeneratePerfReport() + cmd = ( + "cd %s && perf report -D -i perf.data > perf_report.txt" + % heatmap.temp_perf_inchroot + ) + mock_ChrootRunCommand.assert_called_with(heatmap.chromeos_root, cmd) + self.assertEqual(mock_ChrootRunCommand.call_count, 1) + self.assertEqual( + heatmap.perf_report, "/fake/chroot/inchroot_path/perf_report.txt" + ) + + @mock.patch.object(heatmap_generator, "HeatmapGenerator") + def test_GetHeatMap(self, mock_heatmap_generator): + heatmap = make_heatmap() + heatmap._GetHeatMap(10) + self.assertTrue(mock_heatmap_generator.called) + + @mock.patch.object(heat_map.HeatMapProducer, "_EnsureFileInChroot") + @mock.patch.object(heat_map.HeatMapProducer, "_GeneratePerfReport") + @mock.patch.object(heat_map.HeatMapProducer, "_GetHeatMap") + @mock.patch.object(heat_map.HeatMapProducer, "_RemoveFiles") + def test_Run( + self, + mock_remove_files, + mock_get_heatmap, + mock_generate_perf_report, + mock_ensure_file_in_chroot, + ): + heatmap = make_heatmap() + heatmap.Run(10) + mock_ensure_file_in_chroot.assert_called_once_with() + mock_generate_perf_report.assert_called_once_with() + mock_get_heatmap.assert_called_once_with(10) + mock_remove_files.assert_called_once_with() + + @mock.patch.object(heat_map.HeatMapProducer, "_EnsureFileInChroot") + @mock.patch.object( + heat_map.HeatMapProducer, + "_GeneratePerfReport", + new=fake_generate_perf_report_exception, + ) + @mock.patch.object(heat_map.HeatMapProducer, "_GetHeatMap") + @mock.patch.object(heat_map.HeatMapProducer, "_RemoveFiles") + @mock.patch("builtins.print") + def test_Run_with_exception( + self, + mock_print, + mock_remove_files, + mock_get_heatmap, + mock_ensure_file_in_chroot, + ): + heatmap = make_heatmap() + with self.assertRaises(Exception): + heatmap.Run(10) + mock_ensure_file_in_chroot.assert_called_once_with() + mock_get_heatmap.assert_not_called() + mock_remove_files.assert_called_once_with() + mock_print.assert_not_called() + + @mock.patch("argparse.ArgumentParser.error", fake_parser_error) + @mock.patch.object(os.path, "isfile") + @mock.patch.object(heat_map, "IsARepoRoot") + def test_main_arg_format(self, mock_IsARepoRoot, mock_isfile): + """Test wrong arg format are detected.""" + args = [ + "--chromeos_root=/fake/chroot/", + "--perf_data=/path/to/perf.data", + ] + + # Test --chromeos_root format + mock_IsARepoRoot.return_value = False + with self.assertRaises(Exception) as msg: + heat_map.main(args) + self.assertIn("does not contain .repo dir.", str(msg.exception)) + + # Test --perf_data format + mock_IsARepoRoot.return_value = True + mock_isfile.return_value = False + with self.assertRaises(Exception) as msg: + heat_map.main(args) + self.assertIn("Cannot find perf_data", str(msg.exception)) + + # Test --hugepage format + mock_isfile.return_value = True + args.append("--hugepage=0") + with self.assertRaises(Exception) as msg: + heat_map.main(args) + self.assertIn("Wrong format of hugepage range", str(msg.exception)) + + # Test --hugepage parse + args[-1] = "--hugepage=0,4096" + heat_map.HeatMapProducer = mock.MagicMock() + heat_map.main(args) + heat_map.HeatMapProducer.assert_called_with( + "/fake/chroot/", "/path/to/perf.data", [0, 4096], None, "" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/heatmaps/heatmap_generator.py b/heatmaps/heatmap_generator.py index 0dd6ad28..703c37d4 100644 --- a/heatmaps/heatmap_generator.py +++ b/heatmaps/heatmap_generator.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2018 The Chromium OS Authors. All rights reserved. +# Copyright 2018 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -13,7 +13,6 @@ performed by another script perf-to-inst-page.sh). It can also analyze the symbol names in hot pages. """ -from __future__ import division, print_function import bisect import collections @@ -23,445 +22,503 @@ import subprocess from cros_utils import command_executer -HugepageRange = collections.namedtuple('HugepageRange', ['start', 'end']) +HugepageRange = collections.namedtuple("HugepageRange", ["start", "end"]) -class MMap(object): - """Class to store mmap information in perf report. - - We assume ASLR is disabled, so MMap for all Chrome is assumed to be - the same. This class deals with the case hugepage creates several - mmaps for Chrome but should be merged together. In these case, we - assume the first MMAP is not affected by the bug and use the MMAP. - """ - - def __init__(self, addr, size, offset): - self.start_address = addr - self.size = size - self.offset = offset - - def __str__(self): - return '(%x, %x, %x)' % (self.start_address, self.size, self.offset) - - def merge(self, mmap): - # This function should not be needed, since we should only have - # one MMAP on Chrome of each process. This function only deals with - # images that is affected by http://crbug.com/931465. - # This function is only checking a few conditions to make sure - # the bug is within our expectation. - - if self.start_address == mmap.start_address: - assert self.size >= mmap.size, \ - 'Original MMAP size(%x) is smaller than the forked process(%x).' % ( - self.size, mmap.size) - # The case that the MMAP is forked from the previous process - # No need to do anything, OR - # The case where hugepage causes a small Chrome mmap. - # In this case, we use the prior MMAP for the whole Chrome - return - - assert self.start_address < mmap.start_address, \ - 'Original MMAP starting address(%x) is larger than the forked' \ - 'process(%x).' % (self.start_address, mmap.start_address) - - assert self.start_address + self.size >= mmap.start_address + mmap.size, \ - 'MMAP of the forked process exceeds the end of original MMAP.' +class MMap(object): + """Class to store mmap information in perf report. + + We assume ASLR is disabled, so MMap for all Chrome is assumed to be + the same. This class deals with the case hugepage creates several + mmaps for Chrome but should be merged together. In these case, we + assume the first MMAP is not affected by the bug and use the MMAP. + """ + + def __init__(self, addr, size, offset): + self.start_address = addr + self.size = size + self.offset = offset + + def __str__(self): + return "(%x, %x, %x)" % (self.start_address, self.size, self.offset) + + def merge(self, mmap): + # This function should not be needed, since we should only have + # one MMAP on Chrome of each process. This function only deals with + # images that is affected by http://crbug.com/931465. + + # This function is only checking a few conditions to make sure + # the bug is within our expectation. + + if self.start_address == mmap.start_address: + assert ( + self.size >= mmap.size + ), "Original MMAP size(%x) is smaller than the forked process(%x)." % ( + self.size, + mmap.size, + ) + # The case that the MMAP is forked from the previous process + # No need to do anything, OR + # The case where hugepage causes a small Chrome mmap. + # In this case, we use the prior MMAP for the whole Chrome + return + + assert self.start_address < mmap.start_address, ( + "Original MMAP starting address(%x) is larger than the forked" + "process(%x)." % (self.start_address, mmap.start_address) + ) + + assert ( + self.start_address + self.size >= mmap.start_address + mmap.size + ), "MMAP of the forked process exceeds the end of original MMAP." class HeatmapGenerator(object): - """Class to generate heat map with a perf report, containing mmaps and - - samples. This class contains two interfaces with other modules: - draw() and analyze(). - - draw() draws a heatmap with the sample information given in the perf report - analyze() prints out the symbol names in hottest pages with the given - chrome binary - """ - - def __init__(self, - perf_report, - page_size, - hugepage, - title, - log_level='verbose'): - self.perf_report = perf_report - # Pick 1G as a relatively large number. All addresses less than it will - # be recorded. The actual heatmap will show up to a boundary of the - # largest address in text segment. - self.max_addr = 1024 * 1024 * 1024 - self.ce = command_executer.GetCommandExecuter(log_level=log_level) - self.dir = os.path.dirname(os.path.realpath(__file__)) - with open(perf_report, 'r', encoding='utf-8') as f: - self.perf_report_contents = f.readlines() - # Write histogram results to a text file, in order to use gnu plot to draw - self.hist_temp_output = open('out.txt', 'w', encoding='utf-8') - self.processes = {} - self.deleted_processes = {} - self.count = 0 - if hugepage: - self.hugepage = HugepageRange(start=hugepage[0], end=hugepage[1]) - else: - self.hugepage = None - self.title = title - self.symbol_addresses = [] - self.symbol_names = [] - self.page_size = page_size - - def _parse_perf_sample(self, line): - # In a perf report, generated with -D, a PERF_RECORD_SAMPLE command should - # look like this: TODO: some arguments are unknown - # - # cpuid cycle unknown [unknown]: PERF_RECORD_SAMPLE(IP, 0x2): pid/tid: - # 0xaddr period: period addr: addr - # ... thread: threadname:tid - # ...... dso: process - # - # This is an example: - # 1 136712833349 0x6a558 [0x30]: PERF_RECORD_SAMPLE(IP, 0x2): 5227/5227: - # 0x55555683b810 period: 372151 addr: 0 - # ... thread: chrome:5227 - # ...... dso: /opt/google/chrome/chrome - # - # For this function, the 7th argument (args[6]) after spltting with spaces - # is pid/tid. We use the combination of the two as the pid. - # Also, we add an assertion here to check the tid in the 7th argument( - # args[6]) and the 15th argument(arg[14]) are the same - # - # The function returns the ((pid,tid), address) pair if the sampling - # is on Chrome. Otherwise, return (None, None) pair. - - if 'thread: chrome' not in line or \ - 'dso: /opt/google/chrome/chrome' not in line: - return None, None - args = line.split(' ') - pid_raw = args[6].split('/') - assert pid_raw[1][:-1] == args[14].split(':')[1][:-1], \ - 'TID in %s of sample is not the same: %s/%s' % ( - line[:-1], pid_raw[1][:-1], args[14].split(':')[1][:-1]) - key = (int(pid_raw[0]), int(pid_raw[1][:-1])) - address = int(args[7], base=16) - return key, address - - def _parse_perf_record(self, line): - # In a perf report, generated with -D, a PERF_RECORD_MMAP2 command should - # look like this: TODO: some arguments are unknown - # - # cpuid cycle unknown [unknown]: PERF_RECORD_MMAP2 pid/tid: - # [0xaddr(0xlength) @ pageoffset maj:min ino ino_generation]: - # permission process - # - # This is an example. - # 2 136690556823 0xa6898 [0x80]: PERF_RECORD_MMAP2 5227/5227: - # [0x555556496000(0x8d1b000) @ 0xf42000 b3:03 92844 1892514370]: - # r-xp /opt/google/chrome/chrome - # - # For this function, the 6th argument (args[5]) after spltting with spaces - # is pid/tid. We use the combination of the two as the pid. - # The 7th argument (args[6]) is the [0xaddr(0xlength). We can peel the - # string to get the address and size of the mmap. - # The 9th argument (args[8]) is the page offset. - # The function returns the ((pid,tid), mmap) pair if the mmap is for Chrome - # is on Chrome. Otherwise, return (None, None) pair. - - if 'chrome/chrome' not in line: - return None, None - args = line.split(' ') - pid_raw = args[5].split('/') - assert pid_raw[0] == pid_raw[1][:-1], \ - 'PID in %s of mmap is not the same: %s/%s' % ( - line[:-1], pid_raw[0], pid_raw[1]) - pid = (int(pid_raw[0]), int(pid_raw[1][:-1])) - address_raw = args[6].split('(') - start_address = int(address_raw[0][1:], base=16) - size = int(address_raw[1][:-1], base=16) - offset = int(args[8], base=16) - # Return an mmap object instead of only starting address, - # in case there are many mmaps for the sample PID - return pid, MMap(start_address, size, offset) - - def _parse_pair_event(self, arg): - # This function is called by the _parse_* functions that has a pattern of - # pids like: (pid:tid):(pid:tid), i.e. - # PERF_RECORD_FORK and PERF_RECORD_COMM - _, remain = arg.split('(', 1) - pid1, remain = remain.split(':', 1) - pid2, remain = remain.split(')', 1) - _, remain = remain.split('(', 1) - pid3, remain = remain.split(':', 1) - pid4, remain = remain.split(')', 1) - return (int(pid1), int(pid2)), (int(pid3), int(pid4)) - - def _process_perf_record(self, line): - # This function calls _parse_perf_record() to get information from - # PERF_RECORD_MMAP2. It records the mmap object for each pid (a pair of - # pid,tid), into a dictionary. - pid, mmap = self._parse_perf_record(line) - if pid is None: - # PID = None meaning the mmap is not for chrome - return - if pid in self.processes: - # This should never happen for a correct profiling result, as we - # should only have one MMAP for Chrome for each process. - # If it happens, see http://crbug.com/931465 - self.processes[pid].merge(mmap) - else: - self.processes[pid] = mmap - - def _process_perf_fork(self, line): - # In a perf report, generated with -D, a PERF_RECORD_FORK command should - # look like this: - # - # cpuid cycle unknown [unknown]: - # PERF_RECORD_FORK(pid_to:tid_to):(pid_from:tid_from) - # - # This is an example. - # 0 0 0x22a8 [0x38]: PERF_RECORD_FORK(1:1):(0:0) - # - # In this function, we need to peel the information of pid:tid pairs - # So we get the last argument and send it to function _parse_pair_event() - # for analysis. - # We use (pid, tid) as the pid. - args = line.split(' ') - pid_to, pid_from = self._parse_pair_event(args[-1]) - if pid_from in self.processes: - assert pid_to not in self.processes - self.processes[pid_to] = MMap(self.processes[pid_from].start_address, - self.processes[pid_from].size, - self.processes[pid_from].offset) - - def _process_perf_exit(self, line): - # In a perf report, generated with -D, a PERF_RECORD_EXIT command should - # look like this: - # - # cpuid cycle unknown [unknown]: - # PERF_RECORD_EXIT(pid1:tid1):(pid2:tid2) - # - # This is an example. - # 1 136082505621 0x30810 [0x38]: PERF_RECORD_EXIT(3851:3851):(3851:3851) - # - # In this function, we need to peel the information of pid:tid pairs - # So we get the last argument and send it to function _parse_pair_event() - # for analysis. - # We use (pid, tid) as the pid. - args = line.split(' ') - pid_to, pid_from = self._parse_pair_event(args[-1]) - assert pid_to == pid_from, '(%d, %d) (%d, %d)' % (pid_to[0], pid_to[1], - pid_from[0], pid_from[1]) - if pid_to in self.processes: - # Don't delete the process yet - self.deleted_processes[pid_from] = self.processes[pid_from] - - def _process_perf_sample(self, line): - # This function calls _parse_perf_sample() to get information from - # the perf report. - # It needs to check the starting address of allocated mmap from - # the dictionary (self.processes) to calculate the offset within - # the text section of the sampling. - # The offset is calculated into pages (4KB or 2MB) and writes into - # out.txt together with the total counts, which will be used to - # calculate histogram. - pid, addr = self._parse_perf_sample(line) - if pid is None: - return - - assert pid in self.processes and pid not in self.deleted_processes, \ - 'PID %d not found mmap and not forked from another process' - - start_address = self.processes[pid].start_address - address = addr - start_address - assert address >= 0 and \ - 'addresses accessed in PERF_RECORD_SAMPLE should be larger than' \ - ' the starting address of Chrome' - if address < self.max_addr: - self.count += 1 - line = '%d/%d: %d %d' % (pid[0], pid[1], self.count, - address // self.page_size * self.page_size) - if self.hugepage: - if self.hugepage.start <= address < self.hugepage.end: - line += ' hugepage' + """Class to generate heat map with a perf report, containing mmaps and + + samples. This class contains two interfaces with other modules: + draw() and analyze(). + + draw() draws a heatmap with the sample information given in the perf report + analyze() prints out the symbol names in hottest pages with the given + chrome binary + """ + + def __init__( + self, perf_report, page_size, hugepage, title, log_level="verbose" + ): + self.perf_report = perf_report + # Pick 1G as a relatively large number. All addresses less than it will + # be recorded. The actual heatmap will show up to a boundary of the + # largest address in text segment. + self.max_addr = 1024 * 1024 * 1024 + self.ce = command_executer.GetCommandExecuter(log_level=log_level) + self.dir = os.path.dirname(os.path.realpath(__file__)) + with open(perf_report, "r", encoding="utf-8") as f: + self.perf_report_contents = f.readlines() + # Write histogram results to a text file, in order to use gnu plot to draw + self.hist_temp_output = open("out.txt", "w", encoding="utf-8") + self.processes = {} + self.deleted_processes = {} + self.count = 0 + if hugepage: + self.hugepage = HugepageRange(start=hugepage[0], end=hugepage[1]) + else: + self.hugepage = None + self.title = title + self.symbol_addresses = [] + self.symbol_names = [] + self.page_size = page_size + + def _parse_perf_sample(self, line): + # In a perf report, generated with -D, a PERF_RECORD_SAMPLE command should + # look like this: TODO: some arguments are unknown + # + # cpuid cycle unknown [unknown]: PERF_RECORD_SAMPLE(IP, 0x2): pid/tid: + # 0xaddr period: period addr: addr + # ... thread: threadname:tid + # ...... dso: process + # + # This is an example: + # 1 136712833349 0x6a558 [0x30]: PERF_RECORD_SAMPLE(IP, 0x2): 5227/5227: + # 0x55555683b810 period: 372151 addr: 0 + # ... thread: chrome:5227 + # ...... dso: /opt/google/chrome/chrome + # + # For this function, the 7th argument (args[6]) after spltting with spaces + # is pid/tid. We use the combination of the two as the pid. + # Also, we add an assertion here to check the tid in the 7th argument( + # args[6]) and the 15th argument(arg[14]) are the same + # + # The function returns the ((pid,tid), address) pair if the sampling + # is on Chrome. Otherwise, return (None, None) pair. + + if ( + "thread: chrome" not in line + or "dso: /opt/google/chrome/chrome" not in line + ): + return None, None + args = line.split(" ") + pid_raw = args[6].split("/") + assert ( + pid_raw[1][:-1] == args[14].split(":")[1][:-1] + ), "TID in %s of sample is not the same: %s/%s" % ( + line[:-1], + pid_raw[1][:-1], + args[14].split(":")[1][:-1], + ) + key = (int(pid_raw[0]), int(pid_raw[1][:-1])) + address = int(args[7], base=16) + return key, address + + def _parse_perf_record(self, line): + # In a perf report, generated with -D, a PERF_RECORD_MMAP2 command should + # look like this: TODO: some arguments are unknown + # + # cpuid cycle unknown [unknown]: PERF_RECORD_MMAP2 pid/tid: + # [0xaddr(0xlength) @ pageoffset maj:min ino ino_generation]: + # permission process + # + # This is an example. + # 2 136690556823 0xa6898 [0x80]: PERF_RECORD_MMAP2 5227/5227: + # [0x555556496000(0x8d1b000) @ 0xf42000 b3:03 92844 1892514370]: + # r-xp /opt/google/chrome/chrome + # + # For this function, the 6th argument (args[5]) after spltting with spaces + # is pid/tid. We use the combination of the two as the pid. + # The 7th argument (args[6]) is the [0xaddr(0xlength). We can peel the + # string to get the address and size of the mmap. + # The 9th argument (args[8]) is the page offset. + # The function returns the ((pid,tid), mmap) pair if the mmap is for Chrome + # is on Chrome. Otherwise, return (None, None) pair. + + if "chrome/chrome" not in line: + return None, None + args = line.split(" ") + pid_raw = args[5].split("/") + assert ( + pid_raw[0] == pid_raw[1][:-1] + ), "PID in %s of mmap is not the same: %s/%s" % ( + line[:-1], + pid_raw[0], + pid_raw[1], + ) + pid = (int(pid_raw[0]), int(pid_raw[1][:-1])) + address_raw = args[6].split("(") + start_address = int(address_raw[0][1:], base=16) + size = int(address_raw[1][:-1], base=16) + offset = int(args[8], base=16) + # Return an mmap object instead of only starting address, + # in case there are many mmaps for the sample PID + return pid, MMap(start_address, size, offset) + + def _parse_pair_event(self, arg): + # This function is called by the _parse_* functions that has a pattern of + # pids like: (pid:tid):(pid:tid), i.e. + # PERF_RECORD_FORK and PERF_RECORD_COMM + _, remain = arg.split("(", 1) + pid1, remain = remain.split(":", 1) + pid2, remain = remain.split(")", 1) + _, remain = remain.split("(", 1) + pid3, remain = remain.split(":", 1) + pid4, remain = remain.split(")", 1) + return (int(pid1), int(pid2)), (int(pid3), int(pid4)) + + def _process_perf_record(self, line): + # This function calls _parse_perf_record() to get information from + # PERF_RECORD_MMAP2. It records the mmap object for each pid (a pair of + # pid,tid), into a dictionary. + pid, mmap = self._parse_perf_record(line) + if pid is None: + # PID = None meaning the mmap is not for chrome + return + if pid in self.processes: + # This should never happen for a correct profiling result, as we + # should only have one MMAP for Chrome for each process. + # If it happens, see http://crbug.com/931465 + self.processes[pid].merge(mmap) else: - line += ' smallpage' - print(line, file=self.hist_temp_output) - - def _read_perf_report(self): - # Serve as main function to read perf report, generated by -D - lines = iter(self.perf_report_contents) - for line in lines: - if 'PERF_RECORD_MMAP' in line: - self._process_perf_record(line) - elif 'PERF_RECORD_FORK' in line: - self._process_perf_fork(line) - elif 'PERF_RECORD_EXIT' in line: - self._process_perf_exit(line) - elif 'PERF_RECORD_SAMPLE' in line: - # Perf sample is multi-line - self._process_perf_sample(line + next(lines) + next(lines)) - self.hist_temp_output.close() - - def _draw_heat_map(self): - # Calls a script (perf-to-inst-page.sh) to calculate histogram - # of results written in out.txt and also generate pngs for - # heat maps. - heatmap_script = os.path.join(self.dir, 'perf-to-inst-page.sh') - if self.hugepage: - hp_arg = 'hugepage' - else: - hp_arg = 'none' - - cmd = '{0} {1} {2}'.format(heatmap_script, pipes.quote(self.title), hp_arg) - retval = self.ce.RunCommand(cmd) - if retval: - raise RuntimeError('Failed to run script to generate heatmap') - - def _restore_histogram(self): - # When hugepage is used, there are two files inst-histo-{hp,sp}.txt - # So we need to read in all the files. - names = [x for x in os.listdir('.') if 'inst-histo' in x and '.txt' in x] - hist = {} - for n in names: - with open(n, encoding='utf-8') as f: - for l in f.readlines(): - num, addr = l.strip().split(' ') - assert int(addr) not in hist - hist[int(addr)] = int(num) - return hist - - def _read_symbols_from_binary(self, binary): - # FIXME: We are using nm to read symbol names from Chrome binary - # for now. Can we get perf to hand us symbol names, instead of - # using nm in the future? - # - # Get all the symbols (and their starting addresses) that fall into - # the page. Will be used to print out information of hot pages - # Each line shows the information of a symbol: - # [symbol value (0xaddr)] [symbol type] [symbol name] - # For some symbols, the [symbol name] field might be missing. - # e.g. - # 0000000001129da0 t Builtins_LdaNamedPropertyHandler - - # Generate a list of symbols from nm tool and check each line - # to extract symbols names - text_section_start = 0 - for l in subprocess.check_output(['nm', '-n', binary]).split('\n'): - args = l.strip().split(' ') - if len(args) < 3: - # No name field - continue - addr_raw, symbol_type, name = args - addr = int(addr_raw, base=16) - if 't' not in symbol_type and 'T' not in symbol_type: - # Filter out symbols not in text sections - continue - if not self.symbol_addresses: - # The first symbol in text sections - text_section_start = addr - self.symbol_addresses.append(0) - self.symbol_names.append(name) - else: - assert text_section_start != 0, \ - 'The starting address of text section has not been found' - if addr == self.symbol_addresses[-1]: - # if the same address has multiple symbols, put them together - # and separate symbol names with '/' - self.symbol_names[-1] += '/' + name + self.processes[pid] = mmap + + def _process_perf_fork(self, line): + # In a perf report, generated with -D, a PERF_RECORD_FORK command should + # look like this: + # + # cpuid cycle unknown [unknown]: + # PERF_RECORD_FORK(pid_to:tid_to):(pid_from:tid_from) + # + # This is an example. + # 0 0 0x22a8 [0x38]: PERF_RECORD_FORK(1:1):(0:0) + # + # In this function, we need to peel the information of pid:tid pairs + # So we get the last argument and send it to function _parse_pair_event() + # for analysis. + # We use (pid, tid) as the pid. + args = line.split(" ") + pid_to, pid_from = self._parse_pair_event(args[-1]) + if pid_from in self.processes: + assert pid_to not in self.processes + self.processes[pid_to] = MMap( + self.processes[pid_from].start_address, + self.processes[pid_from].size, + self.processes[pid_from].offset, + ) + + def _process_perf_exit(self, line): + # In a perf report, generated with -D, a PERF_RECORD_EXIT command should + # look like this: + # + # cpuid cycle unknown [unknown]: + # PERF_RECORD_EXIT(pid1:tid1):(pid2:tid2) + # + # This is an example. + # 1 136082505621 0x30810 [0x38]: PERF_RECORD_EXIT(3851:3851):(3851:3851) + # + # In this function, we need to peel the information of pid:tid pairs + # So we get the last argument and send it to function _parse_pair_event() + # for analysis. + # We use (pid, tid) as the pid. + args = line.split(" ") + pid_to, pid_from = self._parse_pair_event(args[-1]) + assert pid_to == pid_from, "(%d, %d) (%d, %d)" % ( + pid_to[0], + pid_to[1], + pid_from[0], + pid_from[1], + ) + if pid_to in self.processes: + # Don't delete the process yet + self.deleted_processes[pid_from] = self.processes[pid_from] + + def _process_perf_sample(self, line): + # This function calls _parse_perf_sample() to get information from + # the perf report. + # It needs to check the starting address of allocated mmap from + # the dictionary (self.processes) to calculate the offset within + # the text section of the sampling. + # The offset is calculated into pages (4KB or 2MB) and writes into + # out.txt together with the total counts, which will be used to + # calculate histogram. + pid, addr = self._parse_perf_sample(line) + if pid is None: + return + + assert ( + pid in self.processes and pid not in self.deleted_processes + ), "PID %d not found mmap and not forked from another process" + + start_address = self.processes[pid].start_address + address = addr - start_address + assert ( + address >= 0 + and "addresses accessed in PERF_RECORD_SAMPLE should be larger than" + " the starting address of Chrome" + ) + if address < self.max_addr: + self.count += 1 + line = "%d/%d: %d %d" % ( + pid[0], + pid[1], + self.count, + address // self.page_size * self.page_size, + ) + if self.hugepage: + if self.hugepage.start <= address < self.hugepage.end: + line += " hugepage" + else: + line += " smallpage" + print(line, file=self.hist_temp_output) + + def _read_perf_report(self): + # Serve as main function to read perf report, generated by -D + lines = iter(self.perf_report_contents) + for line in lines: + if "PERF_RECORD_MMAP" in line: + self._process_perf_record(line) + elif "PERF_RECORD_FORK" in line: + self._process_perf_fork(line) + elif "PERF_RECORD_EXIT" in line: + self._process_perf_exit(line) + elif "PERF_RECORD_SAMPLE" in line: + # Perf sample is multi-line + self._process_perf_sample(line + next(lines) + next(lines)) + self.hist_temp_output.close() + + def _draw_heat_map(self): + # Calls a script (perf-to-inst-page.sh) to calculate histogram + # of results written in out.txt and also generate pngs for + # heat maps. + heatmap_script = os.path.join(self.dir, "perf-to-inst-page.sh") + if self.hugepage: + hp_arg = "hugepage" else: - # The output of nm -n command is already sorted by address - # Insert to the end will result in a sorted array for bisect - self.symbol_addresses.append(addr - text_section_start) - self.symbol_names.append(name) - - def _map_addr_to_symbol(self, addr): - # Find out the symbol name - assert self.symbol_addresses - index = bisect.bisect(self.symbol_addresses, addr) - assert 0 < index <= len(self.symbol_names), \ - 'Failed to find an index (%d) in the list (len=%d)' % ( - index, len(self.symbol_names)) - return self.symbol_names[index - 1] - - def _print_symbols_in_hot_pages(self, fp, pages_to_show): - # Print symbols in all the pages of interest - for page_num, sample_num in pages_to_show: - print( - '----------------------------------------------------------', file=fp) - print( - 'Page Offset: %d MB, Count: %d' % (page_num // 1024 // 1024, - sample_num), - file=fp) - - symbol_counts = collections.Counter() - # Read Sample File and find out the occurance of symbols in the page - lines = iter(self.perf_report_contents) - for line in lines: - if 'PERF_RECORD_SAMPLE' in line: - pid, addr = self._parse_perf_sample(line + next(lines) + next(lines)) - if pid is None: - # The sampling is not on Chrome - continue - if addr // self.page_size != ( - self.processes[pid].start_address + page_num) // self.page_size: - # Sampling not in the current page - continue - - name = self._map_addr_to_symbol(addr - - self.processes[pid].start_address) - assert name, 'Failed to find symbol name of addr %x' % addr - symbol_counts[name] += 1 - - assert sum(symbol_counts.values()) == sample_num, \ - 'Symbol name matching missing for some addresses: %d vs %d' % ( - sum(symbol_counts.values()), sample_num) - - # Print out the symbol names sorted by the number of samples in - # the page - for name, count in sorted( - symbol_counts.items(), key=lambda kv: kv[1], reverse=True): - if count == 0: - break - print('> %s : %d' % (name, count), file=fp) - print('\n\n', file=fp) - - def draw(self): - # First read perf report to process information and save histogram - # into a text file - self._read_perf_report() - # Then use gnu plot to draw heat map - self._draw_heat_map() - - def analyze(self, binary, top_n): - # Read histogram from histo.txt - hist = self._restore_histogram() - # Sort the pages in histogram - sorted_hist = sorted(hist.items(), key=lambda value: value[1], reverse=True) - - # Generate symbolizations - self._read_symbols_from_binary(binary) - - # Write hottest pages - with open('addr2symbol.txt', 'w', encoding='utf-8') as fp: - if self.hugepage: - # Print hugepage region first - print( - 'Hugepage top %d hot pages (%d MB - %d MB):' % - (top_n, self.hugepage.start // 1024 // 1024, - self.hugepage.end // 1024 // 1024), - file=fp) - pages_to_print = [(k, v) - for k, v in sorted_hist - if self.hugepage.start <= k < self.hugepage.end - ][:top_n] - self._print_symbols_in_hot_pages(fp, pages_to_print) - print('==========================================', file=fp) - print('Top %d hot pages landed outside of hugepage:' % top_n, file=fp) - # Then print outside pages - pages_to_print = [(k, v) - for k, v in sorted_hist - if k < self.hugepage.start or k >= self.hugepage.end - ][:top_n] - self._print_symbols_in_hot_pages(fp, pages_to_print) - else: - # Print top_n hottest pages. - pages_to_print = sorted_hist[:top_n] - self._print_symbols_in_hot_pages(fp, pages_to_print) + hp_arg = "none" + + cmd = "{0} {1} {2}".format( + heatmap_script, pipes.quote(self.title), hp_arg + ) + retval = self.ce.RunCommand(cmd) + if retval: + raise RuntimeError("Failed to run script to generate heatmap") + + def _restore_histogram(self): + # When hugepage is used, there are two files inst-histo-{hp,sp}.txt + # So we need to read in all the files. + names = [ + x for x in os.listdir(".") if "inst-histo" in x and ".txt" in x + ] + hist = {} + for n in names: + with open(n, encoding="utf-8") as f: + for l in f.readlines(): + num, addr = l.strip().split(" ") + assert int(addr) not in hist + hist[int(addr)] = int(num) + return hist + + def _read_symbols_from_binary(self, binary): + # FIXME: We are using nm to read symbol names from Chrome binary + # for now. Can we get perf to hand us symbol names, instead of + # using nm in the future? + # + # Get all the symbols (and their starting addresses) that fall into + # the page. Will be used to print out information of hot pages + # Each line shows the information of a symbol: + # [symbol value (0xaddr)] [symbol type] [symbol name] + # For some symbols, the [symbol name] field might be missing. + # e.g. + # 0000000001129da0 t Builtins_LdaNamedPropertyHandler + + # Generate a list of symbols from nm tool and check each line + # to extract symbols names + text_section_start = 0 + for l in subprocess.check_output(["nm", "-n", binary]).split("\n"): + args = l.strip().split(" ") + if len(args) < 3: + # No name field + continue + addr_raw, symbol_type, name = args + addr = int(addr_raw, base=16) + if "t" not in symbol_type and "T" not in symbol_type: + # Filter out symbols not in text sections + continue + if not self.symbol_addresses: + # The first symbol in text sections + text_section_start = addr + self.symbol_addresses.append(0) + self.symbol_names.append(name) + else: + assert ( + text_section_start != 0 + ), "The starting address of text section has not been found" + if addr == self.symbol_addresses[-1]: + # if the same address has multiple symbols, put them together + # and separate symbol names with '/' + self.symbol_names[-1] += "/" + name + else: + # The output of nm -n command is already sorted by address + # Insert to the end will result in a sorted array for bisect + self.symbol_addresses.append(addr - text_section_start) + self.symbol_names.append(name) + + def _map_addr_to_symbol(self, addr): + # Find out the symbol name + assert self.symbol_addresses + index = bisect.bisect(self.symbol_addresses, addr) + assert ( + 0 < index <= len(self.symbol_names) + ), "Failed to find an index (%d) in the list (len=%d)" % ( + index, + len(self.symbol_names), + ) + return self.symbol_names[index - 1] + + def _print_symbols_in_hot_pages(self, fp, pages_to_show): + # Print symbols in all the pages of interest + for page_num, sample_num in pages_to_show: + print( + "----------------------------------------------------------", + file=fp, + ) + print( + "Page Offset: %d MB, Count: %d" + % (page_num // 1024 // 1024, sample_num), + file=fp, + ) + + symbol_counts = collections.Counter() + # Read Sample File and find out the occurance of symbols in the page + lines = iter(self.perf_report_contents) + for line in lines: + if "PERF_RECORD_SAMPLE" in line: + pid, addr = self._parse_perf_sample( + line + next(lines) + next(lines) + ) + if pid is None: + # The sampling is not on Chrome + continue + if ( + addr // self.page_size + != (self.processes[pid].start_address + page_num) + // self.page_size + ): + # Sampling not in the current page + continue + + name = self._map_addr_to_symbol( + addr - self.processes[pid].start_address + ) + assert name, "Failed to find symbol name of addr %x" % addr + symbol_counts[name] += 1 + + assert ( + sum(symbol_counts.values()) == sample_num + ), "Symbol name matching missing for some addresses: %d vs %d" % ( + sum(symbol_counts.values()), + sample_num, + ) + + # Print out the symbol names sorted by the number of samples in + # the page + for name, count in sorted( + symbol_counts.items(), key=lambda kv: kv[1], reverse=True + ): + if count == 0: + break + print("> %s : %d" % (name, count), file=fp) + print("\n\n", file=fp) + + def draw(self): + # First read perf report to process information and save histogram + # into a text file + self._read_perf_report() + # Then use gnu plot to draw heat map + self._draw_heat_map() + + def analyze(self, binary, top_n): + # Read histogram from histo.txt + hist = self._restore_histogram() + # Sort the pages in histogram + sorted_hist = sorted( + hist.items(), key=lambda value: value[1], reverse=True + ) + + # Generate symbolizations + self._read_symbols_from_binary(binary) + + # Write hottest pages + with open("addr2symbol.txt", "w", encoding="utf-8") as fp: + if self.hugepage: + # Print hugepage region first + print( + "Hugepage top %d hot pages (%d MB - %d MB):" + % ( + top_n, + self.hugepage.start // 1024 // 1024, + self.hugepage.end // 1024 // 1024, + ), + file=fp, + ) + pages_to_print = [ + (k, v) + for k, v in sorted_hist + if self.hugepage.start <= k < self.hugepage.end + ][:top_n] + self._print_symbols_in_hot_pages(fp, pages_to_print) + print("==========================================", file=fp) + print( + "Top %d hot pages landed outside of hugepage:" % top_n, + file=fp, + ) + # Then print outside pages + pages_to_print = [ + (k, v) + for k, v in sorted_hist + if k < self.hugepage.start or k >= self.hugepage.end + ][:top_n] + self._print_symbols_in_hot_pages(fp, pages_to_print) + else: + # Print top_n hottest pages. + pages_to_print = sorted_hist[:top_n] + self._print_symbols_in_hot_pages(fp, pages_to_print) diff --git a/heatmaps/heatmap_generator_test.py b/heatmaps/heatmap_generator_test.py index 5008c653..898c7370 100755 --- a/heatmaps/heatmap_generator_test.py +++ b/heatmaps/heatmap_generator_test.py @@ -1,316 +1,326 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2018 The Chromium OS Authors. All rights reserved. +# Copyright 2018 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Tests for heatmap_generator.py.""" -from __future__ import division, print_function - -import unittest.mock as mock -import unittest import os +import unittest +import unittest.mock as mock from heatmaps import heatmap_generator def _write_perf_mmap(pid, tid, addr, size, fp): - print( - '0 0 0 0 PERF_RECORD_MMAP2 %d/%d: ' - '[%x(%x) @ 0x0 0:0 0 0] ' - 'r-xp /opt/google/chrome/chrome\n' % (pid, tid, addr, size), - file=fp) + print( + "0 0 0 0 PERF_RECORD_MMAP2 %d/%d: " + "[%x(%x) @ 0x0 0:0 0 0] " + "r-xp /opt/google/chrome/chrome\n" % (pid, tid, addr, size), + file=fp, + ) def _write_perf_fork(pid_from, tid_from, pid_to, tid_to, fp): - print( - '0 0 0 0 PERF_RECORD_FORK(%d:%d):(%d:%d)\n' % (pid_to, tid_to, pid_from, - tid_from), - file=fp) + print( + "0 0 0 0 PERF_RECORD_FORK(%d:%d):(%d:%d)\n" + % (pid_to, tid_to, pid_from, tid_from), + file=fp, + ) def _write_perf_exit(pid_from, tid_from, pid_to, tid_to, fp): - print( - '0 0 0 0 PERF_RECORD_EXIT(%d:%d):(%d:%d)\n' % (pid_to, tid_to, pid_from, - tid_from), - file=fp) + print( + "0 0 0 0 PERF_RECORD_EXIT(%d:%d):(%d:%d)\n" + % (pid_to, tid_to, pid_from, tid_from), + file=fp, + ) def _write_perf_sample(pid, tid, addr, fp): - print( - '0 0 0 0 PERF_RECORD_SAMPLE(IP, 0x2): ' - '%d/%d: %x period: 100000 addr: 0' % (pid, tid, addr), - file=fp) - print(' ... thread: chrome:%d' % tid, file=fp) - print(' ...... dso: /opt/google/chrome/chrome\n', file=fp) + print( + "0 0 0 0 PERF_RECORD_SAMPLE(IP, 0x2): " + "%d/%d: %x period: 100000 addr: 0" % (pid, tid, addr), + file=fp, + ) + print(" ... thread: chrome:%d" % tid, file=fp) + print(" ...... dso: /opt/google/chrome/chrome\n", file=fp) def _heatmap(file_name, page_size=4096, hugepage=None, analyze=False, top_n=10): - generator = heatmap_generator.HeatmapGenerator( - file_name, page_size, hugepage, '', - log_level='none') # Don't log to stdout - generator.draw() - if analyze: - generator.analyze('/path/to/chrome', top_n) + generator = heatmap_generator.HeatmapGenerator( + file_name, page_size, hugepage, "", log_level="none" + ) # Don't log to stdout + generator.draw() + if analyze: + generator.analyze("/path/to/chrome", top_n) def _cleanup(file_name): - files = [ - file_name, 'out.txt', 'inst-histo.txt', 'inst-histo-hp.txt', - 'inst-histo-sp.txt', 'heat_map.png', 'timeline.png', 'addr2symbol.txt' - ] - for f in files: - if os.path.exists(f): - os.remove(f) + files = [ + file_name, + "out.txt", + "inst-histo.txt", + "inst-histo-hp.txt", + "inst-histo-sp.txt", + "heat_map.png", + "timeline.png", + "addr2symbol.txt", + ] + for f in files: + if os.path.exists(f): + os.remove(f) class HeatmapGeneratorDrawTests(unittest.TestCase): - """All of our tests for heatmap_generator.draw() and related.""" - - def test_with_one_mmap_one_sample(self): - """Tests one perf record and one sample.""" - fname = 'test.txt' - with open(fname, 'w') as f: - _write_perf_mmap(101, 101, 0xABCD000, 0x100, f) - _write_perf_sample(101, 101, 0xABCD101, f) - self.addCleanup(_cleanup, fname) - _heatmap(fname) - self.assertIn('out.txt', os.listdir('.')) - with open('out.txt') as f: - lines = f.readlines() - self.assertEqual(len(lines), 1) - self.assertIn('101/101: 1 0', lines[0]) - - def test_with_one_mmap_multiple_samples(self): - """Tests one perf record and three samples.""" - fname = 'test.txt' - with open(fname, 'w') as f: - _write_perf_mmap(101, 101, 0xABCD000, 0x100, f) - _write_perf_sample(101, 101, 0xABCD101, f) - _write_perf_sample(101, 101, 0xABCD102, f) - _write_perf_sample(101, 101, 0xABCE102, f) - self.addCleanup(_cleanup, fname) - _heatmap(fname) - self.assertIn('out.txt', os.listdir('.')) - with open('out.txt') as f: - lines = f.readlines() - self.assertEqual(len(lines), 3) - self.assertIn('101/101: 1 0', lines[0]) - self.assertIn('101/101: 2 0', lines[1]) - self.assertIn('101/101: 3 4096', lines[2]) - - def test_with_fork_and_exit(self): - """Tests perf fork and perf exit.""" - fname = 'test_fork.txt' - with open(fname, 'w') as f: - _write_perf_mmap(101, 101, 0xABCD000, 0x100, f) - _write_perf_fork(101, 101, 202, 202, f) - _write_perf_sample(101, 101, 0xABCD101, f) - _write_perf_sample(202, 202, 0xABCE101, f) - _write_perf_exit(202, 202, 202, 202, f) - self.addCleanup(_cleanup, fname) - _heatmap(fname) - self.assertIn('out.txt', os.listdir('.')) - with open('out.txt') as f: - lines = f.readlines() - self.assertEqual(len(lines), 2) - self.assertIn('101/101: 1 0', lines[0]) - self.assertIn('202/202: 2 4096', lines[1]) - - def test_hugepage_creates_two_chrome_mmaps(self): - """Test two chrome mmaps for the same process.""" - fname = 'test_hugepage.txt' - with open(fname, 'w') as f: - _write_perf_mmap(101, 101, 0xABCD000, 0x1000, f) - _write_perf_fork(101, 101, 202, 202, f) - _write_perf_mmap(202, 202, 0xABCD000, 0x100, f) - _write_perf_mmap(202, 202, 0xABCD300, 0xD00, f) - _write_perf_sample(101, 101, 0xABCD102, f) - _write_perf_sample(202, 202, 0xABCD102, f) - self.addCleanup(_cleanup, fname) - _heatmap(fname) - self.assertIn('out.txt', os.listdir('.')) - with open('out.txt') as f: - lines = f.readlines() - self.assertEqual(len(lines), 2) - self.assertIn('101/101: 1 0', lines[0]) - self.assertIn('202/202: 2 0', lines[1]) - - def test_hugepage_creates_two_chrome_mmaps_fail(self): - """Test two chrome mmaps for the same process.""" - fname = 'test_hugepage.txt' - # Cases where first_mmap.size < second_mmap.size - with open(fname, 'w') as f: - _write_perf_mmap(101, 101, 0xABCD000, 0x1000, f) - _write_perf_fork(101, 101, 202, 202, f) - _write_perf_mmap(202, 202, 0xABCD000, 0x10000, f) - self.addCleanup(_cleanup, fname) - with self.assertRaises(AssertionError) as msg: - _heatmap(fname) - self.assertIn('Original MMAP size', str(msg.exception)) - - # Cases where first_mmap.address > second_mmap.address - with open(fname, 'w') as f: - _write_perf_mmap(101, 101, 0xABCD000, 0x1000, f) - _write_perf_fork(101, 101, 202, 202, f) - _write_perf_mmap(202, 202, 0xABCC000, 0x10000, f) - with self.assertRaises(AssertionError) as msg: - _heatmap(fname) - self.assertIn('Original MMAP starting address', str(msg.exception)) - - # Cases where first_mmap.address + size < - # second_mmap.address + second_mmap.size - with open(fname, 'w') as f: - _write_perf_mmap(101, 101, 0xABCD000, 0x1000, f) - _write_perf_fork(101, 101, 202, 202, f) - _write_perf_mmap(202, 202, 0xABCD100, 0x10000, f) - with self.assertRaises(AssertionError) as msg: - _heatmap(fname) - self.assertIn('exceeds the end of original MMAP', str(msg.exception)) - - def test_histogram(self): - """Tests if the tool can generate correct histogram. - - In the tool, histogram is generated from statistics - of perf samples (saved to out.txt). The histogram is - generated by perf-to-inst-page.sh and saved to - inst-histo.txt. It will be used to draw heat maps. - """ - fname = 'test_histo.txt' - with open(fname, 'w') as f: - _write_perf_mmap(101, 101, 0xABCD000, 0x100, f) - for i in range(100): - _write_perf_sample(101, 101, 0xABCD000 + i, f) - _write_perf_sample(101, 101, 0xABCE000 + i, f) - _write_perf_sample(101, 101, 0xABFD000 + i, f) - _write_perf_sample(101, 101, 0xAFCD000 + i, f) - self.addCleanup(_cleanup, fname) - _heatmap(fname) - self.assertIn('inst-histo.txt', os.listdir('.')) - with open('inst-histo.txt') as f: - lines = f.readlines() - self.assertEqual(len(lines), 4) - self.assertIn('100 0', lines[0]) - self.assertIn('100 4096', lines[1]) - self.assertIn('100 196608', lines[2]) - self.assertIn('100 4194304', lines[3]) - - def test_histogram_two_mb_page(self): - """Tests handling of 2MB page.""" - fname = 'test_histo.txt' - with open(fname, 'w') as f: - _write_perf_mmap(101, 101, 0xABCD000, 0x100, f) - for i in range(100): - _write_perf_sample(101, 101, 0xABCD000 + i, f) - _write_perf_sample(101, 101, 0xABCE000 + i, f) - _write_perf_sample(101, 101, 0xABFD000 + i, f) - _write_perf_sample(101, 101, 0xAFCD000 + i, f) - self.addCleanup(_cleanup, fname) - _heatmap(fname, page_size=2 * 1024 * 1024) - self.assertIn('inst-histo.txt', os.listdir('.')) - with open('inst-histo.txt') as f: - lines = f.readlines() - self.assertEqual(len(lines), 2) - self.assertIn('300 0', lines[0]) - self.assertIn('100 4194304', lines[1]) - - def test_histogram_in_and_out_hugepage(self): - """Tests handling the case of separating samples in and out huge page.""" - fname = 'test_histo.txt' - with open(fname, 'w') as f: - _write_perf_mmap(101, 101, 0xABCD000, 0x100, f) - for i in range(100): - _write_perf_sample(101, 101, 0xABCD000 + i, f) - _write_perf_sample(101, 101, 0xABCE000 + i, f) - _write_perf_sample(101, 101, 0xABFD000 + i, f) - _write_perf_sample(101, 101, 0xAFCD000 + i, f) - self.addCleanup(_cleanup, fname) - _heatmap(fname, hugepage=[0, 8192]) - file_list = os.listdir('.') - self.assertNotIn('inst-histo.txt', file_list) - self.assertIn('inst-histo-hp.txt', file_list) - self.assertIn('inst-histo-sp.txt', file_list) - with open('inst-histo-hp.txt') as f: - lines = f.readlines() - self.assertEqual(len(lines), 2) - self.assertIn('100 0', lines[0]) - self.assertIn('100 4096', lines[1]) - with open('inst-histo-sp.txt') as f: - lines = f.readlines() - self.assertEqual(len(lines), 2) - self.assertIn('100 196608', lines[0]) - self.assertIn('100 4194304', lines[1]) + """All of our tests for heatmap_generator.draw() and related.""" + + def test_with_one_mmap_one_sample(self): + """Tests one perf record and one sample.""" + fname = "test.txt" + with open(fname, "w") as f: + _write_perf_mmap(101, 101, 0xABCD000, 0x100, f) + _write_perf_sample(101, 101, 0xABCD101, f) + self.addCleanup(_cleanup, fname) + _heatmap(fname) + self.assertIn("out.txt", os.listdir(".")) + with open("out.txt") as f: + lines = f.readlines() + self.assertEqual(len(lines), 1) + self.assertIn("101/101: 1 0", lines[0]) + + def test_with_one_mmap_multiple_samples(self): + """Tests one perf record and three samples.""" + fname = "test.txt" + with open(fname, "w") as f: + _write_perf_mmap(101, 101, 0xABCD000, 0x100, f) + _write_perf_sample(101, 101, 0xABCD101, f) + _write_perf_sample(101, 101, 0xABCD102, f) + _write_perf_sample(101, 101, 0xABCE102, f) + self.addCleanup(_cleanup, fname) + _heatmap(fname) + self.assertIn("out.txt", os.listdir(".")) + with open("out.txt") as f: + lines = f.readlines() + self.assertEqual(len(lines), 3) + self.assertIn("101/101: 1 0", lines[0]) + self.assertIn("101/101: 2 0", lines[1]) + self.assertIn("101/101: 3 4096", lines[2]) + + def test_with_fork_and_exit(self): + """Tests perf fork and perf exit.""" + fname = "test_fork.txt" + with open(fname, "w") as f: + _write_perf_mmap(101, 101, 0xABCD000, 0x100, f) + _write_perf_fork(101, 101, 202, 202, f) + _write_perf_sample(101, 101, 0xABCD101, f) + _write_perf_sample(202, 202, 0xABCE101, f) + _write_perf_exit(202, 202, 202, 202, f) + self.addCleanup(_cleanup, fname) + _heatmap(fname) + self.assertIn("out.txt", os.listdir(".")) + with open("out.txt") as f: + lines = f.readlines() + self.assertEqual(len(lines), 2) + self.assertIn("101/101: 1 0", lines[0]) + self.assertIn("202/202: 2 4096", lines[1]) + + def test_hugepage_creates_two_chrome_mmaps(self): + """Test two chrome mmaps for the same process.""" + fname = "test_hugepage.txt" + with open(fname, "w") as f: + _write_perf_mmap(101, 101, 0xABCD000, 0x1000, f) + _write_perf_fork(101, 101, 202, 202, f) + _write_perf_mmap(202, 202, 0xABCD000, 0x100, f) + _write_perf_mmap(202, 202, 0xABCD300, 0xD00, f) + _write_perf_sample(101, 101, 0xABCD102, f) + _write_perf_sample(202, 202, 0xABCD102, f) + self.addCleanup(_cleanup, fname) + _heatmap(fname) + self.assertIn("out.txt", os.listdir(".")) + with open("out.txt") as f: + lines = f.readlines() + self.assertEqual(len(lines), 2) + self.assertIn("101/101: 1 0", lines[0]) + self.assertIn("202/202: 2 0", lines[1]) + + def test_hugepage_creates_two_chrome_mmaps_fail(self): + """Test two chrome mmaps for the same process.""" + fname = "test_hugepage.txt" + # Cases where first_mmap.size < second_mmap.size + with open(fname, "w") as f: + _write_perf_mmap(101, 101, 0xABCD000, 0x1000, f) + _write_perf_fork(101, 101, 202, 202, f) + _write_perf_mmap(202, 202, 0xABCD000, 0x10000, f) + self.addCleanup(_cleanup, fname) + with self.assertRaises(AssertionError) as msg: + _heatmap(fname) + self.assertIn("Original MMAP size", str(msg.exception)) + + # Cases where first_mmap.address > second_mmap.address + with open(fname, "w") as f: + _write_perf_mmap(101, 101, 0xABCD000, 0x1000, f) + _write_perf_fork(101, 101, 202, 202, f) + _write_perf_mmap(202, 202, 0xABCC000, 0x10000, f) + with self.assertRaises(AssertionError) as msg: + _heatmap(fname) + self.assertIn("Original MMAP starting address", str(msg.exception)) + + # Cases where first_mmap.address + size < + # second_mmap.address + second_mmap.size + with open(fname, "w") as f: + _write_perf_mmap(101, 101, 0xABCD000, 0x1000, f) + _write_perf_fork(101, 101, 202, 202, f) + _write_perf_mmap(202, 202, 0xABCD100, 0x10000, f) + with self.assertRaises(AssertionError) as msg: + _heatmap(fname) + self.assertIn("exceeds the end of original MMAP", str(msg.exception)) + + def test_histogram(self): + """Tests if the tool can generate correct histogram. + + In the tool, histogram is generated from statistics + of perf samples (saved to out.txt). The histogram is + generated by perf-to-inst-page.sh and saved to + inst-histo.txt. It will be used to draw heat maps. + """ + fname = "test_histo.txt" + with open(fname, "w") as f: + _write_perf_mmap(101, 101, 0xABCD000, 0x100, f) + for i in range(100): + _write_perf_sample(101, 101, 0xABCD000 + i, f) + _write_perf_sample(101, 101, 0xABCE000 + i, f) + _write_perf_sample(101, 101, 0xABFD000 + i, f) + _write_perf_sample(101, 101, 0xAFCD000 + i, f) + self.addCleanup(_cleanup, fname) + _heatmap(fname) + self.assertIn("inst-histo.txt", os.listdir(".")) + with open("inst-histo.txt") as f: + lines = f.readlines() + self.assertEqual(len(lines), 4) + self.assertIn("100 0", lines[0]) + self.assertIn("100 4096", lines[1]) + self.assertIn("100 196608", lines[2]) + self.assertIn("100 4194304", lines[3]) + + def test_histogram_two_mb_page(self): + """Tests handling of 2MB page.""" + fname = "test_histo.txt" + with open(fname, "w") as f: + _write_perf_mmap(101, 101, 0xABCD000, 0x100, f) + for i in range(100): + _write_perf_sample(101, 101, 0xABCD000 + i, f) + _write_perf_sample(101, 101, 0xABCE000 + i, f) + _write_perf_sample(101, 101, 0xABFD000 + i, f) + _write_perf_sample(101, 101, 0xAFCD000 + i, f) + self.addCleanup(_cleanup, fname) + _heatmap(fname, page_size=2 * 1024 * 1024) + self.assertIn("inst-histo.txt", os.listdir(".")) + with open("inst-histo.txt") as f: + lines = f.readlines() + self.assertEqual(len(lines), 2) + self.assertIn("300 0", lines[0]) + self.assertIn("100 4194304", lines[1]) + + def test_histogram_in_and_out_hugepage(self): + """Tests handling the case of separating samples in and out huge page.""" + fname = "test_histo.txt" + with open(fname, "w") as f: + _write_perf_mmap(101, 101, 0xABCD000, 0x100, f) + for i in range(100): + _write_perf_sample(101, 101, 0xABCD000 + i, f) + _write_perf_sample(101, 101, 0xABCE000 + i, f) + _write_perf_sample(101, 101, 0xABFD000 + i, f) + _write_perf_sample(101, 101, 0xAFCD000 + i, f) + self.addCleanup(_cleanup, fname) + _heatmap(fname, hugepage=[0, 8192]) + file_list = os.listdir(".") + self.assertNotIn("inst-histo.txt", file_list) + self.assertIn("inst-histo-hp.txt", file_list) + self.assertIn("inst-histo-sp.txt", file_list) + with open("inst-histo-hp.txt") as f: + lines = f.readlines() + self.assertEqual(len(lines), 2) + self.assertIn("100 0", lines[0]) + self.assertIn("100 4096", lines[1]) + with open("inst-histo-sp.txt") as f: + lines = f.readlines() + self.assertEqual(len(lines), 2) + self.assertIn("100 196608", lines[0]) + self.assertIn("100 4194304", lines[1]) class HeatmapGeneratorAnalyzeTests(unittest.TestCase): - """All of our tests for heatmap_generator.analyze() and related.""" - - def setUp(self): - # Use the same perf report for testing - self.fname = 'test_histo.txt' - with open(self.fname, 'w') as f: - _write_perf_mmap(101, 101, 0xABCD000, 0x100, f) - for i in range(10): - _write_perf_sample(101, 101, 0xABCD000 + i, f) - _write_perf_sample(101, 101, 0xABCE000 + i, f) - _write_perf_sample(101, 101, 0xABFD000 + i, f) - self.nm = ('000000000abcd000 t Func1@Page1\n' - '000000000abcd001 t Func2@Page1\n' - '000000000abcd0a0 t Func3@Page1andFunc1@Page2\n' - '000000000abce010 t Func2@Page2\n' - '000000000abfd000 t Func1@Page3\n') - - def tearDown(self): - _cleanup(self.fname) - - @mock.patch('subprocess.check_output') - def test_analyze_hot_pages_with_hp_top(self, mock_nm): - """Test if the analyze() can print the top page with hugepage.""" - mock_nm.return_value = self.nm - _heatmap(self.fname, hugepage=[0, 8192], analyze=True, top_n=1) - file_list = os.listdir('.') - self.assertIn('addr2symbol.txt', file_list) - with open('addr2symbol.txt') as f: - contents = f.read() - self.assertIn('Func2@Page1 : 9', contents) - self.assertIn('Func1@Page1 : 1', contents) - self.assertIn('Func1@Page3 : 10', contents) - # Only displaying one page in hugepage - self.assertNotIn('Func3@Page1andFunc1@Page2 : 10', contents) - - @mock.patch('subprocess.check_output') - def test_analyze_hot_pages_without_hp_top(self, mock_nm): - """Test if the analyze() can print the top page without hugepage.""" - mock_nm.return_value = self.nm - _heatmap(self.fname, analyze=True, top_n=1) - file_list = os.listdir('.') - self.assertIn('addr2symbol.txt', file_list) - with open('addr2symbol.txt') as f: - contents = f.read() - self.assertIn('Func2@Page1 : 9', contents) - self.assertIn('Func1@Page1 : 1', contents) - # Only displaying one page - self.assertNotIn('Func3@Page1andFunc1@Page2 : 10', contents) - self.assertNotIn('Func1@Page3 : 10', contents) - - @mock.patch('subprocess.check_output') - def test_analyze_hot_pages_with_hp_top10(self, mock_nm): - """Test if the analyze() can print with default top 10.""" - mock_nm.return_value = self.nm - _heatmap(self.fname, analyze=True) - # Make sure nm command is called correctly. - mock_nm.assert_called_with(['nm', '-n', '/path/to/chrome']) - file_list = os.listdir('.') - self.assertIn('addr2symbol.txt', file_list) - with open('addr2symbol.txt') as f: - contents = f.read() - self.assertIn('Func2@Page1 : 9', contents) - self.assertIn('Func1@Page1 : 1', contents) - self.assertIn('Func3@Page1andFunc1@Page2 : 10', contents) - self.assertIn('Func1@Page3 : 10', contents) - - -if __name__ == '__main__': - unittest.main() + """All of our tests for heatmap_generator.analyze() and related.""" + + def setUp(self): + # Use the same perf report for testing + self.fname = "test_histo.txt" + with open(self.fname, "w") as f: + _write_perf_mmap(101, 101, 0xABCD000, 0x100, f) + for i in range(10): + _write_perf_sample(101, 101, 0xABCD000 + i, f) + _write_perf_sample(101, 101, 0xABCE000 + i, f) + _write_perf_sample(101, 101, 0xABFD000 + i, f) + self.nm = ( + "000000000abcd000 t Func1@Page1\n" + "000000000abcd001 t Func2@Page1\n" + "000000000abcd0a0 t Func3@Page1andFunc1@Page2\n" + "000000000abce010 t Func2@Page2\n" + "000000000abfd000 t Func1@Page3\n" + ) + + def tearDown(self): + _cleanup(self.fname) + + @mock.patch("subprocess.check_output") + def test_analyze_hot_pages_with_hp_top(self, mock_nm): + """Test if the analyze() can print the top page with hugepage.""" + mock_nm.return_value = self.nm + _heatmap(self.fname, hugepage=[0, 8192], analyze=True, top_n=1) + file_list = os.listdir(".") + self.assertIn("addr2symbol.txt", file_list) + with open("addr2symbol.txt") as f: + contents = f.read() + self.assertIn("Func2@Page1 : 9", contents) + self.assertIn("Func1@Page1 : 1", contents) + self.assertIn("Func1@Page3 : 10", contents) + # Only displaying one page in hugepage + self.assertNotIn("Func3@Page1andFunc1@Page2 : 10", contents) + + @mock.patch("subprocess.check_output") + def test_analyze_hot_pages_without_hp_top(self, mock_nm): + """Test if the analyze() can print the top page without hugepage.""" + mock_nm.return_value = self.nm + _heatmap(self.fname, analyze=True, top_n=1) + file_list = os.listdir(".") + self.assertIn("addr2symbol.txt", file_list) + with open("addr2symbol.txt") as f: + contents = f.read() + self.assertIn("Func2@Page1 : 9", contents) + self.assertIn("Func1@Page1 : 1", contents) + # Only displaying one page + self.assertNotIn("Func3@Page1andFunc1@Page2 : 10", contents) + self.assertNotIn("Func1@Page3 : 10", contents) + + @mock.patch("subprocess.check_output") + def test_analyze_hot_pages_with_hp_top10(self, mock_nm): + """Test if the analyze() can print with default top 10.""" + mock_nm.return_value = self.nm + _heatmap(self.fname, analyze=True) + # Make sure nm command is called correctly. + mock_nm.assert_called_with(["nm", "-n", "/path/to/chrome"]) + file_list = os.listdir(".") + self.assertIn("addr2symbol.txt", file_list) + with open("addr2symbol.txt") as f: + contents = f.read() + self.assertIn("Func2@Page1 : 9", contents) + self.assertIn("Func1@Page1 : 1", contents) + self.assertIn("Func3@Page1andFunc1@Page2 : 10", contents) + self.assertIn("Func1@Page3 : 10", contents) + + +if __name__ == "__main__": + unittest.main() diff --git a/heatmaps/perf-to-inst-page.sh b/heatmaps/perf-to-inst-page.sh index d6acd5ed..5be1a2b9 100755 --- a/heatmaps/perf-to-inst-page.sh +++ b/heatmaps/perf-to-inst-page.sh @@ -1,5 +1,5 @@ #! /bin/bash -u -# Copyright 2015 The Chromium OS Authors. All rights reserved. +# Copyright 2015 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. |