crosperf: generate perf report with correct debug files

This patch fixes the issue in chromium:946588.

This patch makes perf report no longer use hard code debug directories.
There are several different situations:

1) When running tests on a downloaded image, it will download debug.tgz
from gs, extract it to debug_files in /tmp. Options --symfs and
--vmlinux will depend on this directory, and throw a warning to user
that --kallsyms cannot be applied.

2) If running with downloaded image and debug.tgz could not work, then
we will try to use local build, but give user a warning that it may not
match real symbols well.

3) When running tests with local build, try to find debug info from
/build/$board directory.

Thus, this patch added a new field in label, called 'debug_path', if
this is manually set in experiment file, then crosperf will directly use
the location.

Downloading of debug.tgz will only happen when perf_args is set in
global settings.

TEST=Passed all unit tests, tested with eve and sand.
BUG=chromium:946588

Change-Id: I7f35d1216d912c8526d5501748f951face1273aa
Reviewed-on: https://chromium-review.googlesource.com/1561780
Commit-Ready: Zhizhou Yang <zhizhouy@google.com>
Tested-by: Zhizhou Yang <zhizhouy@google.com>
Reviewed-by: Manoj Gupta <manojgupta@chromium.org>

16 files changed, 288 insertions, 110 deletions

diff --git a/crosperf/benchmark_run_unittest.py b/crosperf/benchmark_run_unittest.py
index 0fdc1696..7030001d 100755
--- a/crosperf/benchmark_run_unittest.py
+++ b/crosperf/benchmark_run_unittest.py
@@ -4,6 +4,7 @@
 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+
 """Testing of benchmark_run."""
 
 from __future__ import print_function
@@ -51,6 +52,7 @@ class BenchmarkRunTest(unittest.TestCase):
         'test1',
         'image1',
         'autotest_dir',
+        'debug_dir',
         '/tmp/test_benchmark_run',
         'x86-alex',
         'chromeos2-row1-rack4-host9.cros',
@@ -73,6 +75,7 @@ class BenchmarkRunTest(unittest.TestCase):
         'test1',
         'image1',
         'autotest_dir',
+        'debug_dir',
         '/tmp/test_benchmark_run',
         'x86-alex',
         'chromeos2-row1-rack4-host9.cros',
diff --git a/crosperf/download_images.py b/crosperf/download_images.py
index ad0a812b..0eb9b8a1 100644
--- a/crosperf/download_images.py
+++ b/crosperf/download_images.py
@@ -1,6 +1,8 @@
+# -*- coding: utf-8 -*-
 # Copyright (c) 2014-2015 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+
 """Download images from Cloud Storage."""
 
 from __future__ import print_function
@@ -132,19 +134,17 @@ class ImageDownloader(object):
     if retval != 0:
       print('(Warning: Could not remove file chromiumos_test_image.tar.xz .)')
 
-  def DownloadSingleAutotestFile(self, chromeos_root, build_id,
-                                 package_file_name):
+  def DownloadSingleFile(self, chromeos_root, build_id, package_file_name):
     # Verify if package files exist
     status = 0
-    gs_package_name = ('gs://chromeos-image-archive/%s/%s' %
-                       (build_id, package_file_name))
+    gs_package_name = (
+        'gs://chromeos-image-archive/%s/%s' % (build_id, package_file_name))
     gsutil_cmd = os.path.join(chromeos_root, GS_UTIL)
     if not test_flag.GetTestMode():
       cmd = '%s ls %s' % (gsutil_cmd, gs_package_name)
       status = self._ce.RunCommand(cmd)
     if status != 0:
-      raise MissingFile(
-          'Cannot find autotest package file: %s.' % package_file_name)
+      raise MissingFile('Cannot find package file: %s.' % package_file_name)
 
     if self.log_level == 'average':
       self._logger.LogOutput('Preparing to download %s package to local '
@@ -167,16 +167,16 @@ class ImageDownloader(object):
       if status != 0 or not os.path.exists(package_path):
         raise MissingFile('Cannot download package: %s .' % package_path)
 
-  def UncompressSingleAutotestFile(self, chromeos_root, build_id,
-                                   package_file_name, uncompress_cmd):
+  def UncompressSingleFile(self, chromeos_root, build_id, package_file_name,
+                           uncompress_cmd):
     # Uncompress file
     download_path = os.path.join(chromeos_root, 'chroot/tmp', build_id)
-    command = ('cd %s ; %s %s' % (download_path, uncompress_cmd,
-                                  package_file_name))
+    command = (
+        'cd %s ; %s %s' % (download_path, uncompress_cmd, package_file_name))
 
     if self.log_level != 'verbose':
       self._logger.LogOutput('CMD: %s' % command)
-      print('(Uncompressing autotest file %s .)' % package_file_name)
+      print('(Uncompressing file %s .)' % package_file_name)
     retval = self._ce.RunCommand(command)
     if retval != 0:
       raise MissingFile('Cannot uncompress file: %s.' % package_file_name)
@@ -184,17 +184,17 @@ class ImageDownloader(object):
     command = ('cd %s ; rm -f %s' % (download_path, package_file_name))
     if self.log_level != 'verbose':
       self._logger.LogOutput('CMD: %s' % command)
-      print('(Removing processed autotest file %s .)' % package_file_name)
+      print('(Removing processed file %s .)' % package_file_name)
     # try removing file, its ok to have an error, print if encountered
     retval = self._ce.RunCommand(command)
     if retval != 0:
       print('(Warning: Could not remove file %s .)' % package_file_name)
 
-  def VerifyAutotestFilesExist(self, chromeos_root, build_id, package_file):
+  def VerifyFileExists(self, chromeos_root, build_id, package_file):
     # Quickly verify if the files are there
     status = 0
-    gs_package_name = ('gs://chromeos-image-archive/%s/%s' % (build_id,
-                                                              package_file))
+    gs_package_name = (
+        'gs://chromeos-image-archive/%s/%s' % (build_id, package_file))
     gsutil_cmd = os.path.join(chromeos_root, GS_UTIL)
     if not test_flag.GetTestMode():
       cmd = '%s ls %s' % (gsutil_cmd, gs_package_name)
@@ -223,8 +223,8 @@ class ImageDownloader(object):
     if not os.path.exists(autotest_path):
       # Quickly verify if the files are present on server
       # If not, just exit with warning
-      status = self.VerifyAutotestFilesExist(chromeos_root, build_id,
-                                             autotest_packages_name)
+      status = self.VerifyFileExists(chromeos_root, build_id,
+                                     autotest_packages_name)
       if status != 0:
         default_autotest_dir = '~/trunk/src/third_party/autotest/files'
         print(
@@ -233,19 +233,18 @@ class ImageDownloader(object):
         return default_autotest_dir
 
       # Files exist on server, download and uncompress them
-      self.DownloadSingleAutotestFile(chromeos_root, build_id,
-                                      autotest_packages_name)
-      self.DownloadSingleAutotestFile(chromeos_root, build_id,
-                                      autotest_server_package_name)
-      self.DownloadSingleAutotestFile(chromeos_root, build_id,
-                                      autotest_control_files_name)
-
-      self.UncompressSingleAutotestFile(chromeos_root, build_id,
-                                        autotest_packages_name, 'tar -xvf ')
-      self.UncompressSingleAutotestFile(
-          chromeos_root, build_id, autotest_server_package_name, 'tar -jxvf ')
-      self.UncompressSingleAutotestFile(
-          chromeos_root, build_id, autotest_control_files_name, 'tar -xvf ')
+      self.DownloadSingleFile(chromeos_root, build_id, autotest_packages_name)
+      self.DownloadSingleFile(chromeos_root, build_id,
+                              autotest_server_package_name)
+      self.DownloadSingleFile(chromeos_root, build_id,
+                              autotest_control_files_name)
+
+      self.UncompressSingleFile(chromeos_root, build_id, autotest_packages_name,
+                                'tar -xf ')
+      self.UncompressSingleFile(chromeos_root, build_id,
+                                autotest_server_package_name, 'tar -jxf ')
+      self.UncompressSingleFile(chromeos_root, build_id,
+                                autotest_control_files_name, 'tar -xf ')
       # Rename created autotest directory to autotest_files
       command = ('cd %s ; mv autotest autotest_files' % download_path)
       if self.log_level != 'verbose':
@@ -257,7 +256,44 @@ class ImageDownloader(object):
 
     return autotest_rel_path
 
-  def Run(self, chromeos_root, xbuddy_label, autotest_path):
+  def DownloadDebugFile(self, chromeos_root, build_id):
+    # Download autest package files (3 files)
+    debug_archive_name = 'debug.tgz'
+
+    download_path = os.path.join(chromeos_root, 'chroot/tmp', build_id)
+    # Debug directory relative path wrt chroot
+    debug_rel_path = os.path.join('/tmp', build_id, 'debug_files')
+    # Debug path to download files
+    debug_path = os.path.join(chromeos_root, 'chroot/tmp', build_id,
+                              'debug_files')
+
+    if not os.path.exists(debug_path):
+      # Quickly verify if the file is present on server
+      # If not, just exit with warning
+      status = self.VerifyFileExists(chromeos_root, build_id,
+                                     debug_archive_name)
+      if status != 0:
+        self._logger.LogOutput('WARNING: Could not find debug archive on gs')
+        return ''
+
+      # File exists on server, download and uncompress it
+      self.DownloadSingleFile(chromeos_root, build_id, debug_archive_name)
+
+      self.UncompressSingleFile(chromeos_root, build_id, debug_archive_name,
+                                'tar -xf ')
+      # Rename created autotest directory to autotest_files
+      command = ('cd %s ; mv debug debug_files' % download_path)
+      if self.log_level != 'verbose':
+        self._logger.LogOutput('CMD: %s' % command)
+        print('(Moving downloaded debug files to debug_files)')
+      retval = self._ce.RunCommand(command)
+      if retval != 0:
+        raise MissingFile('Could not create directory debug_files')
+
+    return debug_rel_path
+
+  def Run(self, chromeos_root, xbuddy_label, autotest_path, debug_path,
+          perf_args):
     build_id = self.GetBuildID(chromeos_root, xbuddy_label)
     image_name = ('gs://chromeos-image-archive/%s/chromiumos_test_image.tar.xz'
                   % build_id)
@@ -281,4 +317,7 @@ class ImageDownloader(object):
     if autotest_path == '':
       autotest_path = self.DownloadAutotestFiles(chromeos_root, build_id)
 
-    return image_path, autotest_path
+    if debug_path == '' and perf_args:
+      debug_path = self.DownloadDebugFile(chromeos_root, build_id)
+
+    return image_path, autotest_path, debug_path
diff --git a/crosperf/download_images_unittest.py b/crosperf/download_images_unittest.py
index 349a2dbb..8d9b9e72 100755
--- a/crosperf/download_images_unittest.py
+++ b/crosperf/download_images_unittest.py
@@ -1,6 +1,9 @@
 #!/usr/bin/env python2
-#
-# Copyright 2014 Google Inc.  All Rights Reserved
+#-*- coding: utf-8 -*-
+# Copyright 2019 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
 """Download image unittest."""
 
 from __future__ import print_function
@@ -27,6 +30,7 @@ class ImageDownloaderTestcast(unittest.TestCase):
     self.called_uncompress_image = False
     self.called_get_build_id = False
     self.called_download_autotest_files = False
+    self.called_download_debug_file = False
 
   @mock.patch.object(os, 'makedirs')
   @mock.patch.object(os.path, 'exists')
@@ -126,8 +130,8 @@ class ImageDownloaderTestcast(unittest.TestCase):
     # 2nd arg must be exception handler
     except_handler_string = 'RunCommandExceptionHandler.HandleException'
     self.assertTrue(
-        except_handler_string in repr(
-            mock_cmd_exec.RunCommand.call_args_list[0][1]))
+        except_handler_string in repr(mock_cmd_exec.RunCommand.call_args_list[0]
+                                      [1]))
 
     # Call 2, should have 2 arguments
     self.assertEqual(len(mock_cmd_exec.RunCommand.call_args_list[1]), 2)
@@ -160,13 +164,17 @@ class ImageDownloaderTestcast(unittest.TestCase):
     test_chroot = '/usr/local/home/chromeos'
     test_build_id = 'remote/lumpy/latest-dev'
     test_empty_autotest_path = ''
+    test_empty_debug_path = ''
     test_autotest_path = '/tmp/autotest'
+    test_debug_path = '/tmp/debug'
+    perf_args = '-a'
 
     # Set values to test/check.
     self.called_download_image = False
     self.called_uncompress_image = False
     self.called_get_build_id = False
     self.called_download_autotest_files = False
+    self.called_download_debug_file = False
 
     # Define fake stub functions for Run to call
     def FakeGetBuildID(unused_root, unused_xbuddy_label):
@@ -197,6 +205,12 @@ class ImageDownloaderTestcast(unittest.TestCase):
       self.called_download_autotest_files = True
       return 'autotest'
 
+    def FakeDownloadDebugFile(root, build_id):
+      if root or build_id:
+        pass
+      self.called_download_debug_file = True
+      return 'debug'
+
     # Initialize downloader
     downloader = download_images.ImageDownloader(logger_to_use=MOCK_LOGGER)
 
@@ -205,46 +219,58 @@ class ImageDownloaderTestcast(unittest.TestCase):
     downloader.UncompressImage = FakeUncompressImage
     downloader.DownloadImage = GoodDownloadImage
     downloader.DownloadAutotestFiles = FakeDownloadAutotestFiles
+    downloader.DownloadDebugFile = FakeDownloadDebugFile
 
     # Call Run.
-    image_path, autotest_path = downloader.Run(test_chroot, test_build_id,
-                                               test_empty_autotest_path)
+    image_path, autotest_path, debug_path = downloader.Run(
+        test_chroot, test_build_id, test_empty_autotest_path,
+        test_empty_debug_path, perf_args)
 
     # Make sure it called both _DownloadImage and _UncompressImage
     self.assertTrue(self.called_download_image)
     self.assertTrue(self.called_uncompress_image)
     # Make sure it called DownloadAutotestFiles
     self.assertTrue(self.called_download_autotest_files)
-    # Make sure it returned an image and  autotest path returned from this call
+    # Make sure it called DownloadDebugFile
+    self.assertTrue(self.called_download_debug_file)
+    # Make sure it returned an image and autotest path returned from this call
     self.assertTrue(image_path == 'chromiumos_test_image.bin')
     self.assertTrue(autotest_path == 'autotest')
+    self.assertTrue(debug_path == 'debug')
 
-    # Call Run with a non-empty autotest path
+    # Call Run with a non-empty autotest and debug path
     self.called_download_autotest_files = False
+    self.called_download_debug_file = False
 
-    image_path, autotest_path = downloader.Run(test_chroot, test_build_id,
-                                               test_autotest_path)
+    image_path, autotest_path, debug_path = downloader.Run(
+        test_chroot, test_build_id, test_autotest_path, test_debug_path,
+        perf_args)
 
     # Verify that downloadAutotestFiles was not called
     self.assertFalse(self.called_download_autotest_files)
     # Make sure it returned the specified autotest path returned from this call
     self.assertTrue(autotest_path == test_autotest_path)
+    # Make sure it returned the specified debug path returned from this call
+    self.assertTrue(debug_path == test_debug_path)
 
     # Reset values; Now use fake stub that simulates DownloadImage failing.
     self.called_download_image = False
     self.called_uncompress_image = False
     self.called_download_autotest_files = False
+    self.called_download_debug_file = False
     downloader.DownloadImage = BadDownloadImage
 
     # Call Run again.
     self.assertRaises(download_images.MissingImage, downloader.Run, test_chroot,
-                      test_autotest_path, test_build_id)
+                      test_autotest_path, test_debug_path, test_build_id,
+                      perf_args)
 
     # Verify that UncompressImage and downloadAutotestFiles were not called,
     # since _DownloadImage "failed"
     self.assertTrue(self.called_download_image)
     self.assertFalse(self.called_uncompress_image)
     self.assertFalse(self.called_download_autotest_files)
+    self.assertFalse(self.called_download_debug_file)
 
 
 if __name__ == '__main__':
diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py
index b1e12be9..81b5e544 100644
--- a/crosperf/experiment_factory.py
+++ b/crosperf/experiment_factory.py
@@ -2,6 +2,7 @@
 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+
 """A module to generate experiments."""
 
 from __future__ import print_function
@@ -322,6 +323,7 @@ class ExperimentFactory(object):
       label_name = label_settings.name
       image = label_settings.GetField('chromeos_image')
       autotest_path = label_settings.GetField('autotest_path')
+      debug_path = label_settings.GetField('debug_path')
       chromeos_root = label_settings.GetField('chromeos_root')
       my_remote = label_settings.GetField('remote')
       compiler = label_settings.GetField('compiler')
@@ -335,8 +337,9 @@ class ExperimentFactory(object):
         build = label_settings.GetField('build')
         if len(build) == 0:
           raise RuntimeError("Can not have empty 'build' field!")
-        image, autotest_path = label_settings.GetXbuddyPath(
-            build, autotest_path, board, chromeos_root, log_level)
+        image, autotest_path, debug_path = label_settings.GetXbuddyPath(
+            build, autotest_path, debug_path, board, chromeos_root, log_level,
+            perf_args)
 
       cache_dir = label_settings.GetField('cache_dir')
       chrome_src = label_settings.GetField('chrome_src')
@@ -354,13 +357,14 @@ class ExperimentFactory(object):
       image_args = label_settings.GetField('image_args')
       if test_flag.GetTestMode():
         # pylint: disable=too-many-function-args
-        label = MockLabel(label_name, image, autotest_path, chromeos_root,
-                          board, my_remote, image_args, cache_dir, cache_only,
-                          log_level, compiler, chrome_src)
+        label = MockLabel(label_name, image, autotest_path, debug_path,
+                          chromeos_root, board, my_remote, image_args,
+                          cache_dir, cache_only, log_level, compiler,
+                          chrome_src)
       else:
-        label = Label(label_name, image, autotest_path, chromeos_root, board,
-                      my_remote, image_args, cache_dir, cache_only, log_level,
-                      compiler, chrome_src)
+        label = Label(label_name, image, autotest_path, debug_path,
+                      chromeos_root, board, my_remote, image_args, cache_dir,
+                      cache_only, log_level, compiler, chrome_src)
       labels.append(label)
 
     if not labels:
diff --git a/crosperf/experiment_factory_unittest.py b/crosperf/experiment_factory_unittest.py
index b0c795eb..a85f0f7f 100755
--- a/crosperf/experiment_factory_unittest.py
+++ b/crosperf/experiment_factory_unittest.py
@@ -4,6 +4,7 @@
 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+
 """Unit test for experiment_factory.py"""
 
 from __future__ import print_function
@@ -276,13 +277,17 @@ class ExperimentFactoryTest(unittest.TestCase):
         return []
       return ['fake_chromeos_machine1.cros', 'fake_chromeos_machine2.cros']
 
-    def FakeGetXbuddyPath(build, autotest_dir, board, chroot, log_level):
+    def FakeGetXbuddyPath(build, autotest_dir, debug_dir, board, chroot,
+                          log_level, perf_args):
       autotest_path = autotest_dir
       if not autotest_path:
         autotest_path = 'fake_autotest_path'
+      debug_path = debug_dir
+      if not debug_path and perf_args:
+        debug_path = 'fake_debug_path'
       if not build or not board or not chroot or not log_level:
-        return '', autotest_path
-      return 'fake_image_path', autotest_path
+        return '', autotest_path, debug_path
+      return 'fake_image_path', autotest_path, debug_path
 
     ef = ExperimentFactory()
     ef.AppendBenchmarkSet = FakeAppendBenchmarkSet
diff --git a/crosperf/experiment_file.py b/crosperf/experiment_file.py
index 12f9d5df..41a2b809 100644
--- a/crosperf/experiment_file.py
+++ b/crosperf/experiment_file.py
@@ -1,6 +1,8 @@
+# -*- coding: utf-8 -*-
 # Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+
 """The experiment file module. It manages the input file of crosperf."""
 
 from __future__ import print_function
@@ -164,11 +166,22 @@ class ExperimentFile(object):
               autotest_path = ''
               if autotest_field.assigned:
                 autotest_path = autotest_field.GetString()
-              image_path, autotest_path = settings.GetXbuddyPath(
-                  value, autotest_path, board, chromeos_root, 'quiet')
+              debug_field = settings.fields['debug_path']
+              debug_path = ''
+              if debug_field.assigned:
+                debug_path = autotest_field.GetString()
+              perf_args_field = self.global_settings.fields['perf_args']
+              perf_args = ''
+              if perf_args_field.assigned:
+                perf_args = perf_args_field.GetString()
+              image_path, autotest_path, debug_path = settings.GetXbuddyPath(
+                  value, autotest_path, debug_path, board, chromeos_root,
+                  'quiet', perf_args)
               res += '\t#actual_image: %s\n' % image_path
               if not autotest_field.assigned:
                 res += '\t#actual_autotest_path: %s\n' % autotest_path
+              if not debug_field.assigned:
+                res += '\t#actual_debug_path: %s\n' % debug_path
 
         res += '}\n\n'
 
diff --git a/crosperf/label.py b/crosperf/label.py
index d993c15c..f11e5007 100644
--- a/crosperf/label.py
+++ b/crosperf/label.py
@@ -1,6 +1,8 @@
+# -*- coding: utf-8 -*-
 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+
 """The label of benchamrks."""
 
 from __future__ import print_function
@@ -20,6 +22,7 @@ class Label(object):
                name,
                chromeos_image,
                autotest_path,
+               debug_path,
                chromeos_root,
                board,
                remote,
@@ -40,6 +43,7 @@ class Label(object):
     self.name = name
     self.chromeos_image = chromeos_image
     self.autotest_path = autotest_path
+    self.debug_path = debug_path
     self.board = board
     self.remote = remote
     self.image_args = image_args
@@ -53,9 +57,9 @@ class Label(object):
       if self.image_type == 'local':
         chromeos_root = FileUtils().ChromeOSRootFromImage(chromeos_image)
       if not chromeos_root:
-        raise RuntimeError("No ChromeOS root given for label '%s' and could "
-                           "not determine one from image path: '%s'." %
-                           (name, chromeos_image))
+        raise RuntimeError(
+            "No ChromeOS root given for label '%s' and could "
+            "not determine one from image path: '%s'." % (name, chromeos_image))
     else:
       chromeos_root = FileUtils().CanonicalizeChromeOSRoot(chromeos_root)
       if not chromeos_root:
@@ -120,6 +124,7 @@ class MockLabel(object):
                name,
                chromeos_image,
                autotest_path,
+               debug_path,
                chromeos_root,
                board,
                remote,
@@ -132,6 +137,7 @@ class MockLabel(object):
     self.name = name
     self.chromeos_image = chromeos_image
     self.autotest_path = autotest_path
+    self.debug_path = debug_path
     self.board = board
     self.remote = remote
     self.cache_dir = cache_dir
diff --git a/crosperf/machine_manager_unittest.py b/crosperf/machine_manager_unittest.py
index 3663ab81..a0c8b924 100755
--- a/crosperf/machine_manager_unittest.py
+++ b/crosperf/machine_manager_unittest.py
@@ -53,11 +53,11 @@ class MyMachineManager(machine_manager.MachineManager):
 CHROMEOS_ROOT = '/tmp/chromeos-root'
 MACHINE_NAMES = ['lumpy1', 'lumpy2', 'lumpy3', 'daisy1', 'daisy2']
 LABEL_LUMPY = label.MockLabel(
-    'lumpy', 'lumpy_chromeos_image', 'autotest_dir', CHROMEOS_ROOT, 'lumpy',
-    ['lumpy1', 'lumpy2', 'lumpy3', 'lumpy4'], '', '', False, 'average,'
+    'lumpy', 'lumpy_chromeos_image', 'autotest_dir', 'debug_dir', CHROMEOS_ROOT,
+    'lumpy', ['lumpy1', 'lumpy2', 'lumpy3', 'lumpy4'], '', '', False, 'average,'
     'gcc', None)
 LABEL_MIX = label.MockLabel('mix', 'chromeos_image', 'autotest_dir',
-                            CHROMEOS_ROOT, 'mix',
+                            'debug_dir', CHROMEOS_ROOT, 'mix',
                             ['daisy1', 'daisy2', 'lumpy3', 'lumpy4'], '', '',
                             False, 'average', 'gcc', None)
 
diff --git a/crosperf/mock_instance.py b/crosperf/mock_instance.py
index 758108fa..ece07db3 100644
--- a/crosperf/mock_instance.py
+++ b/crosperf/mock_instance.py
@@ -1,6 +1,8 @@
+# -*- coding: utf-8 -*-
 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+
 """This contains some mock instances for testing."""
 
 from __future__ import print_function
@@ -13,6 +15,7 @@ label1 = MockLabel(
     'test1',
     'image1',
     'autotest_dir',
+    'debug_dir',
     '/tmp/test_benchmark_run',
     'x86-alex',
     'chromeos-alex1',
@@ -26,6 +29,7 @@ label2 = MockLabel(
     'test2',
     'image2',
     'autotest_dir',
+    'debug_dir',
     '/tmp/test_benchmark_run_2',
     'x86-alex',
     'chromeos-alex2',
diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py
index 39c127bf..bef78cb4 100644
--- a/crosperf/results_cache.py
+++ b/crosperf/results_cache.py
@@ -266,6 +266,18 @@ class Result(object):
           self.FindFilesInResultsDir('-name results-chart.json').splitlines()
     return result
 
+  def _CheckDebugPath(self, option, path):
+    relative_path = path[1:]
+    out_chroot_path = os.path.join(self.chromeos_root, 'chroot', relative_path)
+    if os.path.exists(out_chroot_path):
+      if option == 'kallsyms':
+        path = os.path.join(path, 'System.map-*')
+      return '--' + option + ' ' + path
+    else:
+      print('** WARNING **: --%s option not applied, %s does not exist' %
+            (option, out_chroot_path))
+      return ''
+
   def GeneratePerfReportFiles(self):
     perf_report_files = []
     for perf_data_file in self.perf_data_files:
@@ -285,15 +297,37 @@ class Result(object):
       if os.path.exists(perf_path):
         perf_file = '/usr/bin/perf'
 
-      command = ('%s report '
-                 '-n '
-                 '--symfs /build/%s '
-                 '--vmlinux /build/%s/usr/lib/debug/boot/vmlinux '
-                 '--kallsyms /build/%s/boot/System.map-* '
-                 '-i %s --stdio '
-                 '> %s' % (perf_file, self.board, self.board, self.board,
-                           chroot_perf_data_file, chroot_perf_report_file))
-      self.ce.ChrootRunCommand(self.chromeos_root, command)
+      debug_path = self.label.debug_path
+
+      if debug_path:
+        symfs = '--symfs ' + debug_path
+        vmlinux = '--vmlinux ' + os.path.join(debug_path, 'boot', 'vmlinux')
+        kallsyms = ''
+        print('** WARNING **: --kallsyms option not applied, no System.map-* '
+              'for downloaded image.')
+      else:
+        if self.label.image_type != 'local':
+          print('** WARNING **: Using local debug info in /build, this may '
+                'not match the downloaded image.')
+        build_path = os.path.join('/build', self.board)
+        symfs = self._CheckDebugPath('symfs', build_path)
+        vmlinux_path = os.path.join(build_path, 'usr/lib/debug/boot/vmlinux')
+        vmlinux = self._CheckDebugPath('vmlinux', vmlinux_path)
+        kallsyms_path = os.path.join(build_path, 'boot')
+        kallsyms = self._CheckDebugPath('kallsyms', kallsyms_path)
+
+      command = ('%s report -n %s %s %s -i %s --stdio > %s' %
+                 (perf_file, symfs, vmlinux, kallsyms, chroot_perf_data_file,
+                  chroot_perf_report_file))
+      if self.log_level != 'verbose':
+        self._logger.LogOutput('Generating perf report...\nCMD: %s' % command)
+      exit_code = self.ce.ChrootRunCommand(self.chromeos_root, command)
+      if exit_code == 0:
+        if self.log_level != 'verbose':
+          self._logger.LogOutput('Perf report generated successfully.')
+      else:
+        raise RuntimeError(
+            'Perf report not generated correctly. CMD: %s' % command)
 
       # Add a keyval to the dictionary for the events captured.
       perf_report_files.append(
diff --git a/crosperf/results_cache_unittest.py b/crosperf/results_cache_unittest.py
index fcf2872d..c201c9d8 100755
--- a/crosperf/results_cache_unittest.py
+++ b/crosperf/results_cache_unittest.py
@@ -4,6 +4,7 @@
 # Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+
 """Module of result cache unittest."""
 
 from __future__ import print_function
@@ -193,9 +194,9 @@ class ResultTest(unittest.TestCase):
     self.callGatherPerfResults = False
     self.mock_logger = mock.Mock(spec=logger.Logger)
     self.mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter)
-    self.mock_label = MockLabel('mock_label', 'chromeos_image', 'autotest_dir',
-                                '/tmp', 'lumpy', 'remote', 'image_args',
-                                'cache_dir', 'average', 'gcc', None)
+    self.mock_label = MockLabel(
+        'mock_label', 'chromeos_image', 'autotest_dir', 'debug_dir', '/tmp',
+        'lumpy', 'remote', 'image_args', 'cache_dir', 'average', 'gcc', None)
 
   def testCreateFromRun(self):
     result = MockResult.CreateFromRun(logger.GetLogger(), 'average',
@@ -559,14 +560,33 @@ class ResultTest(unittest.TestCase):
     self.result.board = 'lumpy'
     mock_getpath.return_value = fake_file
     self.result.ce.ChrootRunCommand = mock_chrootruncmd
+    mock_chrootruncmd.return_value = 0
+    # Debug path not found
+    self.result.label.debug_path = ''
+    tmp = self.result.GeneratePerfReportFiles()
+    self.assertEqual(tmp, ['/tmp/chroot%s' % fake_file])
+    self.assertEqual(mock_chrootruncmd.call_args_list[0][0],
+                     ('/tmp', ('/usr/sbin/perf report -n    '
+                               '-i %s --stdio > %s') % (fake_file, fake_file)))
+
+  @mock.patch.object(misc, 'GetInsideChrootPath')
+  @mock.patch.object(command_executer.CommandExecuter, 'ChrootRunCommand')
+  def test_generate_perf_report_files_debug(self, mock_chrootruncmd,
+                                            mock_getpath):
+    fake_file = '/usr/chromeos/chroot/tmp/results/fake_file'
+    self.result.perf_data_files = ['/tmp/results/perf.data']
+    self.result.board = 'lumpy'
+    mock_getpath.return_value = fake_file
+    self.result.ce.ChrootRunCommand = mock_chrootruncmd
+    mock_chrootruncmd.return_value = 0
+    # Debug path found
+    self.result.label.debug_path = '/tmp/debug'
     tmp = self.result.GeneratePerfReportFiles()
     self.assertEqual(tmp, ['/tmp/chroot%s' % fake_file])
     self.assertEqual(mock_chrootruncmd.call_args_list[0][0],
-                     ('/tmp',
-                      ('/usr/sbin/perf report -n --symfs /build/lumpy '
-                       '--vmlinux /build/lumpy/usr/lib/debug/boot/vmlinux '
-                       '--kallsyms /build/lumpy/boot/System.map-* -i '
-                       '%s --stdio > %s') % (fake_file, fake_file)))
+                     ('/tmp', ('/usr/sbin/perf report -n --symfs /tmp/debug '
+                               '--vmlinux /tmp/debug/boot/vmlinux  '
+                               '-i %s --stdio > %s') % (fake_file, fake_file)))
 
   @mock.patch.object(misc, 'GetOutsideChrootPath')
   def test_populate_from_run(self, mock_getpath):
@@ -975,9 +995,9 @@ class TelemetryResultTest(unittest.TestCase):
     self.result = None
     self.mock_logger = mock.Mock(spec=logger.Logger)
     self.mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter)
-    self.mock_label = MockLabel('mock_label', 'chromeos_image', 'autotest_dir',
-                                '/tmp', 'lumpy', 'remote', 'image_args',
-                                'cache_dir', 'average', 'gcc', None)
+    self.mock_label = MockLabel(
+        'mock_label', 'chromeos_image', 'autotest_dir', 'debug_dir', '/tmp',
+        'lumpy', 'remote', 'image_args', 'cache_dir', 'average', 'gcc', None)
     self.mock_machine = machine_manager.MockCrosMachine(
         'falco.cros', '/tmp/chromeos', 'average')
 
@@ -1018,9 +1038,9 @@ class ResultsCacheTest(unittest.TestCase):
     super(ResultsCacheTest, self).__init__(*args, **kwargs)
     self.fakeCacheReturnResult = None
     self.mock_logger = mock.Mock(spec=logger.Logger)
-    self.mock_label = MockLabel('mock_label', 'chromeos_image', 'autotest_dir',
-                                '/tmp', 'lumpy', 'remote', 'image_args',
-                                'cache_dir', 'average', 'gcc', None)
+    self.mock_label = MockLabel(
+        'mock_label', 'chromeos_image', 'autotest_dir', 'debug_dir', '/tmp',
+        'lumpy', 'remote', 'image_args', 'cache_dir', 'average', 'gcc', None)
 
   def setUp(self):
     self.results_cache = ResultsCache()
diff --git a/crosperf/settings.py b/crosperf/settings.py
index 8d5a25fd..290abfc2 100644
--- a/crosperf/settings.py
+++ b/crosperf/settings.py
@@ -1,4 +1,8 @@
-# Copyright 2011 Google Inc. All Rights Reserved.
+#-*- coding: utf-8 -*-
+# Copyright 2019 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
 """Module to get the settings from experiment file."""
 
 from __future__ import print_function
@@ -39,8 +43,8 @@ class Settings(object):
   def GetField(self, name):
     """Get the value of a field with a given name."""
     if name not in self.fields:
-      raise SyntaxError("Field '%s' not a valid field in '%s' settings." %
-                        (name, self.name))
+      raise SyntaxError(
+          "Field '%s' not a valid field in '%s' settings." % (name, self.name))
     field = self.fields[name]
     if not field.assigned and field.required:
       raise SyntaxError("Required field '%s' not defined in '%s' settings." %
@@ -66,8 +70,8 @@ class Settings(object):
       if not self.fields[name].assigned and self.fields[name].required:
         raise SyntaxError('Field %s is invalid.' % name)
 
-  def GetXbuddyPath(self, path_str, autotest_path, board, chromeos_root,
-                    log_level):
+  def GetXbuddyPath(self, path_str, autotest_path, debug_path, board,
+                    chromeos_root, log_level, perf_args):
     prefix = 'remote'
     l = logger.GetLogger()
     if (path_str.find('trybot') < 0 and path_str.find('toolchain') < 0 and
@@ -76,6 +80,7 @@ class Settings(object):
     else:
       xbuddy_path = '%s/%s' % (prefix, path_str)
     image_downloader = ImageDownloader(l, log_level)
-    image_and_autotest_path = image_downloader.Run(
-        misc.CanonicalizePath(chromeos_root), xbuddy_path, autotest_path)
-    return image_and_autotest_path
+    # Returns three variables: image, autotest_path, debug_path
+    return image_downloader.Run(
+        misc.CanonicalizePath(chromeos_root), xbuddy_path, autotest_path,
+        debug_path, perf_args)
diff --git a/crosperf/settings_factory.py b/crosperf/settings_factory.py
index 82956501..1b30e11c 100644
--- a/crosperf/settings_factory.py
+++ b/crosperf/settings_factory.py
@@ -2,6 +2,7 @@
 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+
 """Setting files for global, benchmark and labels."""
 
 from __future__ import print_function
@@ -81,6 +82,12 @@ class LabelSettings(Settings):
             'files.'))
     self.AddField(
         TextField(
+            'debug_path',
+            required=False,
+            description='Debug info directory relative to chroot which has '
+            'symbols and vmlinux that can be used by perf tool.'))
+    self.AddField(
+        TextField(
             'chromeos_root',
             description='The path to a chromeos checkout which '
             'contains a src/scripts directory. Defaults to '
diff --git a/crosperf/settings_factory_unittest.py b/crosperf/settings_factory_unittest.py
index 729a8d06..24fd1fce 100755
--- a/crosperf/settings_factory_unittest.py
+++ b/crosperf/settings_factory_unittest.py
@@ -4,6 +4,7 @@
 # Copyright 2017 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+
 """Unittest for crosperf."""
 
 from __future__ import print_function
@@ -32,7 +33,7 @@ class LabelSettingsTest(unittest.TestCase):
   def test_init(self):
     res = settings_factory.LabelSettings('l_settings')
     self.assertIsNotNone(res)
-    self.assertEqual(len(res.fields), 9)
+    self.assertEqual(len(res.fields), 10)
     self.assertEqual(res.GetField('chromeos_image'), '')
     self.assertEqual(res.GetField('autotest_path'), '')
     self.assertEqual(res.GetField('chromeos_root'), '')
@@ -86,7 +87,7 @@ class SettingsFactoryTest(unittest.TestCase):
     l_settings = settings_factory.SettingsFactory().GetSettings(
         'label', 'label')
     self.assertIsInstance(l_settings, settings_factory.LabelSettings)
-    self.assertEqual(len(l_settings.fields), 9)
+    self.assertEqual(len(l_settings.fields), 10)
 
     b_settings = settings_factory.SettingsFactory().GetSettings(
         'benchmark', 'benchmark')
diff --git a/crosperf/settings_unittest.py b/crosperf/settings_unittest.py
index fea55c05..b9d87e9e 100755
--- a/crosperf/settings_unittest.py
+++ b/crosperf/settings_unittest.py
@@ -1,6 +1,9 @@
 #!/usr/bin/env python2
-#
-# Copyright 2014 Google Inc.  All Rights Reserved.
+#-*- coding: utf-8 -*-
+# Copyright 2019 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
 """unittest for settings."""
 
 from __future__ import print_function
@@ -197,27 +200,34 @@ class TestSettings(unittest.TestCase):
     official_str = 'lumpy-release/R34-5417.0.0'
     xbuddy_str = 'latest-dev'
     autotest_path = ''
+    debug_path = ''
+    perf_args = '-a'
 
-    self.settings.GetXbuddyPath(trybot_str, autotest_path, board, chromeos_root,
-                                log_level)
+    self.settings.GetXbuddyPath(trybot_str, autotest_path, debug_path, board,
+                                chromeos_root, log_level, perf_args)
     self.assertEqual(mock_run.call_count, 1)
-    self.assertEqual(mock_run.call_args_list[0][0],
-                     ('/tmp/chromeos',
-                      'remote/trybot-lumpy-paladin/R34-5417.0.0-b1506', ''))
+    self.assertEqual(mock_run.call_args_list[0][0], (
+        '/tmp/chromeos',
+        'remote/trybot-lumpy-paladin/R34-5417.0.0-b1506',
+        '',
+        '',
+        '-a',
+    ))
 
     mock_run.reset_mock()
-    self.settings.GetXbuddyPath(official_str, autotest_path, board,
-                                chromeos_root, log_level)
+    self.settings.GetXbuddyPath(official_str, autotest_path, debug_path, board,
+                                chromeos_root, log_level, perf_args)
     self.assertEqual(mock_run.call_count, 1)
-    self.assertEqual(mock_run.call_args_list[0][0],
-                     ('/tmp/chromeos', 'remote/lumpy-release/R34-5417.0.0', ''))
+    self.assertEqual(
+        mock_run.call_args_list[0][0],
+        ('/tmp/chromeos', 'remote/lumpy-release/R34-5417.0.0', '', '', '-a'))
 
     mock_run.reset_mock()
-    self.settings.GetXbuddyPath(xbuddy_str, autotest_path, board, chromeos_root,
-                                log_level)
+    self.settings.GetXbuddyPath(xbuddy_str, autotest_path, debug_path, board,
+                                chromeos_root, log_level, perf_args)
     self.assertEqual(mock_run.call_count, 1)
     self.assertEqual(mock_run.call_args_list[0][0],
-                     ('/tmp/chromeos', 'remote/lumpy/latest-dev', ''))
+                     ('/tmp/chromeos', 'remote/lumpy/latest-dev', '', '', '-a'))
 
     if mock_logger:
       return
diff --git a/crosperf/suite_runner_unittest.py b/crosperf/suite_runner_unittest.py
index a2b1fbeb..4da27e17 100755
--- a/crosperf/suite_runner_unittest.py
+++ b/crosperf/suite_runner_unittest.py
@@ -4,6 +4,7 @@
 # Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+
 """Unittest for suite_runner."""
 
 from __future__ import print_function
@@ -31,7 +32,7 @@ class SuiteRunnerTest(unittest.TestCase):
   mock_cmd_term = mock.Mock(spec=command_executer.CommandTerminator)
   mock_logger = mock.Mock(spec=logger.Logger)
   mock_label = label.MockLabel(
-      'lumpy', 'lumpy_chromeos_image', '', '/tmp/chromeos', 'lumpy',
+      'lumpy', 'lumpy_chromeos_image', '', '', '/tmp/chromeos', 'lumpy',
       ['lumpy1.cros', 'lumpy.cros2'], '', '', False, 'average', 'gcc', '')
   telemetry_crosperf_bench = Benchmark(
       'b1_test',  # name