Improve oprofile scripts.

Now support importing dumps and running reports using a simple script.
Eliminated some spurious errors and warnings in the oprofile tools
when profiling Android libraries.

Change-Id: I618cf6f8937a6ab5f45b3d45bdf860792b6bebbe

3 files changed, 341 insertions, 64 deletions

diff --git a/libpp/locate_images.cpp b/libpp/locate_images.cpp
index 426adfd..87c25a0 100644
--- a/libpp/locate_images.cpp
+++ b/libpp/locate_images.cpp
@@ -185,6 +185,13 @@ string const extra_images::find_image_path(string const & image_name,
 		return fixup ? result[0] : image_name;
 	}
 
+#ifdef ANDROID
+	// On Android, we often have both stripped and unstripped versions of the same
+	// library in the image path.  Choose the first one found instead of issuing a
+	// multiple match error.
+	error = image_ok;
+	return fixup ? result[0] : image_name;
+#else
 	// We can't get multiple result except if only one result is prefixed
 	// by archive_path or by root_path.
 	size_t count = 0;
@@ -212,6 +219,7 @@ string const extra_images::find_image_path(string const & image_name,
 
 	error = image_multiple_match;
 	return image_name;
+#endif
 }
 
 
diff --git a/libpp/op_header.cpp b/libpp/op_header.cpp
index 041a1e8..938f5dd 100644
--- a/libpp/op_header.cpp
+++ b/libpp/op_header.cpp
@@ -114,6 +114,12 @@ void check_mtime(string const & file, opd_header const & header)
 	} else {
 		static bool warned_already = false;
 
+#ifdef ANDROID
+		// Android symbol files may not have the same timestamp as the stripped
+		// files deployed to the device.  Suppress spurious warnings.
+		if (file.find("/symbols/") == string::npos) {
+#endif
+
 		cerr << "warning: the last modified time of the binary file "
 		     "does not match that of the sample file for " << file
 		     << "\n";
@@ -123,6 +129,10 @@ void check_mtime(string const & file, opd_header const & header)
 			"has been modified since the sample file was created.\n";
 			warned_already = true;
 		}
+
+#ifdef ANDROID
+		}
+#endif
 	}
 }
 
diff --git a/oprofile_android b/oprofile_android
index 3373b0c..4e05090 100755
--- a/oprofile_android
+++ b/oprofile_android
@@ -24,8 +24,56 @@ import sys
 import subprocess
 import getopt
 import re
+import shutil
 
 
+# Find oprofile binaries (compiled on the host)
+try:
+  oprofile_bin_dir = os.environ['OPROFILE_BIN_DIR']
+except:
+  try:
+    android_host_out = os.environ['ANDROID_HOST_OUT']
+  except:
+    print "Either OPROFILE_BIN_DIR or ANDROID_HOST_OUT must be set. Run \". envsetup.sh\" first"
+    sys.exit(1)
+  oprofile_bin_dir = os.path.join(android_host_out, 'bin')
+
+opimport_bin = os.path.join(oprofile_bin_dir, 'opimport')
+opreport_bin = os.path.join(oprofile_bin_dir, 'opreport')
+
+
+# Find symbol directories
+try:
+  android_product_out = os.environ['ANDROID_PRODUCT_OUT']
+except:
+  print "ANDROID_PRODUCT_OUT must be set. Run \". envsetup.sh\" first"
+  sys.exit(1)
+
+symbols_dir = os.path.join(android_product_out, 'symbols')
+system_dir = os.path.join(android_product_out, 'system')
+
+
+def execute(command, echo=True):
+  if echo:
+    print ' '.join(command)
+  popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+  output = ''
+  while True:
+    stdout, stderr = popen.communicate()
+    if echo and len(stdout) != 0:
+      print stdout
+    if echo and len(stderr) != 0:
+      print stderr
+    output += stdout
+    output += stderr
+    rc = popen.poll()
+    if rc is not None:
+      break
+  if echo:
+    print 'exit code: %d' % rc
+  return rc, output
+
+# ADB wrapper
 class Adb:
   def __init__(self, serial_number):
     self._base_args = ['adb']
@@ -34,129 +82,340 @@ class Adb:
       self._base_args.append(serial_number)
 
   def shell(self, command_args, echo=True):
-    print 'adb: %s' % (' '.join(command_args))
-    popen = subprocess.Popen(self._base_args + ['shell'] + command_args,
-      stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    output = ''
-    while True:
-      stdout, stderr = popen.communicate()
-      if echo:
-        print stdout
-        print stderr
-      output += stdout
-      output += stderr
-      rc = popen.poll()
-      if rc is not None:
-        break
-    print 'exit code: %d' % rc
-    return rc, output
+    return self._adb('shell', command_args, echo)
 
+  def pull(self, source, dest, echo=True):
+    return self._adb('pull', [source, dest], echo)
 
+  def _adb(self, command, command_args, echo):
+    return execute(self._base_args + [command] + command_args, echo)
+
+
+# The tool program itself
 class Tool:
   def __init__(self, argv):
     self.argv = argv
     self.verbose = False
+    self.session_dir = '/tmp/oprofile'
 
   def usage(self):
-    print "Usage:" + self.argv[0]
-    print "    -h, --help          : show this help text"
-    print "    -s, --serial=number : the serial number of the device being profiled"
-    print "    -v, --verbose       : show verbose output"
-    print "    --setup             : setup profiler"
-    print "    --shutdown          : shutdown profiler"
-    print "    --start             : start profiling"
-    print "    --stop              : stop profiling"
-    print "    --status            : show profiler status"
+    print "Usage: " + self.argv[0] + " [options] <command> [command args]"
+    print
+    print "  Options:"
+    print
+    print "    -h, --help            : show this help text"
+    print "    -s, --serial=number   : the serial number of the device being profiled"
+    print "    -v, --verbose         : show verbose output"
+    print "    -d, --dir=path        : directory to store oprofile session on the host, default: /tmp/oprofile"
+    print
+    print "  Commands:"
+    print
+    print "    setup [args]          : setup profiler with specified arguments to 'opcontrol --setup'"
+    print "      -t, --timer             : enable timer based profiling"
+    print "      -e, --event=[spec]      : specify an event type to profile, eg. --event=CPU_CYCLES:100000"
+    print "                                (not supported on all devices)"
+    print "      -c, --callgraph=[depth] : specify callgraph capture depth, default is none"
+    print "                                (not supported in timer mode)"
+    print
+    print "    shutdown              : shutdown profiler"
+    print
+    print "    start                 : start profiling"
+    print
+    print "    stop                  : stop profiling"
+    print
+    print "    status                : show profiler status"
+    print
+    print "    import                : dump samples and pull session directory from the device"
+    print "      -f, --force             : remove existing session directory before import" 
+    print
+    print "    report [args]         : generate report with specified arguments to 'opreport'"
+    print "      -l, --symbols           : show symbols"
+    print "      -c, --callgraph         : show callgraph"
+    print "      --help                  : show help for additional opreport options"
     print
 
   def main(self):
+    rc = self.do_main()
+    if rc == 2:
+      print
+      self.usage()
+    return rc
+
+  def do_main(self):
     try:
       opts, args = getopt.getopt(self.argv[1:],
-        "hs:v",
-        ["help", "serial=", "setup", "start", "stop", "status", "shutdown", "verbose"])
+        'hs:vd', ['help', 'serial=', 'dir=', 'verbose'])
     except getopt.GetoptError, e:
-      self.usage()
       print str(e)
-      return 1
+      return 2
 
     serial_number = None
-    command = None
     for o, a in opts:
       if o in ('-h', '--help'):
         self.usage()
         return 0
       elif o in ('-s', '--serial'):
         serial_number = a
+      elif o in ('-d', '--dir'):
+        self.session_dir = a
       elif o in ('-v', '--verbose'):
         self.verbose = True
-      elif o in ('--setup', '--start', '--stop', '--status', '--shutdown'):
-        command = o[2:]
-      else:
-        assert False, 'unhandled option' + o
+
+    if len(args) == 0:
+      print '* A command must be specified.'
+      return 2
+
+    command = args[0]
+    command_args = args[1:]
 
     self.adb = Adb(serial_number)
 
     if command == 'setup':
-      rc = self.setup()
+      rc = self.do_setup(command_args)
     elif command == 'shutdown':
-      rc = self.shutdown()
+      rc = self.do_shutdown(command_args)
     elif command == 'start':
-      rc = self.start()
+      rc = self.do_start(command_args)
     elif command == 'stop':
-      rc = self.stop()
+      rc = self.do_stop(command_args)
     elif command == 'status':
-      rc = self.status()
+      rc = self.do_status(command_args)
+    elif command == 'import':
+      rc = self.do_import(command_args)
+    elif command == 'report':
+      rc = self.do_report(command_args)
     else:
-      self.usage()
-      print 'A command must be specified.'
-      rc = 1
+      print '* Unknown command: ' + command
+      return 2
+
     return rc
 
-  def setup(self):
+  def do_setup(self, command_args):
+    events = []
+    timer = False
+    callgraph = None
+
+    try:
+      opts, args = getopt.getopt(command_args,
+        'te:c:', ['timer', 'event=', 'callgraph='])
+    except getopt.GetoptError, e:
+      print '* Unsupported setup command arguments:', str(e)
+      return 2
+
+    for o, a in opts:
+      if o in ('-t', '--timer'):
+        timer = True
+      elif o in ('-e', '--event'):
+        events.append('--event=' + a)
+      elif o in ('-c', '--callgraph'):
+        callgraph = a
+
+    if len(args) != 0:
+      print '* Unsupported setup command arguments: %s' % (' '.join(args))
+      return 2
+
+    if not timer and len(events) == 0:
+      print '* Must specify --timer or at least one --event argument.'
+      return 2
+
+    if timer and len(events) != 0:
+      print '* --timer and --event cannot be used together.'
+      return 2
+
+    if timer and callgraph is not None:
+      print '* --callgraph cannot be used with --timer.'
+      return 2
+
+    opcontrol_args = events
+    if timer:
+      opcontrol_args.append('--timer')
+    if callgraph is not None:
+      opcontrol_args.append('--callgraph=' + callgraph)
+
+    # Get kernal VMA range.
     rc, output = self.adb.shell(['cat', '/proc/kallsyms'], echo=False)
     if rc != 0:
-      print 'Failed to determine kernel VMA range.'
-      return rc
+      print '* Failed to determine kernel VMA range.'
+      print output
+      return 1
     vma_start = re.search('([0-9a-fA-F]{8}) T _text', output).group(1)
     vma_end = re.search('([0-9a-fA-F]{8}) A _etext', output).group(1)
 
-    rc, output = self.adb.shell(['/system/xbin/opcontrol'] + self.opcontrol_verbose() + [
+    # Setup the profiler.
+    rc, output = self.adb.shell(['/system/xbin/opcontrol'] + self._opcontrol_verbose_arg() + [
       '--reset',
-      '--kernel-range=' + vma_start + '-' + vma_end,
-      #'--event=CPU_CYCLES:100000',
-      '--timer',
+      '--kernel-range=' + vma_start + '-' + vma_end] + opcontrol_args + [
       '--setup',
       '--status', '--verbose-log=all'])
+    if rc != 0:
+      print '* Failed to setup profiler.'
+      return 1
+    return 0
+
+  def do_shutdown(self, command_args):
+    if len(command_args) != 0:
+      print '* Unsupported shutdown command arguments: %s' % (' '.join(command_args))
+      return 2
 
-  def shutdown(self):
-    rc, output = self.adb.shell(['/system/xbin/opcontrol'] + self.opcontrol_verbose() + [
+    rc, output = self.adb.shell(['/system/xbin/opcontrol'] + self._opcontrol_verbose_arg() + [
       '--shutdown'])
     if rc != 0:
-      print 'Failed to shutdown.'
-      return rc
-    return rc
+      print '* Failed to shutdown.'
+      return 1
+    return 0
+
+  def do_start(self, command_args):
+    if len(command_args) != 0:
+      print '* Unsupported start command arguments: %s' % (' '.join(command_args))
+      return 2
 
-  def start(self):
-    rc, output = self.adb.shell(['/system/xbin/opcontrol'] + self.opcontrol_verbose() + [
+    rc, output = self.adb.shell(['/system/xbin/opcontrol'] + self._opcontrol_verbose_arg() + [
       '--start', '--status'])
-    return rc
+    if rc != 0:
+      print '* Failed to start profiler.'
+      return 1
+    return 0
 
-  def stop(self):
-    rc, output = self.adb.shell(['/system/xbin/opcontrol'] + self.opcontrol_verbose() + [
+  def do_stop(self, command_args):
+    if len(command_args) != 0:
+      print '* Unsupported stop command arguments: %s' % (' '.join(command_args))
+      return 2
+
+    rc, output = self.adb.shell(['/system/xbin/opcontrol'] + self._opcontrol_verbose_arg() + [
       '--stop', '--status'])
-    return rc
+    if rc != 0:
+      print '* Failed to stop profiler.'
+      return 1
+    return 0
+
+  def do_status(self, command_args):
+    if len(command_args) != 0:
+      print '* Unsupported status command arguments: %s' % (' '.join(command_args))
+      return 2
 
-  def status(self):
-    rc, output = self.adb.shell(['/system/xbin/opcontrol'] + self.opcontrol_verbose() + [
+    rc, output = self.adb.shell(['/system/xbin/opcontrol'] + self._opcontrol_verbose_arg() + [
       '--status'])
-    return rc
+    if rc != 0:
+      print '* Failed to get profiler status.'
+      return 1
+    return 0
+
+  def do_import(self, command_args):
+    force = False
+
+    try:
+      opts, args = getopt.getopt(command_args,
+        'f', ['force'])
+    except getopt.GetoptError, e:
+      print '* Unsupported import command arguments:', str(e)
+      return 2
+
+    for o, a in opts:
+      if o in ('-f', '--force'):
+        force = True
+
+    if len(args) != 0:
+      print '* Unsupported import command arguments: %s' % (' '.join(args))
+      return 2
+
+    # Create session directory.
+    print 'Creating session directory.'
+    if os.path.exists(self.session_dir):
+      if not force:
+        print "* Session directory already exists: %s" % (self.session_dir)
+        print "* Use --force to remove and recreate the session directory."
+        return 1
+
+      try:
+        shutil.rmtree(self.session_dir)
+      except e:
+        print "* Failed to remove existing session directory: %s" % (self.session_dir)
+        print e
+        return 1
 
-  def opcontrol_verbose(self):
+    try:
+      os.makedirs(self.session_dir)
+    except e:
+      print "* Failed to create session directory: %s" % (self.session_dir)
+      print e
+      return 1
+
+    raw_samples_dir = os.path.join(self.session_dir, 'raw_samples')
+    samples_dir = os.path.join(self.session_dir, 'samples')
+    abi_file = os.path.join(self.session_dir, 'abi')
+
+    # Dump samples.
+    print 'Dumping samples.'
+    rc, output = self.adb.shell(['/system/xbin/opcontrol'] + self._opcontrol_verbose_arg() + [
+      '--dump', '--status'])
+    if rc != 0:
+      print '* Failed to dump samples.'
+      print output
+      return 1
+
+    # Pull samples.
+    print 'Pulling samples from device.'
+    rc, output = self.adb.pull('/data/oprofile/samples/', raw_samples_dir + '/', echo=False)
+    if rc != 0:
+      print '* Failed to pull samples from the device.'
+      print output
+      return 1
+
+    # Pull ABI.
+    print 'Pulling ABI information from device.'
+    rc, output = self.adb.pull('/data/oprofile/abi', abi_file, echo=False)
+    if rc != 0:
+      print '* Failed to pull abi information from the device.'
+      print output
+      return 1
+
+    # Invoke opimport on each sample file to convert it from the device ABI (ARM)
+    # to the host ABI (x86).
+    print 'Importing samples.'
+    for dirpath, dirnames, filenames in os.walk(raw_samples_dir):
+      for filename in filenames:
+        if not re.match('^.*\.log$', filename):
+          in_path = os.path.join(dirpath, filename)
+          out_path = os.path.join(samples_dir, os.path.relpath(in_path, raw_samples_dir))
+          out_dir = os.path.dirname(out_path)
+          try:
+            os.makedirs(out_dir)
+          except e:
+            print "* Failed to create sample directory: %s" % (out_dir)
+            print e
+            return 1
+
+          rc, output = execute([opimport_bin, '-a', abi_file, '-o', out_path, in_path], echo=False)
+          if rc != 0:
+            print '* Failed to import samples.'
+            print output
+            return 1
+
+    # Generate a short summary report.
+    rc, output = self._execute_opreport([])
+    if rc != 0:
+      print '* Failed to generate summary report.'
+      return 1
+    return 0
+
+  def do_report(self, command_args):
+    rc, output = self._execute_opreport(command_args)
+    if rc != 0:
+      print '* Failed to generate report.'
+      return 1
+    return 0
+
+  def _opcontrol_verbose_arg(self):
     if self.verbose:
       return ['--verbose']
     else:
       return []
 
+  def _execute_opreport(self, args):
+    return execute([opreport_bin,
+      '--session-dir=' + self.session_dir,
+      '--image-path=' + symbols_dir + ',' + system_dir] + args)
+
 # Main entry point
 tool = Tool(sys.argv)
 rc = tool.main()