2 files changed, 382 insertions, 0 deletions
diff --git a/deprecated/cwp/interpreter/app_engine_pull.py b/deprecated/cwp/interpreter/app_engine_pull.py
new file mode 100644
index 00000000..d092e2a3
--- /dev/null
+++ b/deprecated/cwp/interpreter/app_engine_pull.py
@@ -0,0 +1,253 @@
+# Copyright 2012 Google Inc. All Rights Reserved.
+# Author: mrdmnd@ (Matt Redmond)
+"""A client to pull data from Bartlett.
+
+Inspired by //depot/google3/experimental/mobile_gwp/database/app_engine_pull.py
+
+The server houses perf.data.gz, board, chrome version for each upload.
+This script first authenticates with a proper @google.com account, then
+downloads a sample (if it's not already cached) and unzips perf.data
+
+  Authenticate(): Gets login info and returns an auth token
+  DownloadSamples(): Download and unzip samples.
+  _GetServePage(): Pulls /serve page from the app engine server
+  _DownloadSampleFromServer(): Downloads a local compressed copy of a sample
+  _UncompressSample(): Decompresses a sample, deleting the compressed version.
+"""
+import cookielib
+import getpass
+import gzip
+import optparse
+import os
+import urllib
+import urllib2
+
+SERVER_NAME = 'http://chromeoswideprofiling.appspot.com'
+APP_NAME = 'chromeoswideprofiling'
+DELIMITER = '~'
+
+
+def Authenticate(server_name):
+  """Gets credentials from user and attempts to retrieve auth token.
+     TODO: Accept OAuth2 instead of password.
+  Args:
+    server_name: (string) URL that the app engine code is living on.
+  Returns:
+    authtoken: (string) The authorization token that can be used
+                        to grab other pages.
+  """
+
+  if server_name.endswith('/'):
+    server_name = server_name.rstrip('/')
+  # Grab username and password from user through stdin.
+  username = raw_input('Email (must be @google.com account): ')
+  password = getpass.getpass('Password: ')
+  # Use a cookie to authenticate with GAE.
+  cookiejar = cookielib.LWPCookieJar()
+  opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar))
+  urllib2.install_opener(opener)
+  # Get an AuthToken from Google accounts service.
+  auth_uri = 'https://www.google.com/accounts/ClientLogin'
+  authreq_data = urllib.urlencode({'Email': username,
+                                   'Passwd': password,
+                                   'service': 'ah',
+                                   'source': APP_NAME,
+                                   'accountType': 'HOSTED_OR_GOOGLE'})
+  auth_req = urllib2.Request(auth_uri, data=authreq_data)
+  try:
+    auth_resp = urllib2.urlopen(auth_req)
+  except urllib2.URLError:
+    print 'Error logging in to Google accounts service.'
+    return None
+  body = auth_resp.read()
+  # Auth response contains several fields.
+  # We care about the part after Auth=
+  auth_resp_dict = dict(x.split('=') for x in body.split('\n') if x)
+  authtoken = auth_resp_dict['Auth']
+  return authtoken
+
+
+def DownloadSamples(server_name, authtoken, output_dir, start, stop):
+  """Download every sample and write unzipped version
+     to output directory.
+  Args:
+    server_name: (string) URL that the app engine code is living on.
+    authtoken:   (string) Authorization token.
+    output_dir   (string) Filepath to write output to.
+    start:       (int)    Index to start downloading from, starting at top.
+    stop:        (int)    Index to stop downloading, non-inclusive. -1 for end.
+  Returns:
+    None
+  """
+
+  if server_name.endswith('/'):
+    server_name = server_name.rstrip('/')
+
+  serve_page_string = _GetServePage(server_name, authtoken)
+  if serve_page_string is None:
+    print 'Error getting /serve page.'
+    return
+
+  sample_list = serve_page_string.split('</br>')
+  print 'Will download:'
+  sample_list_subset = sample_list[start:stop]
+  for sample in sample_list_subset:
+    print sample
+  for sample in sample_list_subset:
+    assert sample, 'Sample should be valid.'
+    sample_info = [s.strip() for s in sample.split(DELIMITER)]
+    key = sample_info[0]
+    time = sample_info[1]
+    time = time.replace(' ', '_')  # No space between date and time.
+    # sample_md5 = sample_info[2]
+    board = sample_info[3]
+    version = sample_info[4]
+
+    # Put a compressed copy of the samples in output directory.
+    _DownloadSampleFromServer(server_name, authtoken, key, time, board, version,
+                              output_dir)
+    _UncompressSample(key, time, board, version, output_dir)
+
+
+def _BuildFilenameFromParams(key, time, board, version):
+  """Return the filename for our sample.
+  Args:
+    key:  (string) Key indexing our sample in the datastore.
+    time: (string) Date that the sample was uploaded.
+    board: (string) Board that the sample was taken on.
+    version: (string) Version string from /etc/lsb-release
+  Returns:
+    filename (string)
+  """
+  filename = DELIMITER.join([key, time, board, version])
+  return filename
+
+
+def _DownloadSampleFromServer(server_name, authtoken, key, time, board, version,
+                              output_dir):
+  """Downloads sample_$(samplekey).gz to current dir.
+  Args:
+    server_name: (string) URL that the app engine code is living on.
+    authtoken:   (string) Authorization token.
+    key:  (string) Key indexing our sample in the datastore
+    time: (string) Date that the sample was uploaded.
+    board: (string) Board that the sample was taken on.
+    version: (string) Version string from /etc/lsb-release
+    output_dir:  (string) Filepath to write to output to.
+  Returns:
+    None
+  """
+  filename = _BuildFilenameFromParams(key, time, board, version)
+  compressed_filename = filename + '.gz'
+
+  if os.path.exists(os.path.join(output_dir, filename)):
+    print 'Already downloaded %s, skipping.' % filename
+    return
+
+  serv_uri = server_name + '/serve/' + key
+  serv_args = {'continue': serv_uri, 'auth': authtoken}
+  full_serv_uri = server_name + '/_ah/login?%s' % urllib.urlencode(serv_args)
+  serv_req = urllib2.Request(full_serv_uri)
+  serv_resp = urllib2.urlopen(serv_req)
+  f = open(os.path.join(output_dir, compressed_filename), 'w+')
+  f.write(serv_resp.read())
+  f.close()
+
+
+def _UncompressSample(key, time, board, version, output_dir):
+  """Uncompresses a given sample.gz file and deletes the compressed version.
+  Args:
+    key: (string) Sample key to uncompress.
+    time: (string) Date that the sample was uploaded.
+    board: (string) Board that the sample was taken on.
+    version: (string) Version string from /etc/lsb-release
+    output_dir: (string) Filepath to find sample key in.
+  Returns:
+    None
+  """
+  filename = _BuildFilenameFromParams(key, time, board, version)
+  compressed_filename = filename + '.gz'
+
+  if os.path.exists(os.path.join(output_dir, filename)):
+    print 'Already decompressed %s, skipping.' % filename
+    return
+
+  out_file = open(os.path.join(output_dir, filename), 'wb')
+  in_file = gzip.open(os.path.join(output_dir, compressed_filename), 'rb')
+  out_file.write(in_file.read())
+  in_file.close()
+  out_file.close()
+  os.remove(os.path.join(output_dir, compressed_filename))
+
+
+def _DeleteSampleFromServer(server_name, authtoken, key):
+  """Opens the /delete page with the specified key
+     to delete the sample off the datastore.
+    Args:
+      server_name: (string) URL that the app engine code is living on.
+      authtoken:   (string) Authorization token.
+      key:  (string) Key to delete.
+    Returns:
+      None
+  """
+
+  serv_uri = server_name + '/del/' + key
+  serv_args = {'continue': serv_uri, 'auth': authtoken}
+  full_serv_uri = server_name + '/_ah/login?%s' % urllib.urlencode(serv_args)
+  serv_req = urllib2.Request(full_serv_uri)
+  urllib2.urlopen(serv_req)
+
+
+def _GetServePage(server_name, authtoken):
+  """Opens the /serve page and lists all keys.
+  Args:
+    server_name: (string) URL the app engine code is living on.
+    authtoken:   (string) Authorization token.
+  Returns:
+    The text of the /serve page (including HTML tags)
+  """
+
+  serv_uri = server_name + '/serve'
+  serv_args = {'continue': serv_uri, 'auth': authtoken}
+  full_serv_uri = server_name + '/_ah/login?%s' % urllib.urlencode(serv_args)
+  serv_req = urllib2.Request(full_serv_uri)
+  serv_resp = urllib2.urlopen(serv_req)
+  return serv_resp.read()
+
+
+def main():
+  parser = optparse.OptionParser()
+  parser.add_option('--output_dir',
+                    dest='output_dir',
+                    action='store',
+                    help='Path to output perf data files.')
+  parser.add_option('--start',
+                    dest='start_ind',
+                    action='store',
+                    default=0,
+                    help='Start index.')
+  parser.add_option('--stop',
+                    dest='stop_ind',
+                    action='store',
+                    default=-1,
+                    help='Stop index.')
+  options = parser.parse_args()[0]
+  if not options.output_dir:
+    print 'Must specify --output_dir.'
+    return 1
+  if not os.path.exists(options.output_dir):
+    print 'Specified output_dir does not exist.'
+    return 1
+
+  authtoken = Authenticate(SERVER_NAME)
+  if not authtoken:
+    print 'Could not obtain authtoken, exiting.'
+    return 1
+  DownloadSamples(SERVER_NAME, authtoken, options.output_dir, options.start_ind,
+                  options.stop_ind)
+  print 'Downloaded samples.'
+  return 0
+
+
+if __name__ == '__main__':
+  exit(main())
diff --git a/deprecated/cwp/interpreter/symbolizer.py b/deprecated/cwp/interpreter/symbolizer.py
new file mode 100644
index 00000000..4ece480d
--- /dev/null
+++ b/deprecated/cwp/interpreter/symbolizer.py
@@ -0,0 +1,129 @@
+# Copyright 2012 Google Inc. All Rights Reserved.
+"""A script that symbolizes perf.data files."""
+import optparse
+import os
+import shutil
+from subprocess import call
+from subprocess import PIPE
+from subprocess import Popen
+from cros_utils import misc
+
+GSUTIL_CMD = 'gsutil cp gs://chromeos-image-archive/%s-release/%s/debug.tgz %s'
+TAR_CMD = 'tar -zxvf %s -C %s'
+PERF_BINARY = '/google/data/ro/projects/perf/perf'
+VMLINUX_FLAG = ' --vmlinux=/usr/lib/debug/boot/vmlinux'
+PERF_CMD = PERF_BINARY + ' report -i %s -n --symfs=%s' + VMLINUX_FLAG
+
+
+def main():
+  parser = optparse.OptionParser()
+  parser.add_option('--in', dest='in_dir')
+  parser.add_option('--out', dest='out_dir')
+  parser.add_option('--cache', dest='cache')
+  (opts, _) = parser.parse_args()
+  if not _ValidateOpts(opts):
+    return 1
+  else:
+    for filename in os.listdir(opts.in_dir):
+      try:
+        _DownloadSymbols(filename, opts.cache)
+        _PerfReport(filename, opts.in_dir, opts.out_dir, opts.cache)
+      except:
+        print 'Exception caught. Continuing...'
+  return 0
+
+
+def _ValidateOpts(opts):
+  """Ensures all directories exist, before attempting to populate."""
+  if not os.path.exists(opts.in_dir):
+    print "Input directory doesn't exist."
+    return False
+  if not os.path.exists(opts.out_dir):
+    print "Output directory doesn't exist. Creating it..."
+    os.makedirs(opts.out_dir)
+  if not os.path.exists(opts.cache):
+    print "Cache directory doesn't exist."
+    return False
+  return True
+
+
+def _ParseFilename(filename, canonical=False):
+  """Returns a tuple (key, time, board, lsb_version).
+     If canonical is True, instead returns (database_key, board, canonical_vers)
+     canonical_vers includes the revision string.
+  """
+  key, time, board, vers = filename.split('~')
+  if canonical:
+    vers = misc.GetChromeOSVersionFromLSBVersion(vers)
+  return (key, time, board, vers)
+
+
+def _FormReleaseDir(board, version):
+  return '%s-release~%s' % (board, version)
+
+
+def _DownloadSymbols(filename, cache):
+  """ Incrementally downloads appropriate symbols.
+      We store the downloads in cache, with each set of symbols in a TLD
+      named like cache/$board-release~$canonical_vers/usr/lib/debug
+  """
+  _, _, board, vers = _ParseFilename(filename, canonical=True)
+  tmp_suffix = '.tmp'
+
+  tarball_subdir = _FormReleaseDir(board, vers)
+  tarball_dir = os.path.join(cache, tarball_subdir)
+  tarball_path = os.path.join(tarball_dir, 'debug.tgz')
+
+  symbol_subdir = os.path.join('usr', 'lib')
+  symbol_dir = os.path.join(tarball_dir, symbol_subdir)
+
+  if os.path.isdir(symbol_dir):
+    print 'Symbol directory %s exists, skipping download.' % symbol_dir
+    return
+  else:
+    # First download using gsutil.
+    if not os.path.isfile(tarball_path):
+      download_cmd = GSUTIL_CMD % (board, vers, tarball_path + tmp_suffix)
+      print 'Downloading symbols for %s' % filename
+      print download_cmd
+      ret = call(download_cmd.split())
+      if ret != 0:
+        print 'gsutil returned non-zero error code: %s.' % ret
+        # Clean up the empty directory structures.
+        os.remove(tarball_path + tmp_suffix)
+        raise IOError
+
+      shutil.move(tarball_path + tmp_suffix, tarball_path)
+
+    # Next, untar the tarball.
+    os.makedirs(symbol_dir + tmp_suffix)
+    extract_cmd = TAR_CMD % (tarball_path, symbol_dir + tmp_suffix)
+    print 'Extracting symbols for %s' % filename
+    print extract_cmd
+    ret = call(extract_cmd.split())
+    if ret != 0:
+      print 'tar returned non-zero code: %s.' % ret
+      raise IOError
+    shutil.move(symbol_dir + tmp_suffix, symbol_dir)
+    os.remove(tarball_path)
+
+
+def _PerfReport(filename, in_dir, out_dir, cache):
+  """ Call perf report on the file, storing output to the output dir.
+      The output is currently stored as $out_dir/$filename
+  """
+  _, _, board, vers = _ParseFilename(filename, canonical=True)
+  symbol_cache_tld = _FormReleaseDir(board, vers)
+  input_file = os.path.join(in_dir, filename)
+  symfs = os.path.join(cache, symbol_cache_tld)
+  report_cmd = PERF_CMD % (input_file, symfs)
+  print 'Reporting.'
+  print report_cmd
+  report_proc = Popen(report_cmd.split(), stdout=PIPE)
+  outfile = open(os.path.join(out_dir, filename), 'w')
+  outfile.write(report_proc.stdout.read())
+  outfile.close()
+
+
+if __name__ == '__main__':
+  exit(main())