diff options
author | zhizhouy <zhizhouy@google.com> | 2020-04-06 17:17:55 -0700 |
---|---|---|
committer | Commit Bot <commit-bot@chromium.org> | 2020-04-07 11:00:50 +0000 |
commit | 47158bff1d1fb6e405548538b4a053e5b51d023e (patch) | |
tree | fb38ebf54d7aa284ae7014d06d10baad5516455c /deprecated/cwp/interpreter | |
parent | 85cd836b985d14c8332dc3b0a1241f06af48b5cd (diff) | |
download | toolchain-utils-47158bff1d1fb6e405548538b4a053e5b51d023e.tar.gz |
toolchain-utils: remove deprecated scripts
Per NextAction date in crbug.com/1051236, we would like to remove all
deprecated scripts in toolchain-utils for cleanup.
BUG=chromium:1051236
TEST=None
Change-Id: I06e4f3c39812d33daf47271325b5775c93929ab8
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2137942
Reviewed-by: George Burgess <gbiv@chromium.org>
Tested-by: Zhizhou Yang <zhizhouy@google.com>
Auto-Submit: Zhizhou Yang <zhizhouy@google.com>
Commit-Queue: George Burgess <gbiv@chromium.org>
Diffstat (limited to 'deprecated/cwp/interpreter')
-rw-r--r-- | deprecated/cwp/interpreter/app_engine_pull.py | 253 | ||||
-rw-r--r-- | deprecated/cwp/interpreter/symbolizer.py | 129 |
2 files changed, 0 insertions, 382 deletions
diff --git a/deprecated/cwp/interpreter/app_engine_pull.py b/deprecated/cwp/interpreter/app_engine_pull.py deleted file mode 100644 index d092e2a3..00000000 --- a/deprecated/cwp/interpreter/app_engine_pull.py +++ /dev/null @@ -1,253 +0,0 @@ -# Copyright 2012 Google Inc. All Rights Reserved. -# Author: mrdmnd@ (Matt Redmond) -"""A client to pull data from Bartlett. - -Inspired by //depot/google3/experimental/mobile_gwp/database/app_engine_pull.py - -The server houses perf.data.gz, board, chrome version for each upload. -This script first authenticates with a proper @google.com account, then -downloads a sample (if it's not already cached) and unzips perf.data - - Authenticate(): Gets login info and returns an auth token - DownloadSamples(): Download and unzip samples. - _GetServePage(): Pulls /serve page from the app engine server - _DownloadSampleFromServer(): Downloads a local compressed copy of a sample - _UncompressSample(): Decompresses a sample, deleting the compressed version. -""" -import cookielib -import getpass -import gzip -import optparse -import os -import urllib -import urllib2 - -SERVER_NAME = 'http://chromeoswideprofiling.appspot.com' -APP_NAME = 'chromeoswideprofiling' -DELIMITER = '~' - - -def Authenticate(server_name): - """Gets credentials from user and attempts to retrieve auth token. - TODO: Accept OAuth2 instead of password. - Args: - server_name: (string) URL that the app engine code is living on. - Returns: - authtoken: (string) The authorization token that can be used - to grab other pages. - """ - - if server_name.endswith('/'): - server_name = server_name.rstrip('/') - # Grab username and password from user through stdin. - username = raw_input('Email (must be @google.com account): ') - password = getpass.getpass('Password: ') - # Use a cookie to authenticate with GAE. - cookiejar = cookielib.LWPCookieJar() - opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar)) - urllib2.install_opener(opener) - # Get an AuthToken from Google accounts service. - auth_uri = 'https://www.google.com/accounts/ClientLogin' - authreq_data = urllib.urlencode({'Email': username, - 'Passwd': password, - 'service': 'ah', - 'source': APP_NAME, - 'accountType': 'HOSTED_OR_GOOGLE'}) - auth_req = urllib2.Request(auth_uri, data=authreq_data) - try: - auth_resp = urllib2.urlopen(auth_req) - except urllib2.URLError: - print 'Error logging in to Google accounts service.' - return None - body = auth_resp.read() - # Auth response contains several fields. - # We care about the part after Auth= - auth_resp_dict = dict(x.split('=') for x in body.split('\n') if x) - authtoken = auth_resp_dict['Auth'] - return authtoken - - -def DownloadSamples(server_name, authtoken, output_dir, start, stop): - """Download every sample and write unzipped version - to output directory. - Args: - server_name: (string) URL that the app engine code is living on. - authtoken: (string) Authorization token. - output_dir (string) Filepath to write output to. - start: (int) Index to start downloading from, starting at top. - stop: (int) Index to stop downloading, non-inclusive. -1 for end. - Returns: - None - """ - - if server_name.endswith('/'): - server_name = server_name.rstrip('/') - - serve_page_string = _GetServePage(server_name, authtoken) - if serve_page_string is None: - print 'Error getting /serve page.' - return - - sample_list = serve_page_string.split('</br>') - print 'Will download:' - sample_list_subset = sample_list[start:stop] - for sample in sample_list_subset: - print sample - for sample in sample_list_subset: - assert sample, 'Sample should be valid.' - sample_info = [s.strip() for s in sample.split(DELIMITER)] - key = sample_info[0] - time = sample_info[1] - time = time.replace(' ', '_') # No space between date and time. - # sample_md5 = sample_info[2] - board = sample_info[3] - version = sample_info[4] - - # Put a compressed copy of the samples in output directory. - _DownloadSampleFromServer(server_name, authtoken, key, time, board, version, - output_dir) - _UncompressSample(key, time, board, version, output_dir) - - -def _BuildFilenameFromParams(key, time, board, version): - """Return the filename for our sample. - Args: - key: (string) Key indexing our sample in the datastore. - time: (string) Date that the sample was uploaded. - board: (string) Board that the sample was taken on. - version: (string) Version string from /etc/lsb-release - Returns: - filename (string) - """ - filename = DELIMITER.join([key, time, board, version]) - return filename - - -def _DownloadSampleFromServer(server_name, authtoken, key, time, board, version, - output_dir): - """Downloads sample_$(samplekey).gz to current dir. - Args: - server_name: (string) URL that the app engine code is living on. - authtoken: (string) Authorization token. - key: (string) Key indexing our sample in the datastore - time: (string) Date that the sample was uploaded. - board: (string) Board that the sample was taken on. - version: (string) Version string from /etc/lsb-release - output_dir: (string) Filepath to write to output to. - Returns: - None - """ - filename = _BuildFilenameFromParams(key, time, board, version) - compressed_filename = filename + '.gz' - - if os.path.exists(os.path.join(output_dir, filename)): - print 'Already downloaded %s, skipping.' % filename - return - - serv_uri = server_name + '/serve/' + key - serv_args = {'continue': serv_uri, 'auth': authtoken} - full_serv_uri = server_name + '/_ah/login?%s' % urllib.urlencode(serv_args) - serv_req = urllib2.Request(full_serv_uri) - serv_resp = urllib2.urlopen(serv_req) - f = open(os.path.join(output_dir, compressed_filename), 'w+') - f.write(serv_resp.read()) - f.close() - - -def _UncompressSample(key, time, board, version, output_dir): - """Uncompresses a given sample.gz file and deletes the compressed version. - Args: - key: (string) Sample key to uncompress. - time: (string) Date that the sample was uploaded. - board: (string) Board that the sample was taken on. - version: (string) Version string from /etc/lsb-release - output_dir: (string) Filepath to find sample key in. - Returns: - None - """ - filename = _BuildFilenameFromParams(key, time, board, version) - compressed_filename = filename + '.gz' - - if os.path.exists(os.path.join(output_dir, filename)): - print 'Already decompressed %s, skipping.' % filename - return - - out_file = open(os.path.join(output_dir, filename), 'wb') - in_file = gzip.open(os.path.join(output_dir, compressed_filename), 'rb') - out_file.write(in_file.read()) - in_file.close() - out_file.close() - os.remove(os.path.join(output_dir, compressed_filename)) - - -def _DeleteSampleFromServer(server_name, authtoken, key): - """Opens the /delete page with the specified key - to delete the sample off the datastore. - Args: - server_name: (string) URL that the app engine code is living on. - authtoken: (string) Authorization token. - key: (string) Key to delete. - Returns: - None - """ - - serv_uri = server_name + '/del/' + key - serv_args = {'continue': serv_uri, 'auth': authtoken} - full_serv_uri = server_name + '/_ah/login?%s' % urllib.urlencode(serv_args) - serv_req = urllib2.Request(full_serv_uri) - urllib2.urlopen(serv_req) - - -def _GetServePage(server_name, authtoken): - """Opens the /serve page and lists all keys. - Args: - server_name: (string) URL the app engine code is living on. - authtoken: (string) Authorization token. - Returns: - The text of the /serve page (including HTML tags) - """ - - serv_uri = server_name + '/serve' - serv_args = {'continue': serv_uri, 'auth': authtoken} - full_serv_uri = server_name + '/_ah/login?%s' % urllib.urlencode(serv_args) - serv_req = urllib2.Request(full_serv_uri) - serv_resp = urllib2.urlopen(serv_req) - return serv_resp.read() - - -def main(): - parser = optparse.OptionParser() - parser.add_option('--output_dir', - dest='output_dir', - action='store', - help='Path to output perf data files.') - parser.add_option('--start', - dest='start_ind', - action='store', - default=0, - help='Start index.') - parser.add_option('--stop', - dest='stop_ind', - action='store', - default=-1, - help='Stop index.') - options = parser.parse_args()[0] - if not options.output_dir: - print 'Must specify --output_dir.' - return 1 - if not os.path.exists(options.output_dir): - print 'Specified output_dir does not exist.' - return 1 - - authtoken = Authenticate(SERVER_NAME) - if not authtoken: - print 'Could not obtain authtoken, exiting.' - return 1 - DownloadSamples(SERVER_NAME, authtoken, options.output_dir, options.start_ind, - options.stop_ind) - print 'Downloaded samples.' - return 0 - - -if __name__ == '__main__': - exit(main()) diff --git a/deprecated/cwp/interpreter/symbolizer.py b/deprecated/cwp/interpreter/symbolizer.py deleted file mode 100644 index 4ece480d..00000000 --- a/deprecated/cwp/interpreter/symbolizer.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2012 Google Inc. All Rights Reserved. -"""A script that symbolizes perf.data files.""" -import optparse -import os -import shutil -from subprocess import call -from subprocess import PIPE -from subprocess import Popen -from cros_utils import misc - -GSUTIL_CMD = 'gsutil cp gs://chromeos-image-archive/%s-release/%s/debug.tgz %s' -TAR_CMD = 'tar -zxvf %s -C %s' -PERF_BINARY = '/google/data/ro/projects/perf/perf' -VMLINUX_FLAG = ' --vmlinux=/usr/lib/debug/boot/vmlinux' -PERF_CMD = PERF_BINARY + ' report -i %s -n --symfs=%s' + VMLINUX_FLAG - - -def main(): - parser = optparse.OptionParser() - parser.add_option('--in', dest='in_dir') - parser.add_option('--out', dest='out_dir') - parser.add_option('--cache', dest='cache') - (opts, _) = parser.parse_args() - if not _ValidateOpts(opts): - return 1 - else: - for filename in os.listdir(opts.in_dir): - try: - _DownloadSymbols(filename, opts.cache) - _PerfReport(filename, opts.in_dir, opts.out_dir, opts.cache) - except: - print 'Exception caught. Continuing...' - return 0 - - -def _ValidateOpts(opts): - """Ensures all directories exist, before attempting to populate.""" - if not os.path.exists(opts.in_dir): - print "Input directory doesn't exist." - return False - if not os.path.exists(opts.out_dir): - print "Output directory doesn't exist. Creating it..." - os.makedirs(opts.out_dir) - if not os.path.exists(opts.cache): - print "Cache directory doesn't exist." - return False - return True - - -def _ParseFilename(filename, canonical=False): - """Returns a tuple (key, time, board, lsb_version). - If canonical is True, instead returns (database_key, board, canonical_vers) - canonical_vers includes the revision string. - """ - key, time, board, vers = filename.split('~') - if canonical: - vers = misc.GetChromeOSVersionFromLSBVersion(vers) - return (key, time, board, vers) - - -def _FormReleaseDir(board, version): - return '%s-release~%s' % (board, version) - - -def _DownloadSymbols(filename, cache): - """ Incrementally downloads appropriate symbols. - We store the downloads in cache, with each set of symbols in a TLD - named like cache/$board-release~$canonical_vers/usr/lib/debug - """ - _, _, board, vers = _ParseFilename(filename, canonical=True) - tmp_suffix = '.tmp' - - tarball_subdir = _FormReleaseDir(board, vers) - tarball_dir = os.path.join(cache, tarball_subdir) - tarball_path = os.path.join(tarball_dir, 'debug.tgz') - - symbol_subdir = os.path.join('usr', 'lib') - symbol_dir = os.path.join(tarball_dir, symbol_subdir) - - if os.path.isdir(symbol_dir): - print 'Symbol directory %s exists, skipping download.' % symbol_dir - return - else: - # First download using gsutil. - if not os.path.isfile(tarball_path): - download_cmd = GSUTIL_CMD % (board, vers, tarball_path + tmp_suffix) - print 'Downloading symbols for %s' % filename - print download_cmd - ret = call(download_cmd.split()) - if ret != 0: - print 'gsutil returned non-zero error code: %s.' % ret - # Clean up the empty directory structures. - os.remove(tarball_path + tmp_suffix) - raise IOError - - shutil.move(tarball_path + tmp_suffix, tarball_path) - - # Next, untar the tarball. - os.makedirs(symbol_dir + tmp_suffix) - extract_cmd = TAR_CMD % (tarball_path, symbol_dir + tmp_suffix) - print 'Extracting symbols for %s' % filename - print extract_cmd - ret = call(extract_cmd.split()) - if ret != 0: - print 'tar returned non-zero code: %s.' % ret - raise IOError - shutil.move(symbol_dir + tmp_suffix, symbol_dir) - os.remove(tarball_path) - - -def _PerfReport(filename, in_dir, out_dir, cache): - """ Call perf report on the file, storing output to the output dir. - The output is currently stored as $out_dir/$filename - """ - _, _, board, vers = _ParseFilename(filename, canonical=True) - symbol_cache_tld = _FormReleaseDir(board, vers) - input_file = os.path.join(in_dir, filename) - symfs = os.path.join(cache, symbol_cache_tld) - report_cmd = PERF_CMD % (input_file, symfs) - print 'Reporting.' - print report_cmd - report_proc = Popen(report_cmd.split(), stdout=PIPE) - outfile = open(os.path.join(out_dir, filename), 'w') - outfile.write(report_proc.stdout.read()) - outfile.close() - - -if __name__ == '__main__': - exit(main()) |