diff options
Diffstat (limited to 'deprecated/cwp')
| -rw-r--r-- | deprecated/cwp/bartlett/app.yaml | 22 | ||||
| -rwxr-xr-x | deprecated/cwp/bartlett/server.py | 153 | ||||
| -rw-r--r-- | deprecated/cwp/bartlett/static/favicon.ico | bin | 198 -> 0 bytes | |||
| -rw-r--r-- | deprecated/cwp/bartlett/test/server_tester.py | 101 | ||||
| -rwxr-xr-x | deprecated/cwp/bartlett/update_appengine_server | 1 | ||||
| -rw-r--r-- | deprecated/cwp/demo_pipeline.sh | 55 | ||||
| -rw-r--r-- | deprecated/cwp/interpreter/app_engine_pull.py | 253 | ||||
| -rw-r--r-- | deprecated/cwp/interpreter/symbolizer.py | 129 | ||||
| -rw-r--r-- | deprecated/cwp/performance/experiment_gen.py | 138 |
9 files changed, 0 insertions, 852 deletions
diff --git a/deprecated/cwp/bartlett/app.yaml b/deprecated/cwp/bartlett/app.yaml deleted file mode 100644 index 60010f70..00000000 --- a/deprecated/cwp/bartlett/app.yaml +++ /dev/null @@ -1,22 +0,0 @@ -application: chromeoswideprofiling -version: 1 -runtime: python -api_version: 1 - -handlers: -- url: /favicon.ico - static_files: static/favicon.ico - upload: static/favicon.ico -- url: /remote_api - script: $PYTHON_LIB/google/appengine/ext/remote_api/handler.py - login: admin -- url: / - script: server.py -- url: /upload - script: server.py -- url: /serve - script: server.py -- url: /serve/.* - script: server.py -- url: /del/.* - script: server.py diff --git a/deprecated/cwp/bartlett/server.py b/deprecated/cwp/bartlett/server.py deleted file mode 100755 index 8fb7d61e..00000000 --- a/deprecated/cwp/bartlett/server.py +++ /dev/null @@ -1,153 +0,0 @@ -#!/usr/bin/python2 -# Copyright 2012 Google Inc. All Rights Reserved. -# Author: mrdmnd@ (Matt Redmond) -# Based off of code in //depot/google3/experimental/mobile_gwp -"""Code to transport profile data between a user's machine and the CWP servers. - Pages: - "/": the main page for the app, left blank so that users cannot access - the file upload but left in the code for debugging purposes - "/upload": Updates the datastore with a new file. the upload depends on - the format which is templated on the main page ("/") - input includes: - profile_data: the zipped file containing profile data - board: the architecture we ran on - chromeos_version: the chromeos_version - "/serve": Lists all of the files in the datastore. Each line is a new entry - in the datastore. The format is key~date, where key is the entry's - key in the datastore and date is the file upload time and date. - (Authentication Required) - "/serve/([^/]+)?": For downloading a file of profile data, ([^/]+)? means - any character sequence so to download the file go to - '/serve/$key' where $key is the datastore key of the file - you want to download. - (Authentication Required) - "/del/([^/]+)?": For deleting an entry in the datastore. To use go to - '/del/$key' where $key is the datastore key of the entry - you want to be deleted form the datastore. - (Authentication Required) - TODO: Add more extensive logging""" - -import cgi -import logging -import md5 -import urllib - -from google.appengine.api import users -from google.appengine.ext import db -from google.appengine.ext import webapp -from google.appengine.ext.webapp.util import run_wsgi_app - -logging.getLogger().setLevel(logging.DEBUG) - - -class FileEntry(db.Model): - profile_data = db.BlobProperty() # The profile data - date = db.DateTimeProperty(auto_now_add=True) # Date it was uploaded - data_md5 = db.ByteStringProperty() # md5 of the profile data - board = db.StringProperty() # board arch - chromeos_version = db.StringProperty() # ChromeOS version - - -class MainPage(webapp.RequestHandler): - """Main page only used as the form template, not actually displayed.""" - - def get(self, response=''): # pylint: disable-msg=C6409 - if response: - self.response.out.write('<html><body>') - self.response.out.write("""<br> - <form action="/upload" enctype="multipart/form-data" method="post"> - <div><label>Profile Data:</label></div> - <div><input type="file" name="profile_data"/></div> - <div><label>Board</label></div> - <div><input type="text" name="board"/></div> - <div><label>ChromeOS Version</label></div> - <div><input type="text" name="chromeos_version"></div> - <div><input type="submit" value="send" name="submit"></div> - </form> - </body> - </html>""") - - -class Upload(webapp.RequestHandler): - """Handler for uploading data to the datastore, accessible by anyone.""" - - def post(self): # pylint: disable-msg=C6409 - """Takes input based on the main page's form.""" - getfile = FileEntry() - f1 = self.request.get('profile_data') - getfile.profile_data = db.Blob(f1) - getfile.data_md5 = md5.new(f1).hexdigest() - getfile.board = self.request.get('board') - getfile.chromeos_version = self.request.get('chromeos_version') - getfile.put() - self.response.out.write(getfile.key()) - #self.redirect('/') - - -class ServeHandler(webapp.RequestHandler): - """Given the entry's key in the database, output the profile data file. Only - accessible from @google.com accounts.""" - - def get(self, resource): # pylint: disable-msg=C6409 - if Authenticate(self): - file_key = str(urllib.unquote(resource)) - request = db.get(file_key) - self.response.out.write(request.profile_data) - - -class ListAll(webapp.RequestHandler): - """Displays all files uploaded. Only accessible by @google.com accounts.""" - - def get(self): # pylint: disable-msg=C6409 - """Displays all information in FileEntry, ~ delimited.""" - if Authenticate(self): - query_str = 'SELECT * FROM FileEntry ORDER BY date ASC' - query = db.GqlQuery(query_str) - delimiter = '~' - - for item in query: - display_list = [item.key(), item.date, item.data_md5, item.board, - item.chromeos_version] - str_list = [cgi.escape(str(i)) for i in display_list] - self.response.out.write(delimiter.join(str_list) + '</br>') - - -class DelEntries(webapp.RequestHandler): - """Deletes entries. Only accessible from @google.com accounts.""" - - def get(self, resource): # pylint: disable-msg=C6409 - """A specific entry is deleted, when the key is given.""" - if Authenticate(self): - fkey = str(urllib.unquote(resource)) - request = db.get(fkey) - if request: - db.delete(fkey) - - -def Authenticate(webpage): - """Some urls are only accessible if logged in with a @google.com account.""" - user = users.get_current_user() - if user is None: - webpage.redirect(users.create_login_url(webpage.request.uri)) - elif user.email().endswith('@google.com'): - return True - else: - webpage.response.out.write('Not Authenticated') - return False - - -def main(): - application = webapp.WSGIApplication( - [ - ('/', MainPage), - ('/upload', Upload), - ('/serve/([^/]+)?', ServeHandler), - ('/serve', ListAll), - ('/del/([^/]+)?', DelEntries), - ], - debug=False) - run_wsgi_app(application) - - -if __name__ == '__main__': - main() diff --git a/deprecated/cwp/bartlett/static/favicon.ico b/deprecated/cwp/bartlett/static/favicon.ico Binary files differdeleted file mode 100644 index 19b58c2e..00000000 --- a/deprecated/cwp/bartlett/static/favicon.ico +++ /dev/null diff --git a/deprecated/cwp/bartlett/test/server_tester.py b/deprecated/cwp/bartlett/test/server_tester.py deleted file mode 100644 index 585da43a..00000000 --- a/deprecated/cwp/bartlett/test/server_tester.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright 2012 Google Inc. All Rights Reserved. -# Author: mrdmnd@ (Matt Redmond) -"""A unit test for sending data to Bartlett. Requires poster module.""" - -import cookielib -import os -import signal -import subprocess -import tempfile -import time -import unittest -import urllib2 - -from poster.encode import multipart_encode -from poster.streaminghttp import register_openers - -SERVER_DIR = '../.' -SERVER_URL = 'http://localhost:8080/' -GET = '_ah/login?email=googler@google.com&action=Login&continue=%s' -AUTH_URL = SERVER_URL + GET - - -class ServerTest(unittest.TestCase): - """A unit test for the bartlett server. Tests upload, serve, and delete.""" - - def setUp(self): - """Instantiate the files and server needed to test upload functionality.""" - self._server_proc = LaunchLocalServer() - self._jar = cookielib.LWPCookieJar() - self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self._jar)) - - # We need these files to not delete when closed, because we have to reopen - # them in read mode after we write and close them. - self.profile_data = tempfile.NamedTemporaryFile(delete=False) - - size = 16 * 1024 - self.profile_data.write(os.urandom(size)) - - def tearDown(self): - self.profile_data.close() - os.remove(self.profile_data.name) - os.kill(self._server_proc.pid, signal.SIGINT) - - def testIntegration(self): # pylint: disable-msg=C6409 - key = self._testUpload() - self._testListAll() - self._testServeKey(key) - self._testDelKey(key) - - def _testUpload(self): # pylint: disable-msg=C6409 - register_openers() - data = {'profile_data': self.profile_data, - 'board': 'x86-zgb', - 'chromeos_version': '2409.0.2012_06_08_1114'} - datagen, headers = multipart_encode(data) - request = urllib2.Request(SERVER_URL + 'upload', datagen, headers) - response = urllib2.urlopen(request).read() - self.assertTrue(response) - return response - - def _testListAll(self): # pylint: disable-msg=C6409 - request = urllib2.Request(AUTH_URL % (SERVER_URL + 'serve')) - response = self.opener.open(request).read() - self.assertTrue(response) - - def _testServeKey(self, key): # pylint: disable-msg=C6409 - request = urllib2.Request(AUTH_URL % (SERVER_URL + 'serve/' + key)) - response = self.opener.open(request).read() - self.assertTrue(response) - - def _testDelKey(self, key): # pylint: disable-msg=C6409 - # There is no response to a delete request. - # We will check the listAll page to ensure there is no data. - request = urllib2.Request(AUTH_URL % (SERVER_URL + 'del/' + key)) - response = self.opener.open(request).read() - request = urllib2.Request(AUTH_URL % (SERVER_URL + 'serve')) - response = self.opener.open(request).read() - self.assertFalse(response) - - -def LaunchLocalServer(): - """Launch and store an authentication cookie with a local server.""" - proc = subprocess.Popen( - ['dev_appserver.py', '--clear_datastore', SERVER_DIR], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - # Wait for server to come up - while True: - time.sleep(1) - try: - request = urllib2.Request(SERVER_URL + 'serve') - response = urllib2.urlopen(request).read() - if response: - break - except urllib2.URLError: - continue - return proc - - -if __name__ == '__main__': - unittest.main() diff --git a/deprecated/cwp/bartlett/update_appengine_server b/deprecated/cwp/bartlett/update_appengine_server deleted file mode 100755 index f3812057..00000000 --- a/deprecated/cwp/bartlett/update_appengine_server +++ /dev/null @@ -1 +0,0 @@ -appcfg.py --oauth2 update . diff --git a/deprecated/cwp/demo_pipeline.sh b/deprecated/cwp/demo_pipeline.sh deleted file mode 100644 index d45c5c44..00000000 --- a/deprecated/cwp/demo_pipeline.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash - -# These should be on the local filesystem. We'll be hitting it hard. -DATA_DIR=/usr/local/google/home/${USER}/ -SYMBOL_CACHE=${DATA_DIR}cache/ -REPORT_DIR=${DATA_DIR}reports/ -SAMPLE_DIR=${DATA_DIR}samples/ -RECORD_FILE=/tmp/profiles.rio -COLUMN_FILE=/tmp/profiles.cio -mkdir -p ${SYMBOL_CACHE} -mkdir -p ${REPORT_DIR} -mkdir -p ${SAMPLE_DIR} - -# Directory that has the scripts app_engine_pull.py and symbolizer.py -INTERPRETER_DIR=/google/src/files/p2/head/depot2/gcctools/chromeos/v14/cwp/interpreter/ -V14_DIR=$(dirname $(dirname ${INTERPRETER_DIR})) - -PYTHONPATH=$PYTHONPATH:$V14_DIR - -# Profile util binary -PROFILE_UTIL_BINARY=/home/mrdmnd/${USER}-profiledb/google3/blaze-bin/perftools/gwp/chromeos/profile_util - -# mr-convert binary -MR_CONVERT_BINARY=/home/build/static/projects/dremel/mr-convert - -CNS_LOC=/cns/ag-d/home/${USER}/profiledb/ - -# Protofile location -PROTO_LOC=${CNS_LOC}cwp_profile_db_entry.proto - -echo "0. Cleaning up old data." -rm /tmp/profiles.* -rm ${REPORT_DIR}* -rm ${SAMPLE_DIR}* - - -echo "Starting CWP Pipeline..." -echo "1. Pulling samples to local filesystem from server." -echo " For demo purposes, UN=${USER}@google.com, PW=xjpbmshkzefutlrm" -python ${INTERPRETER_DIR}app_engine_pull.py --output_dir=${SAMPLE_DIR} -echo "2. Symbolizing samples to perf reports. Hold on..." - -python ${INTERPRETER_DIR}symbolizer.py --in=${SAMPLE_DIR} --out=${REPORT_DIR} --cache=${SYMBOL_CACHE} -echo "3. Loading reports into RecordIO format..." -# Will need to make append_dir more clever / incremental -${PROFILE_UTIL_BINARY} --record=${RECORD_FILE} --append_dir=${REPORT_DIR} -echo "Done." -echo "4. Converting records to columnio." -${MR_CONVERT_BINARY} --mapreduce_input_map=recordio:${RECORD_FILE} --mapreduce_output_map=${COLUMN_FILE}@1 --columnio_mroutput_message_type=CwpProfileDbEntry --columnio_mroutput_protofiles=${PROTO_LOC} -echo "5. Uploading columnio to colossus." -fileutil cp -f /tmp/profiles.cio-* ${CNS_LOC} -echo "6. Let's try some dremel queries..." -echo " dremel> define table t /cns/ag-d/home/${USER}/profiledb/profiles.cio-*" -echo " Like, say, dremel> select sum(frames.count) as count, left(frames.function_name, 80) as name from t group by name order by count desc limit 25;" - diff --git a/deprecated/cwp/interpreter/app_engine_pull.py b/deprecated/cwp/interpreter/app_engine_pull.py deleted file mode 100644 index d092e2a3..00000000 --- a/deprecated/cwp/interpreter/app_engine_pull.py +++ /dev/null @@ -1,253 +0,0 @@ -# Copyright 2012 Google Inc. All Rights Reserved. -# Author: mrdmnd@ (Matt Redmond) -"""A client to pull data from Bartlett. - -Inspired by //depot/google3/experimental/mobile_gwp/database/app_engine_pull.py - -The server houses perf.data.gz, board, chrome version for each upload. -This script first authenticates with a proper @google.com account, then -downloads a sample (if it's not already cached) and unzips perf.data - - Authenticate(): Gets login info and returns an auth token - DownloadSamples(): Download and unzip samples. - _GetServePage(): Pulls /serve page from the app engine server - _DownloadSampleFromServer(): Downloads a local compressed copy of a sample - _UncompressSample(): Decompresses a sample, deleting the compressed version. -""" -import cookielib -import getpass -import gzip -import optparse -import os -import urllib -import urllib2 - -SERVER_NAME = 'http://chromeoswideprofiling.appspot.com' -APP_NAME = 'chromeoswideprofiling' -DELIMITER = '~' - - -def Authenticate(server_name): - """Gets credentials from user and attempts to retrieve auth token. - TODO: Accept OAuth2 instead of password. - Args: - server_name: (string) URL that the app engine code is living on. - Returns: - authtoken: (string) The authorization token that can be used - to grab other pages. - """ - - if server_name.endswith('/'): - server_name = server_name.rstrip('/') - # Grab username and password from user through stdin. - username = raw_input('Email (must be @google.com account): ') - password = getpass.getpass('Password: ') - # Use a cookie to authenticate with GAE. - cookiejar = cookielib.LWPCookieJar() - opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar)) - urllib2.install_opener(opener) - # Get an AuthToken from Google accounts service. - auth_uri = 'https://www.google.com/accounts/ClientLogin' - authreq_data = urllib.urlencode({'Email': username, - 'Passwd': password, - 'service': 'ah', - 'source': APP_NAME, - 'accountType': 'HOSTED_OR_GOOGLE'}) - auth_req = urllib2.Request(auth_uri, data=authreq_data) - try: - auth_resp = urllib2.urlopen(auth_req) - except urllib2.URLError: - print 'Error logging in to Google accounts service.' - return None - body = auth_resp.read() - # Auth response contains several fields. - # We care about the part after Auth= - auth_resp_dict = dict(x.split('=') for x in body.split('\n') if x) - authtoken = auth_resp_dict['Auth'] - return authtoken - - -def DownloadSamples(server_name, authtoken, output_dir, start, stop): - """Download every sample and write unzipped version - to output directory. - Args: - server_name: (string) URL that the app engine code is living on. - authtoken: (string) Authorization token. - output_dir (string) Filepath to write output to. - start: (int) Index to start downloading from, starting at top. - stop: (int) Index to stop downloading, non-inclusive. -1 for end. - Returns: - None - """ - - if server_name.endswith('/'): - server_name = server_name.rstrip('/') - - serve_page_string = _GetServePage(server_name, authtoken) - if serve_page_string is None: - print 'Error getting /serve page.' - return - - sample_list = serve_page_string.split('</br>') - print 'Will download:' - sample_list_subset = sample_list[start:stop] - for sample in sample_list_subset: - print sample - for sample in sample_list_subset: - assert sample, 'Sample should be valid.' - sample_info = [s.strip() for s in sample.split(DELIMITER)] - key = sample_info[0] - time = sample_info[1] - time = time.replace(' ', '_') # No space between date and time. - # sample_md5 = sample_info[2] - board = sample_info[3] - version = sample_info[4] - - # Put a compressed copy of the samples in output directory. - _DownloadSampleFromServer(server_name, authtoken, key, time, board, version, - output_dir) - _UncompressSample(key, time, board, version, output_dir) - - -def _BuildFilenameFromParams(key, time, board, version): - """Return the filename for our sample. - Args: - key: (string) Key indexing our sample in the datastore. - time: (string) Date that the sample was uploaded. - board: (string) Board that the sample was taken on. - version: (string) Version string from /etc/lsb-release - Returns: - filename (string) - """ - filename = DELIMITER.join([key, time, board, version]) - return filename - - -def _DownloadSampleFromServer(server_name, authtoken, key, time, board, version, - output_dir): - """Downloads sample_$(samplekey).gz to current dir. - Args: - server_name: (string) URL that the app engine code is living on. - authtoken: (string) Authorization token. - key: (string) Key indexing our sample in the datastore - time: (string) Date that the sample was uploaded. - board: (string) Board that the sample was taken on. - version: (string) Version string from /etc/lsb-release - output_dir: (string) Filepath to write to output to. - Returns: - None - """ - filename = _BuildFilenameFromParams(key, time, board, version) - compressed_filename = filename + '.gz' - - if os.path.exists(os.path.join(output_dir, filename)): - print 'Already downloaded %s, skipping.' % filename - return - - serv_uri = server_name + '/serve/' + key - serv_args = {'continue': serv_uri, 'auth': authtoken} - full_serv_uri = server_name + '/_ah/login?%s' % urllib.urlencode(serv_args) - serv_req = urllib2.Request(full_serv_uri) - serv_resp = urllib2.urlopen(serv_req) - f = open(os.path.join(output_dir, compressed_filename), 'w+') - f.write(serv_resp.read()) - f.close() - - -def _UncompressSample(key, time, board, version, output_dir): - """Uncompresses a given sample.gz file and deletes the compressed version. - Args: - key: (string) Sample key to uncompress. - time: (string) Date that the sample was uploaded. - board: (string) Board that the sample was taken on. - version: (string) Version string from /etc/lsb-release - output_dir: (string) Filepath to find sample key in. - Returns: - None - """ - filename = _BuildFilenameFromParams(key, time, board, version) - compressed_filename = filename + '.gz' - - if os.path.exists(os.path.join(output_dir, filename)): - print 'Already decompressed %s, skipping.' % filename - return - - out_file = open(os.path.join(output_dir, filename), 'wb') - in_file = gzip.open(os.path.join(output_dir, compressed_filename), 'rb') - out_file.write(in_file.read()) - in_file.close() - out_file.close() - os.remove(os.path.join(output_dir, compressed_filename)) - - -def _DeleteSampleFromServer(server_name, authtoken, key): - """Opens the /delete page with the specified key - to delete the sample off the datastore. - Args: - server_name: (string) URL that the app engine code is living on. - authtoken: (string) Authorization token. - key: (string) Key to delete. - Returns: - None - """ - - serv_uri = server_name + '/del/' + key - serv_args = {'continue': serv_uri, 'auth': authtoken} - full_serv_uri = server_name + '/_ah/login?%s' % urllib.urlencode(serv_args) - serv_req = urllib2.Request(full_serv_uri) - urllib2.urlopen(serv_req) - - -def _GetServePage(server_name, authtoken): - """Opens the /serve page and lists all keys. - Args: - server_name: (string) URL the app engine code is living on. - authtoken: (string) Authorization token. - Returns: - The text of the /serve page (including HTML tags) - """ - - serv_uri = server_name + '/serve' - serv_args = {'continue': serv_uri, 'auth': authtoken} - full_serv_uri = server_name + '/_ah/login?%s' % urllib.urlencode(serv_args) - serv_req = urllib2.Request(full_serv_uri) - serv_resp = urllib2.urlopen(serv_req) - return serv_resp.read() - - -def main(): - parser = optparse.OptionParser() - parser.add_option('--output_dir', - dest='output_dir', - action='store', - help='Path to output perf data files.') - parser.add_option('--start', - dest='start_ind', - action='store', - default=0, - help='Start index.') - parser.add_option('--stop', - dest='stop_ind', - action='store', - default=-1, - help='Stop index.') - options = parser.parse_args()[0] - if not options.output_dir: - print 'Must specify --output_dir.' - return 1 - if not os.path.exists(options.output_dir): - print 'Specified output_dir does not exist.' - return 1 - - authtoken = Authenticate(SERVER_NAME) - if not authtoken: - print 'Could not obtain authtoken, exiting.' - return 1 - DownloadSamples(SERVER_NAME, authtoken, options.output_dir, options.start_ind, - options.stop_ind) - print 'Downloaded samples.' - return 0 - - -if __name__ == '__main__': - exit(main()) diff --git a/deprecated/cwp/interpreter/symbolizer.py b/deprecated/cwp/interpreter/symbolizer.py deleted file mode 100644 index 4ece480d..00000000 --- a/deprecated/cwp/interpreter/symbolizer.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2012 Google Inc. All Rights Reserved. -"""A script that symbolizes perf.data files.""" -import optparse -import os -import shutil -from subprocess import call -from subprocess import PIPE -from subprocess import Popen -from cros_utils import misc - -GSUTIL_CMD = 'gsutil cp gs://chromeos-image-archive/%s-release/%s/debug.tgz %s' -TAR_CMD = 'tar -zxvf %s -C %s' -PERF_BINARY = '/google/data/ro/projects/perf/perf' -VMLINUX_FLAG = ' --vmlinux=/usr/lib/debug/boot/vmlinux' -PERF_CMD = PERF_BINARY + ' report -i %s -n --symfs=%s' + VMLINUX_FLAG - - -def main(): - parser = optparse.OptionParser() - parser.add_option('--in', dest='in_dir') - parser.add_option('--out', dest='out_dir') - parser.add_option('--cache', dest='cache') - (opts, _) = parser.parse_args() - if not _ValidateOpts(opts): - return 1 - else: - for filename in os.listdir(opts.in_dir): - try: - _DownloadSymbols(filename, opts.cache) - _PerfReport(filename, opts.in_dir, opts.out_dir, opts.cache) - except: - print 'Exception caught. Continuing...' - return 0 - - -def _ValidateOpts(opts): - """Ensures all directories exist, before attempting to populate.""" - if not os.path.exists(opts.in_dir): - print "Input directory doesn't exist." - return False - if not os.path.exists(opts.out_dir): - print "Output directory doesn't exist. Creating it..." - os.makedirs(opts.out_dir) - if not os.path.exists(opts.cache): - print "Cache directory doesn't exist." - return False - return True - - -def _ParseFilename(filename, canonical=False): - """Returns a tuple (key, time, board, lsb_version). - If canonical is True, instead returns (database_key, board, canonical_vers) - canonical_vers includes the revision string. - """ - key, time, board, vers = filename.split('~') - if canonical: - vers = misc.GetChromeOSVersionFromLSBVersion(vers) - return (key, time, board, vers) - - -def _FormReleaseDir(board, version): - return '%s-release~%s' % (board, version) - - -def _DownloadSymbols(filename, cache): - """ Incrementally downloads appropriate symbols. - We store the downloads in cache, with each set of symbols in a TLD - named like cache/$board-release~$canonical_vers/usr/lib/debug - """ - _, _, board, vers = _ParseFilename(filename, canonical=True) - tmp_suffix = '.tmp' - - tarball_subdir = _FormReleaseDir(board, vers) - tarball_dir = os.path.join(cache, tarball_subdir) - tarball_path = os.path.join(tarball_dir, 'debug.tgz') - - symbol_subdir = os.path.join('usr', 'lib') - symbol_dir = os.path.join(tarball_dir, symbol_subdir) - - if os.path.isdir(symbol_dir): - print 'Symbol directory %s exists, skipping download.' % symbol_dir - return - else: - # First download using gsutil. - if not os.path.isfile(tarball_path): - download_cmd = GSUTIL_CMD % (board, vers, tarball_path + tmp_suffix) - print 'Downloading symbols for %s' % filename - print download_cmd - ret = call(download_cmd.split()) - if ret != 0: - print 'gsutil returned non-zero error code: %s.' % ret - # Clean up the empty directory structures. - os.remove(tarball_path + tmp_suffix) - raise IOError - - shutil.move(tarball_path + tmp_suffix, tarball_path) - - # Next, untar the tarball. - os.makedirs(symbol_dir + tmp_suffix) - extract_cmd = TAR_CMD % (tarball_path, symbol_dir + tmp_suffix) - print 'Extracting symbols for %s' % filename - print extract_cmd - ret = call(extract_cmd.split()) - if ret != 0: - print 'tar returned non-zero code: %s.' % ret - raise IOError - shutil.move(symbol_dir + tmp_suffix, symbol_dir) - os.remove(tarball_path) - - -def _PerfReport(filename, in_dir, out_dir, cache): - """ Call perf report on the file, storing output to the output dir. - The output is currently stored as $out_dir/$filename - """ - _, _, board, vers = _ParseFilename(filename, canonical=True) - symbol_cache_tld = _FormReleaseDir(board, vers) - input_file = os.path.join(in_dir, filename) - symfs = os.path.join(cache, symbol_cache_tld) - report_cmd = PERF_CMD % (input_file, symfs) - print 'Reporting.' - print report_cmd - report_proc = Popen(report_cmd.split(), stdout=PIPE) - outfile = open(os.path.join(out_dir, filename), 'w') - outfile.write(report_proc.stdout.read()) - outfile.close() - - -if __name__ == '__main__': - exit(main()) diff --git a/deprecated/cwp/performance/experiment_gen.py b/deprecated/cwp/performance/experiment_gen.py deleted file mode 100644 index a12da2c5..00000000 --- a/deprecated/cwp/performance/experiment_gen.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright 2012 Google Inc. All Rights Reserved. -"""This script generates a crosperf overhead-testing experiment file for MoreJS. - -Use: experiment_gen.py --crosperf=/home/mrdmnd/depot2/crosperf --chromeos_root= -/home/mrdmnd/chromiumos --remote-host=chromeos-zgb3.mtv --board=x86-zgb --event= -cycles -F 10 -F 20 -c 10582 -c 10785211 --perf_options="-g" -""" - -import optparse -import subprocess -import sys -import time - -HEADER = """ -board: %s -remote: %s -benchmark: baseline { - iterations: %s - autotest_name: desktopui_PyAutoPerfTests - autotest_args: --args='--iterations=%s perf.PageCyclerTest.testMoreJSFile' -}""" - -EXPERIMENT = """ -benchmark: %s { - iterations: %s - autotest_name: desktopui_PyAutoPerfTests - autotest_args: --args='--iterations=%s perf.PageCyclerTest.testMoreJSFile' --profiler=custom_perf --profiler_args='perf_options="record -a %s %s -e %s"' \n}""" # pylint: disable-msg=C6310 - -DEFAULT_IMAGE = """ -default { - chromeos_image: %s/src/build/images/%s/latest/chromiumos_test_image.bin -}""" - - -def main(): - parser = optparse.OptionParser() - parser.add_option('--crosperf', - dest='crosperf_root', - action='store', - default='/home/mrdmnd/depot2/crosperf', - help='Crosperf root directory.') - parser.add_option('--chromeos_root', - dest='chromeos_root', - action='store', - default='/home/mrdmnd/chromiumos', - help='ChromiumOS root directory.') - parser.add_option('--remote', - dest='remote', - action='store', - help='Host to run test on. Required.') - parser.add_option('--board', - dest='board', - action='store', - help='Board architecture to run on. Required.') - parser.add_option('--event', - dest='event', - action='store', - help='Event to profile. Required.') - parser.add_option('-F', - dest='sampling_frequencies', - action='append', - help='A target frequency to sample at.') - parser.add_option('-c', - dest='sampling_periods', - action='append', - help='A target period to sample at. Event specific.') - parser.add_option('--benchmark-iterations', - dest='benchmark_iterations', - action='store', - default=4, - help='Number of benchmark iters') - parser.add_option('--test-iterations', - dest='test_iterations', - action='store', - default=10, - help='Number of test iters') - parser.add_option('-p', - dest='print_only', - action='store_true', - help='If enabled, will print experiment file and exit.') - parser.add_option('--perf_options', - dest='perf_options', - action='store', - help='Arbitrary flags to perf. Surround with dblquotes.') - options = parser.parse_args()[0] - if options.remote is None: - print '%s requires a remote hostname.' % sys.argv[0] - return 1 - elif options.board is None: - print '%s requires a target board.' % sys.argv[0] - return 1 - elif options.event is None: - print '%s requires an event to profile.' % sys.argv[0] - return 1 - else: - crosperf_root = options.crosperf_root - chromeos_root = options.chromeos_root - remote = options.remote - board = options.board - event = options.event - bench_iters = options.benchmark_iterations - test_iters = options.test_iterations - perf_opts = options.perf_options - # Set up baseline test. - experiment_file = HEADER % (board, remote, bench_iters, test_iters) - # Set up experimental tests. - if options.sampling_frequencies: - for freq in options.sampling_frequencies: - test_string = str(freq) + 'Freq' - experiment_file += EXPERIMENT % (test_string, bench_iters, test_iters, - '-F %s' % freq, '' if perf_opts is None - else perf_opts, event) - if options.sampling_periods: - for period in options.sampling_periods: - test_string = str(period) + 'Period' - experiment_file += EXPERIMENT % ( - test_string, bench_iters, test_iters, '-c %s' % period, '' if - perf_opts is None else perf_opts, event) - # Point to the target image. - experiment_file += DEFAULT_IMAGE % (chromeos_root, board) - if options.print_only: - print experiment_file - else: - current_time = int(round(time.time() * 1000)) - file_name = 'perf_overhead_%s' % str(current_time) - with open(file_name, 'w') as f: - f.write(experiment_file) - try: - process = subprocess.Popen(['%s/crosperf' % crosperf_root, file_name]) - process.communicate() - except OSError: - print 'Could not find crosperf, make sure --crosperf flag is set right.' - return 1 - return 0 - - -if __name__ == '__main__': - exit(main()) |