diff options
Diffstat (limited to 'catapult/common/py_utils')
5 files changed, 119 insertions, 36 deletions
diff --git a/catapult/common/py_utils/py_utils/chrome_binaries.json b/catapult/common/py_utils/py_utils/chrome_binaries.json index badb3642..03f0afd3 100644 --- a/catapult/common/py_utils/py_utils/chrome_binaries.json +++ b/catapult/common/py_utils/py_utils/chrome_binaries.json @@ -12,16 +12,16 @@ "version_in_cs": "85.0.4169.0" }, "win_AMD64": { - "cloud_storage_hash": "c0b02f47afb6bacdd8d12eda6f03488583404b60", + "cloud_storage_hash": "121271e6e917886bd2e5b539dd643e74a3aae395", "download_path": "bin\\reference_build\\chrome-win64-clang.zip", "path_within_archive": "chrome-win64-clang\\chrome.exe", - "version_in_cs": "85.0.4169.0" + "version_in_cs": "86.0.4240.111" }, "win_x86": { - "cloud_storage_hash": "5879958eda4edc5b0eaffeda4c7369a07937af73", + "cloud_storage_hash": "83e9d11b28d01c10bbcccb7b882288b192b5e756", "download_path": "bin\\reference_build\\chrome-win32-clang.zip", "path_within_archive": "chrome-win32-clang\\chrome.exe", - "version_in_cs": "85.0.4169.0" + "version_in_cs": "86.0.4240.111" } } }, @@ -169,4 +169,4 @@ } } } -}
\ No newline at end of file +} diff --git a/catapult/common/py_utils/py_utils/cloud_storage.py b/catapult/common/py_utils/py_utils/cloud_storage.py index a359a065..919add69 100644 --- a/catapult/common/py_utils/py_utils/cloud_storage.py +++ b/catapult/common/py_utils/py_utils/cloud_storage.py @@ -192,10 +192,33 @@ def IsNetworkIOEnabled(): return disable_cloud_storage_env_val != '1' -def List(bucket): - query = 'gs://%s/' % bucket - stdout = _RunCommand(['ls', query]) - return [url[len(query):] for url in stdout.splitlines()] +def List(bucket, prefix=None): + """Returns all paths matching the given prefix in bucket. + + Returned paths are relative to the bucket root. + If path is given, 'gsutil ls gs://<bucket>/<path>' will be executed, otherwise + 'gsutil ls gs://<bucket>' will be executed. + + For more details, see: + https://cloud.google.com/storage/docs/gsutil/commands/ls#directory-by-directory,-flat,-and-recursive-listings + + Args: + bucket: Name of cloud storage bucket to look at. + prefix: Path within the bucket to filter to. + + Returns: + A list of files. All returned path are relative to the bucket root + directory. For example, List('my-bucket', path='foo/') will returns results + of the form ['/foo/123', '/foo/124', ...], as opposed to ['123', '124', + ...]. + """ + bucket_prefix = 'gs://%s' % bucket + if prefix is None: + full_path = bucket_prefix + else: + full_path = '%s/%s' % (bucket_prefix, prefix) + stdout = _RunCommand(['ls', full_path]) + return [url[len(bucket_prefix):] for url in stdout.splitlines()] def ListDirs(bucket, path=''): diff --git a/catapult/common/py_utils/py_utils/cloud_storage_unittest.py b/catapult/common/py_utils/py_utils/cloud_storage_unittest.py index 2e663a76..afd03945 100644 --- a/catapult/common/py_utils/py_utils/cloud_storage_unittest.py +++ b/catapult/common/py_utils/py_utils/cloud_storage_unittest.py @@ -171,6 +171,24 @@ class CloudStorageFakeFsUnitTest(BaseFakeFsUnitTest): cloud_storage._RunCommand = orig_run_command @mock.patch('py_utils.cloud_storage._RunCommand') + def testListNoPrefix(self, mock_run_command): + mock_run_command.return_value = '\n'.join(['gs://bucket/foo-file.txt', + 'gs://bucket/foo1/', + 'gs://bucket/foo2/']) + + self.assertEqual(cloud_storage.List('bucket'), + ['/foo-file.txt', '/foo1/', '/foo2/']) + + @mock.patch('py_utils.cloud_storage._RunCommand') + def testListWithPrefix(self, mock_run_command): + mock_run_command.return_value = '\n'.join(['gs://bucket/foo/foo-file.txt', + 'gs://bucket/foo/foo1/', + 'gs://bucket/foo/foo2/']) + + self.assertEqual(cloud_storage.List('bucket', 'foo'), + ['/foo/foo-file.txt', '/foo/foo1/', '/foo/foo2/']) + + @mock.patch('py_utils.cloud_storage._RunCommand') def testListDirs(self, mock_run_command): mock_run_command.return_value = '\n'.join(['gs://bucket/foo-file.txt', '', diff --git a/catapult/common/py_utils/py_utils/ts_proxy_server.py b/catapult/common/py_utils/py_utils/ts_proxy_server.py index b71d143d..0e168a53 100644 --- a/catapult/common/py_utils/py_utils/ts_proxy_server.py +++ b/catapult/common/py_utils/py_utils/ts_proxy_server.py @@ -196,11 +196,13 @@ class TsProxyServer(object): try: py_utils.WaitFor(lambda: self._proc.poll() is not None, 10) except py_utils.TimeoutException: - try: - # Use a SIGNINT so that it can do graceful cleanup - self._proc.send_signal(signal.SIGINT) - except ValueError: - logging.warning('Unable to stop ts_proxy_server gracefully.\n') + # signal.SIGINT is not supported on Windows. + if not sys.platform.startswith('win'): + try: + # Use a SIGNINT so that it can do graceful cleanup + self._proc.send_signal(signal.SIGINT) + except ValueError: + logging.warning('Unable to stop ts_proxy_server gracefully.\n') self._proc.terminate() _, err = self._proc.communicate() diff --git a/catapult/common/py_utils/py_utils/webpagereplay_go_server.py b/catapult/common/py_utils/py_utils/webpagereplay_go_server.py index 950e8adc..95e44954 100644 --- a/catapult/common/py_utils/py_utils/webpagereplay_go_server.py +++ b/catapult/common/py_utils/py_utils/webpagereplay_go_server.py @@ -17,7 +17,6 @@ import py_utils from py_utils import atexit_with_log from py_utils import binary_manager - _WPR_DIR = os.path.abspath(os.path.join( py_utils.GetCatapultDir(), 'web_page_replay_go')) @@ -31,6 +30,8 @@ CHROME_BINARY_CONFIG = os.path.join( RECORD = '--record' INJECT_SCRIPTS = '--inject_scripts=' +USE_LOCAL_WPR = '--use-local-wpr' +DISABLE_FUZZY_URL_MATCHING = '--disable-fuzzy-url-matching' class ReplayError(Exception): """Catch-all exception for the module.""" @@ -102,16 +103,10 @@ class ReplayServer(object): # subprocess. self._temp_log_file_path = None - # Assign the downloader func and binary_manager - downloader = None - if binary_downloader: - downloader = binary_downloader - else: - configs = [CHROME_BINARY_CONFIG, TELEMETRY_PROJECT_CONFIG] - downloader = binary_manager.BinaryManager(configs).FetchPath - + self._downloader = binary_downloader + self._replay_options = replay_options self._cmd_line = self._GetCommandLine( - self._GetGoBinaryPath(downloader=downloader), http_port, https_port, + self._GetGoBinaryPath(replay_options), http_port, https_port, replay_options, archive_path) if RECORD in replay_options or 'record' in replay_options: @@ -122,19 +117,57 @@ class ReplayServer(object): self.replay_process = None - @classmethod - def _GetGoBinaryPath(cls, downloader): - if not cls._go_binary_path: - cls._go_binary_path = downloader( + def _GetDownloader(self): + """Gets the downloader used to download wpr_go binary from GCS.""" + if ReplayServer._go_binary_path: + # If the _go_binary_path was already set, then no need to use downloader + # to download via binary_manager. + self._downloader = None + elif not self._downloader: + configs = [CHROME_BINARY_CONFIG, TELEMETRY_PROJECT_CONFIG] + self._downloader = binary_manager.BinaryManager(configs).FetchPath + return self._downloader + + def _GetGoBinaryPath(self, replay_options): + """Gets the _go_binary_path if it already set, or downloads it.""" + if USE_LOCAL_WPR in replay_options: + # Build WPR + go_folder = os.path.join(_WPR_DIR, 'src') + cur_cwd = os.getcwd() + os.chdir(go_folder) + try: + print subprocess.check_output(['go', 'build', os.path.join(go_folder, 'wpr.go')]) + except subprocess.CalledProcessError: + exit(1) + os.chdir(cur_cwd) + + return os.path.join(go_folder, 'wpr') + + if not ReplayServer._go_binary_path: + downloader = self._GetDownloader() + if not downloader: + raise RuntimeError('downloader should not be None ' + 'while _go_binary_path is None') + ReplayServer._go_binary_path = downloader( 'wpr_go', py_utils.GetHostOsName(), py_utils.GetHostArchName()) - return cls._go_binary_path + return ReplayServer._go_binary_path @classmethod def SetGoBinaryPath(cls, go_binary_path): """Overrides the _go_binary_path. This allows the server to use WPRGO files retrieved from somewhere - other than GCS, such as CIPD.""" + other than GCS via binary_manager, such as test isolation. + + For chromium project to use WPR, it is encourage to use test isolation, + and therefore should call SetGoBinaryPath to set _go_binary_path. + + For Catapult/Telemetry project, the tradition is to download wpr_go + binary via binary_manager. So do not call SetGoBinaryPath. + """ + if not os.path.exists(go_binary_path): + raise ValueError('SetGoBinaryPath could not set {} as it does not exist' + .format(go_binary_path)) cls._go_binary_path = go_binary_path @property @@ -162,7 +195,8 @@ class ReplayServer(object): """ bad_options = [] for option in options: - if option not in [RECORD, INJECT_SCRIPTS]: + if option not in [RECORD, INJECT_SCRIPTS, + USE_LOCAL_WPR, DISABLE_FUZZY_URL_MATCHING]: bad_options.append(option) if len(bad_options) > 0: raise ValueError("Invalid replay options %s" % bad_options) @@ -172,6 +206,8 @@ class ReplayServer(object): cmd_line.append('record') else: cmd_line.append('replay') + if DISABLE_FUZZY_URL_MATCHING in options: + cmd_line.append('--disable_fuzzy_url_matching') key_file = os.path.join(_WPR_DIR, 'wpr_key.pem') cert_file = os.path.join(_WPR_DIR, 'wpr_cert.pem') inject_script = os.path.join(_WPR_DIR, 'deterministic.js') @@ -347,13 +383,17 @@ class ReplayServer(object): def _CleanUpTempLogFilePath(self, log_level): if not self._temp_log_file_path: return '' - if logging.getLogger('').isEnabledFor(log_level): + if logging.getLogger('').isEnabledFor(log_level) or USE_LOCAL_WPR in self._replay_options: with open(self._temp_log_file_path, 'r') as f: wpr_log_output = f.read() - logging.log(log_level, '\n'.join([ - '************************** WPR LOG *****************************', - wpr_log_output, - '************************** END OF WPR LOG **********************'])) + output = ('************************** WPR LOG *****************************\n' + + '\n'.join(wpr_log_output.split('\n')) + + '************************** END OF WPR LOG **********************') + if logging.getLogger('').isEnabledFor(log_level): + logging.log(log_level, output) + else: + print output + os.remove(self._temp_log_file_path) self._temp_log_file_path = None |