# Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """ Set of basic operations/utilities that are used by the build. """ from contextlib import contextmanager import ast import base64 import cStringIO import copy import errno import fnmatch import glob import math import multiprocessing import os import re import shutil import socket import stat import string # pylint: disable=W0402 import subprocess import sys import threading import time import traceback import urllib import zipfile import zlib try: import json # pylint: disable=F0401 except ImportError: import simplejson as json from common import env BUILD_DIR = os.path.realpath(os.path.join( os.path.dirname(__file__), os.pardir, os.pardir)) WIN_LINK_FUNC = None try: if sys.platform.startswith('win'): import ctypes # There's 4 possibilities on Windows for links: # 1. Symbolic file links; # 2. Symbolic directory links; # 3. Hardlinked files; # 4. Junctioned directories. # (Hardlinked directories don't really exist.) # # 7-Zip does not handle symbolic file links as we want (it puts the # content of the link, not what it refers to, and reports "CRC Error" on # extraction). It does work as expected for symbolic directory links. # Because the majority of the large files are in the root of the staging # directory, we do however need to handle file links, so we do this with # hardlinking. Junctioning requires a huge whack of code, so we take the # slightly odd tactic of using #2 and #3, but not #1 and #4. That is, # hardlinks for files, but symbolic links for directories. def _WIN_LINK_FUNC(src, dst): print 'linking %s -> %s' % (src, dst) if os.path.isdir(src): if not ctypes.windll.kernel32.CreateSymbolicLinkA( str(dst), str(os.path.abspath(src)), 1): raise ctypes.WinError() else: if not ctypes.windll.kernel32.CreateHardLinkA(str(dst), str(src), 0): raise ctypes.WinError() WIN_LINK_FUNC = _WIN_LINK_FUNC except ImportError: # If we don't have ctypes or aren't on Windows, leave WIN_LINK_FUNC as None. pass # Wrapper around git that enforces a timeout. GIT_BIN = os.path.join(BUILD_DIR, 'scripts', 'tools', 'git-with-timeout') # Wrapper around svn that enforces a timeout. SVN_BIN = os.path.join(BUILD_DIR, 'scripts', 'tools', 'svn-with-timeout') # The Google Storage metadata key for the full commit position GS_COMMIT_POSITION_KEY = 'Cr-Commit-Position' # The Google Storage metadata key for the commit position number GS_COMMIT_POSITION_NUMBER_KEY = 'Cr-Commit-Position-Number' # The Google Storage metadata key for the Git commit hash GS_GIT_COMMIT_KEY = 'Cr-Git-Commit' # Regular expression to identify a Git hash GIT_COMMIT_HASH_RE = re.compile(r'[a-fA-F0-9]{40}') # # Regular expression to parse a commit position COMMIT_POSITION_RE = re.compile(r'([^@]+)@{#(\d+)}') # Local errors. class MissingArgument(Exception): pass class PathNotFound(Exception): pass class ExternalError(Exception): pass class NoIdentifiedRevision(Exception): pass def IsWindows(): return sys.platform == 'cygwin' or sys.platform.startswith('win') def IsLinux(): return sys.platform.startswith('linux') def IsMac(): return sys.platform.startswith('darwin') # For chromeos we need to end up with a different platform name, but the # scripts use the values like sys.platform for both the build target and # and the running OS, so this gives us a back door that can be hit to # force different naming then the default for some of the chromeos build # steps. override_platform_name = None def OverridePlatformName(name): """Sets the override for PlatformName()""" global override_platform_name override_platform_name = name def PlatformName(): """Return a string to be used in paths for the platform.""" if override_platform_name: return override_platform_name if IsWindows(): return 'win32' if IsLinux(): return 'linux' if IsMac(): return 'mac' raise NotImplementedError('Unknown platform "%s".' % sys.platform) # Name of the file (inside the packaged build) containing revision number # of that build. Also used for determining the latest packaged build. FULL_BUILD_REVISION_FILENAME = 'FULL_BUILD_REVISION' def IsGitCommit(value): """Returns: If a value is a Git commit hash. This only works on full Git commit hashes. A value qualifies as a Git commit hash if it only contains hexadecimal numbers and is forty characters long. """ if value is None: return False return GIT_COMMIT_HASH_RE.match(str(value)) is not None # GetParentClass allows a class instance to find its parent class using Python's # inspect module. This allows a class instantiated from a module to access # their parent class's methods even after the containing module has been # re-imported and reloaded. # # Also see: # http://code.google.com/p/chromium/issues/detail?id=34089 # http://atlee.ca/blog/2008/11/21/python-reload-danger-here-be-dragons/ # def GetParentClass(obj, n=1): import inspect if inspect.isclass(obj): return inspect.getmro(obj)[n] else: return inspect.getmro(obj.__class__)[n] def MeanAndStandardDeviation(data): """Calculates mean and standard deviation for the values in the list. Args: data: list of numbers Returns: Mean and standard deviation for the numbers in the list. """ n = len(data) if n == 0: return 0.0, 0.0 mean = float(sum(data)) / n variance = sum([(element - mean)**2 for element in data]) / n return mean, math.sqrt(variance) def FilteredMeanAndStandardDeviation(data): """Calculates mean and standard deviation for the values in the list ignoring first occurence of max value (unless there was only one sample). Args: data: list of numbers Returns: Mean and standard deviation for the numbers in the list ignoring first occurence of max value. """ def _FilterMax(array): new_array = copy.copy(array) # making sure we are not creating side-effects if len(new_array) != 1: new_array.remove(max(new_array)) return new_array return MeanAndStandardDeviation(_FilterMax(data)) def HistogramPercentiles(histogram, percentiles): if not 'buckets' in histogram or not 'count' in histogram: return [] computed_percentiles = _ComputePercentiles(histogram['buckets'], histogram['count'], percentiles) output = [] for p in computed_percentiles: output.append({'percentile': p, 'value': computed_percentiles[p]}) return output def GeomMeanAndStdDevFromHistogram(histogram): if not 'buckets' in histogram: return 0.0, 0.0 count = 0 sum_of_logs = 0 for bucket in histogram['buckets']: if 'high' in bucket: bucket['mean'] = (bucket['low'] + bucket['high']) / 2.0 else: bucket['mean'] = bucket['low'] if bucket['mean'] > 0: sum_of_logs += math.log(bucket['mean']) * bucket['count'] count += bucket['count'] if count == 0: return 0.0, 0.0 sum_of_squares = 0 geom_mean = math.exp(sum_of_logs / count) for bucket in histogram['buckets']: if bucket['mean'] > 0: sum_of_squares += (bucket['mean'] - geom_mean) ** 2 * bucket['count'] return geom_mean, math.sqrt(sum_of_squares / count) def _LinearInterpolate(x0, target, x1, y0, y1): """Perform linear interpolation to estimate an intermediate value. We assume for some F, F(x0) == y0, and F(x1) == z1. We return an estimate for what F(target) should be, using linear interpolation. Args: x0: (Float) A location at which some function F() is known. target: (Float) A location at which we need to estimate F(). x1: (Float) A second location at which F() is known. y0: (Float) The value of F(x0). y1: (Float) The value of F(x1). Returns: (Float) The estimated value of F(target). """ if x0 == x1: return (y0 + y1) / 2 return (y1 - y0) * (target - x0) / (x1 - x0) + y0 def _BucketInterpolate(last_percentage, target, next_percentage, bucket_min, bucket_max): """Estimate a minimum which should have the target % of samples below it. We do linear interpolation only if last_percentage and next_percentage are adjacent, and hence we are in a linear section of a histogram. Once they spread further apart we generally get exponentially broader buckets, and we need to interpolate in the log domain (and exponentiate our result). Args: last_percentage: (Float) This is the percentage of samples below bucket_min. target: (Float) A percentage for which we need an estimated bucket. next_percentage: (Float) This is the percentage of samples below bucket_max. bucket_min: (Float) This is the lower value for samples in a bucket. bucket_max: (Float) This exceeds the upper value for samples. Returns: (Float) An estimate of what bucket cutoff would have probably had the target percentage. """ log_domain = False if bucket_min + 1.5 < bucket_max and bucket_min > 0: log_domain = True bucket_min = math.log(bucket_min) bucket_max = math.log(bucket_max) result = _LinearInterpolate( last_percentage, target, next_percentage, bucket_min, bucket_max) if log_domain: result = math.exp(result) return result def _ComputePercentiles(buckets, total, percentiles): """Compute percentiles for the given histogram. Returns estimates for the bucket cutoffs that would probably have the taret percentiles. Args: buckets: (List) A list of buckets representing the histogram to analyze. total: (Float) The total number of samples in the histogram. percentiles: (Tuple) The percentiles we are interested in. Returns: (Dictionary) Map from percentiles to bucket cutoffs. """ if not percentiles: return {} current_count = 0 current_percentage = 0 next_percentile_index = 0 result = {} for bucket in buckets: if bucket['count'] > 0: current_count += bucket['count'] old_percentage = current_percentage current_percentage = float(current_count) / total # Check whether we passed one of the percentiles we're interested in. while (next_percentile_index < len(percentiles) and current_percentage > percentiles[next_percentile_index]): if not 'high' in bucket: result[percentiles[next_percentile_index]] = bucket['low'] else: result[percentiles[next_percentile_index]] = float(_BucketInterpolate( old_percentage, percentiles[next_percentile_index], current_percentage, bucket['low'], bucket['high'])) next_percentile_index += 1 return result class InitializePartiallyWithArguments: # pylint: disable=old-style-class """Function currying implementation. Works for constructors too. Primary use is to be able to construct a class with some constructor arguments beings set ahead of actual initialization. Copy of an ASPN cookbook (#52549). """ def __init__(self, clazz, *args, **kwargs): self.clazz = clazz self.pending = args[:] self.kwargs = kwargs.copy() def __call__(self, *args, **kwargs): if kwargs and self.kwargs: kw = self.kwargs.copy() kw.update(kwargs) else: kw = kwargs or self.kwargs return self.clazz(*(self.pending + args), **kw) def Prepend(filepath, text): """ Prepends text to the file. Creates the file if it does not exist. """ file_data = text if os.path.exists(filepath): file_data += open(filepath).read() f = open(filepath, 'w') f.write(file_data) f.close() def MakeWorldReadable(path): """Change the permissions of the given path to make it world-readable. This is often needed for archived files, so they can be served by web servers or accessed by unprivileged network users.""" # No need to do anything special on Windows. if IsWindows(): return perms = stat.S_IMODE(os.stat(path)[stat.ST_MODE]) if os.path.isdir(path): # Directories need read and exec. os.chmod(path, perms | 0555) else: os.chmod(path, perms | 0444) def MakeParentDirectoriesWorldReadable(path): """Changes the permissions of the given path and its parent directories to make them world-readable. Stops on first directory which is world-readable. This is often needed for archive staging directories, so that they can be served by web servers or accessed by unprivileged network users.""" # No need to do anything special on Windows. if IsWindows(): return while path != os.path.dirname(path): current_permissions = stat.S_IMODE(os.stat(path)[stat.ST_MODE]) if current_permissions & 0555 == 0555: break os.chmod(path, current_permissions | 0555) path = os.path.dirname(path) def MaybeMakeDirectory(*path): """Creates an entire path, if it doesn't already exist.""" file_path = os.path.join(*path) try: os.makedirs(file_path) except OSError, e: if e.errno != errno.EEXIST: raise def RemovePath(*path): """Removes the file or directory at 'path', if it exists.""" file_path = os.path.join(*path) if os.path.exists(file_path): if os.path.isdir(file_path): RemoveDirectory(file_path) else: RemoveFile(file_path) def RemoveFile(*path): """Removes the file located at 'path', if it exists.""" file_path = os.path.join(*path) try: os.remove(file_path) except OSError, e: if e.errno != errno.ENOENT: raise def MoveFile(path, new_path): """Moves the file located at 'path' to 'new_path', if it exists.""" try: RemoveFile(new_path) os.rename(path, new_path) except OSError, e: if e.errno != errno.ENOENT: raise def LocateFiles(pattern, root=os.curdir): """Yeilds files matching pattern found in root and its subdirectories. An exception is thrown if root doesn't exist.""" for path, _, files in os.walk(os.path.abspath(root)): for filename in fnmatch.filter(files, pattern): yield os.path.join(path, filename) def RemoveFilesWildcards(file_wildcard, root=os.curdir): """Removes files matching 'file_wildcard' in root and its subdirectories, if any exists. An exception is thrown if root doesn't exist.""" for item in LocateFiles(file_wildcard, root): try: os.remove(item) except OSError, e: if e.errno != errno.ENOENT: raise def RemoveGlobbedPaths(path_wildcard, root=os.curdir): """Removes all paths matching 'path_wildcard' beneath root. Returns the list of paths removed. An exception is thrown if root doesn't exist.""" if not os.path.exists(root): raise OSError(2, 'No such file or directory', root) full_path_wildcard = os.path.join(path_wildcard, root) paths = glob.glob(full_path_wildcard) for path in paths: # When glob returns directories they end in "/." if path.endswith(os.sep + '.'): path = path[:-2] RemovePath(path) return paths def RemoveDirectory(*path): """Recursively removes a directory, even if it's marked read-only. Remove the directory located at *path, if it exists. shutil.rmtree() doesn't work on Windows if any of the files or directories are read-only, which svn repositories and some .svn files are. We need to be able to force the files to be writable (i.e., deletable) as we traverse the tree. Even with all this, Windows still sometimes fails to delete a file, citing a permission error (maybe something to do with antivirus scans or disk indexing). The best suggestion any of the user forums had was to wait a bit and try again, so we do that too. It's hand-waving, but sometimes it works. :/ """ file_path = os.path.join(*path) if not os.path.exists(file_path): return if sys.platform == 'win32': # Give up and use cmd.exe's rd command. file_path = os.path.normcase(file_path) for _ in xrange(3): print 'RemoveDirectory running %s' % (' '.join( ['cmd.exe', '/c', 'rd', '/q', '/s', file_path])) if not subprocess.call(['cmd.exe', '/c', 'rd', '/q', '/s', file_path]): break print ' Failed' time.sleep(3) return def RemoveWithRetry_non_win(rmfunc, path): if os.path.islink(path): return os.remove(path) else: return rmfunc(path) remove_with_retry = RemoveWithRetry_non_win def RmTreeOnError(function, path, excinfo): r"""This works around a problem whereby python 2.x on Windows has no ability to check for symbolic links. os.path.islink always returns False. But shutil.rmtree will fail if invoked on a symbolic link whose target was deleted before the link. E.g., reproduce like this: > mkdir test > mkdir test\1 > mklink /D test\current test\1 > python -c "import chromium_utils; chromium_utils.RemoveDirectory('test')" To avoid this issue, we pass this error-handling function to rmtree. If we see the exact sort of failure, we ignore it. All other failures we re- raise. """ exception_type = excinfo[0] exception_value = excinfo[1] # If shutil.rmtree encounters a symbolic link on Windows, os.listdir will # fail with a WindowsError exception with an ENOENT errno (i.e., file not # found). We'll ignore that error. Note that WindowsError is not defined # for non-Windows platforms, so we use OSError (of which it is a subclass) # to avoid lint complaints about an undefined global on non-Windows # platforms. if (function is os.listdir) and issubclass(exception_type, OSError): if exception_value.errno == errno.ENOENT: # File does not exist, and we're trying to delete, so we can ignore the # failure. print 'WARNING: Failed to list %s during rmtree. Ignoring.\n' % path else: raise else: raise for root, dirs, files in os.walk(file_path, topdown=False): # For POSIX: making the directory writable guarantees removability. # Windows will ignore the non-read-only bits in the chmod value. os.chmod(root, 0770) for name in files: remove_with_retry(os.remove, os.path.join(root, name)) for name in dirs: remove_with_retry(lambda p: shutil.rmtree(p, onerror=RmTreeOnError), os.path.join(root, name)) remove_with_retry(os.rmdir, file_path) def CopyFileToDir(src_path, dest_dir, dest_fn=None, link_ok=False): """Copies the file found at src_path to the dest_dir directory, with metadata. If dest_fn is specified, the src_path is copied to that name in dest_dir, otherwise it is copied to a file of the same name. Raises PathNotFound if either the file or the directory is not found. """ # Verify the file and directory separately so we can tell them apart and # raise PathNotFound rather than shutil.copyfile's IOError. if not os.path.isfile(src_path): raise PathNotFound('Unable to find file %s' % src_path) if not os.path.isdir(dest_dir): raise PathNotFound('Unable to find dir %s' % dest_dir) src_file = os.path.basename(src_path) if dest_fn: # If we have ctypes and the caller doesn't mind links, use that to # try to make the copy faster on Windows. http://crbug.com/418702. if link_ok and WIN_LINK_FUNC: WIN_LINK_FUNC(src_path, os.path.join(dest_dir, dest_fn)) else: shutil.copy2(src_path, os.path.join(dest_dir, dest_fn)) else: shutil.copy2(src_path, os.path.join(dest_dir, src_file)) def MakeZip(output_dir, archive_name, file_list, file_relative_dir, raise_error=True, remove_archive_directory=True): """Packs files into a new zip archive. Files are first copied into a directory within the output_dir named for the archive_name, which will be created if necessary and emptied if it already exists. The files are then then packed using archive names relative to the output_dir. That is, if the zipfile is unpacked in place, it will create a directory identical to the new archive_name directory, in the output_dir. The zip file will be named as the archive_name, plus '.zip'. Args: output_dir: Absolute path to the directory in which the archive is to be created. archive_dir: Subdirectory of output_dir holding files to be added to the new zipfile. file_list: List of paths to files or subdirectories, relative to the file_relative_dir. file_relative_dir: Absolute path to the directory containing the files and subdirectories in the file_list. raise_error: Whether to raise a PathNotFound error if one of the files in the list is not found. remove_archive_directory: Whether to remove the archive staging directory before copying files over to it. Returns: A tuple consisting of (archive_dir, zip_file_path), where archive_dir is the full path to the newly created archive_name subdirectory. Raises: PathNotFound if any of the files in the list is not found, unless raise_error is False, in which case the error will be ignored. """ start_time = time.clock() # Collect files into the archive directory. archive_dir = os.path.join(output_dir, archive_name) print 'output_dir: %s, archive_name: %s' % (output_dir, archive_name) print 'archive_dir: %s, remove_archive_directory: %s, exists: %s' % ( archive_dir, remove_archive_directory, os.path.exists(archive_dir)) if remove_archive_directory and os.path.exists(archive_dir): # Move it even if it's not a directory as expected. This can happen with # FILES.cfg archive creation where we create an archive staging directory # that is the same name as the ultimate archive name. if not os.path.isdir(archive_dir): print 'Moving old "%s" file to create same name directory.' % archive_dir previous_archive_file = '%s.old' % archive_dir MoveFile(archive_dir, previous_archive_file) else: print 'Removing %s' % archive_dir RemoveDirectory(archive_dir) print 'Now, os.path.exists(%s): %s' % ( archive_dir, os.path.exists(archive_dir)) MaybeMakeDirectory(archive_dir) for needed_file in file_list: needed_file = needed_file.rstrip() # These paths are relative to the file_relative_dir. We need to copy # them over maintaining the relative directories, where applicable. src_path = os.path.join(file_relative_dir, needed_file) dirname, basename = os.path.split(needed_file) try: if os.path.isdir(src_path): if WIN_LINK_FUNC: WIN_LINK_FUNC(src_path, os.path.join(archive_dir, needed_file)) else: shutil.copytree(src_path, os.path.join(archive_dir, needed_file), symlinks=True) elif dirname != '' and basename != '': dest_dir = os.path.join(archive_dir, dirname) MaybeMakeDirectory(dest_dir) CopyFileToDir(src_path, dest_dir, basename, link_ok=True) else: CopyFileToDir(src_path, archive_dir, basename, link_ok=True) except PathNotFound: if raise_error: raise end_time = time.clock() print 'Took %f seconds to create archive directory.' % (end_time - start_time) # Pack the zip file. output_file = '%s.zip' % archive_dir previous_file = '%s_old.zip' % archive_dir MoveFile(output_file, previous_file) # If we have 7z, use that as it's much faster. See http://crbug.com/418702. windows_zip_cmd = None if os.path.exists('C:\\Program Files\\7-Zip\\7z.exe'): windows_zip_cmd = ['C:\\Program Files\\7-Zip\\7z.exe', 'a', '-y', '-mx1'] # On Windows we use the python zip module; on Linux and Mac, we use the zip # command as it will handle links and file bits (executable). Which is much # easier then trying to do that with ZipInfo options. start_time = time.clock() if IsWindows() and not windows_zip_cmd: print 'Creating %s' % output_file def _Addfiles(to_zip_file, dirname, files_to_add): for this_file in files_to_add: archive_name = this_file this_path = os.path.join(dirname, this_file) if os.path.isfile(this_path): # Store files named relative to the outer output_dir. archive_name = this_path.replace(output_dir + os.sep, '') if os.path.getsize(this_path) == 0: compress_method = zipfile.ZIP_STORED else: compress_method = zipfile.ZIP_DEFLATED to_zip_file.write(this_path, archive_name, compress_method) print 'Adding %s' % archive_name zip_file = zipfile.ZipFile(output_file, 'w', zipfile.ZIP_DEFLATED, allowZip64=True) try: os.path.walk(archive_dir, _Addfiles, zip_file) finally: zip_file.close() else: if IsMac() or IsLinux(): zip_cmd = ['zip', '-yr1'] else: zip_cmd = windows_zip_cmd saved_dir = os.getcwd() os.chdir(os.path.dirname(archive_dir)) command = zip_cmd + [output_file, os.path.basename(archive_dir)] result = RunCommand(command) os.chdir(saved_dir) if result and raise_error: raise ExternalError('zip failed: %s => %s' % (str(command), result)) end_time = time.clock() print 'Took %f seconds to create zip.' % (end_time - start_time) return (archive_dir, output_file) def ExtractZip(filename, output_dir, verbose=True): """ Extract the zip archive in the output directory. """ MaybeMakeDirectory(output_dir) # On Linux and Mac, we use the unzip command as it will # handle links and file bits (executable), which is much # easier then trying to do that with ZipInfo options. # # The Mac Version of unzip unfortunately does not support Zip64, whereas # the python module does, so we have to fallback to the python zip module # on Mac if the filesize is greater than 4GB. # # On Windows, try to use 7z if it is installed, otherwise fall back to python # zip module and pray we don't have files larger than 512MB to unzip. unzip_cmd = None if ((IsMac() and os.path.getsize(filename) < 4 * 1024 * 1024 * 1024) or IsLinux()): unzip_cmd = ['unzip', '-o'] elif IsWindows() and os.path.exists('C:\\Program Files\\7-Zip\\7z.exe'): unzip_cmd = ['C:\\Program Files\\7-Zip\\7z.exe', 'x', '-y'] if unzip_cmd: # Make sure path is absolute before changing directories. filepath = os.path.abspath(filename) saved_dir = os.getcwd() os.chdir(output_dir) command = unzip_cmd + [filepath] result = RunCommand(command) os.chdir(saved_dir) if result: raise ExternalError('unzip failed: %s => %s' % (str(command), result)) else: assert IsWindows() or IsMac() zf = zipfile.ZipFile(filename) # TODO(hinoka): This can be multiprocessed. for name in zf.namelist(): if verbose: print 'Extracting %s' % name zf.extract(name, output_dir) if IsMac(): # Restore permission bits. os.chmod(os.path.join(output_dir, name), zf.getinfo(name).external_attr >> 16L) def WindowsPath(path): """Returns a Windows mixed-style absolute path, given a Cygwin absolute path. The version of Python in the Chromium tree uses posixpath for os.path even on Windows, so we convert to a mixed Windows path (that is, a Windows path that uses forward slashes instead of backslashes) manually. """ # TODO(pamg): make this work for other drives too. if path.startswith('/cygdrive/c/'): return path.replace('/cygdrive/c/', 'C:/') return path def FindUpwardParent(start_dir, *desired_list): """Finds the desired object's parent, searching upward from the start_dir. Searches within start_dir and within all its parents looking for the desired directory or file, which may be given in one or more path components. Returns the first directory in which the top desired path component was found, or raises PathNotFound if it wasn't. """ desired_path = os.path.join(*desired_list) last_dir = '' cur_dir = start_dir found_path = os.path.join(cur_dir, desired_path) while not os.path.exists(found_path): last_dir = cur_dir cur_dir = os.path.dirname(cur_dir) if last_dir == cur_dir: raise PathNotFound('Unable to find %s above %s' % (desired_path, start_dir)) found_path = os.path.join(cur_dir, desired_path) # Strip the entire original desired path from the end of the one found # and remove a trailing path separator, if present. found_path = found_path[:len(found_path) - len(desired_path)] if found_path.endswith(os.sep): found_path = found_path[:len(found_path) - 1] return found_path def FindUpward(start_dir, *desired_list): """Returns a path to the desired directory or file, searching upward. Searches within start_dir and within all its parents looking for the desired directory or file, which may be given in one or more path components. Returns the full path to the desired object, or raises PathNotFound if it wasn't found. """ parent = FindUpwardParent(start_dir, *desired_list) return os.path.join(parent, *desired_list) def RunAndPrintDots(function): """Starts a background thread that prints dots while the function runs.""" def Hook(*args, **kwargs): event = threading.Event() def PrintDots(): counter = 0 while not event.isSet(): event.wait(5) sys.stdout.write('.') counter = (counter + 1) % 80 if not counter: sys.stdout.write('\n') sys.stdout.flush() t = threading.Thread(target=PrintDots) t.start() try: return function(*args, **kwargs) finally: event.set() t.join() return Hook class RunCommandFilter(object): """Class that should be subclassed to provide a filter for RunCommand.""" # Method could be a function # pylint: disable=R0201 def FilterLine(self, a_line): """Called for each line of input. The \n is included on a_line. Should return what is to be recorded as the output for this line. A result of None suppresses the line.""" return a_line def FilterDone(self, last_bits): """Acts just like FilterLine, but is called with any data collected after the last newline of the command.""" return last_bits class FilterCapture(RunCommandFilter): """Captures the text and places it into an array.""" def __init__(self): RunCommandFilter.__init__(self) self.text = [] def FilterLine(self, line): self.text.append(line.rstrip()) def FilterDone(self, text): self.text.append(text) def RunCommand(command, parser_func=None, filter_obj=None, pipes=None, print_cmd=True, timeout=None, max_time=None, **kwargs): """Runs the command list, printing its output and returning its exit status. Prints the given command (which should be a list of one or more strings), then runs it and writes its stdout and stderr to the appropriate file handles. If timeout is set, the process will be killed if output is stopped after timeout seconds. If max_time is set, the process will be killed if it runs for more than max_time. If parser_func is not given, the subprocess's output is passed to stdout and stderr directly. If the func is given, each line of the subprocess's stdout/stderr is passed to the func and then written to stdout. If filter_obj is given, all output is run through the filter a line at a time before it is written to stdout. We do not currently support parsing stdout and stderr independent of each other. In previous attempts, this led to output ordering issues. By merging them when either needs to be parsed, we avoid those ordering issues completely. pipes is a list of commands (also a list) that will receive the output of the intial command. For example, if you want to run "python a | python b | c", the "command" will be set to ['python', 'a'], while pipes will be set to [['python', 'b'],['c']] """ def TimedFlush(timeout, fh, kill_event): """Flush fh every timeout seconds until kill_event is true.""" while True: try: fh.flush() # File handle is closed, exit. except ValueError: break # Wait for kill signal or timeout. if kill_event.wait(timeout): break # TODO(all): nsylvain's CommandRunner in buildbot_slave is based on this # method. Update it when changes are introduced here. def ProcessRead(readfh, writefh, parser_func=None, filter_obj=None, log_event=None): writefh.flush() # Python on Windows writes the buffer only when it reaches 4k. Ideally # we would flush a minimum of 10 seconds. However, we only write and # flush no more often than 20 seconds to avoid flooding the master with # network traffic from unbuffered output. kill_event = threading.Event() flush_thread = threading.Thread( target=TimedFlush, args=(20, writefh, kill_event)) flush_thread.daemon = True flush_thread.start() try: in_byte = readfh.read(1) in_line = cStringIO.StringIO() while in_byte: # Capture all characters except \r. if in_byte != '\r': in_line.write(in_byte) # Write and flush on newline. if in_byte == '\n': if log_event: log_event.set() if parser_func: parser_func(in_line.getvalue().strip()) if filter_obj: filtered_line = filter_obj.FilterLine(in_line.getvalue()) if filtered_line is not None: writefh.write(filtered_line) else: writefh.write(in_line.getvalue()) in_line = cStringIO.StringIO() in_byte = readfh.read(1) if log_event and in_line.getvalue(): log_event.set() # Write remaining data and flush on EOF. if parser_func: parser_func(in_line.getvalue().strip()) if filter_obj: if in_line.getvalue(): filtered_line = filter_obj.FilterDone(in_line.getvalue()) if filtered_line is not None: writefh.write(filtered_line) else: if in_line.getvalue(): writefh.write(in_line.getvalue()) finally: kill_event.set() flush_thread.join() writefh.flush() pipes = pipes or [] # Print the given command (which should be a list of one or more strings). if print_cmd: print '\n' + subprocess.list2cmdline(command) + '\n', for pipe in pipes: print ' | ' + subprocess.list2cmdline(pipe) + '\n', sys.stdout.flush() sys.stderr.flush() if not (parser_func or filter_obj or pipes or timeout or max_time): # Run the command. The stdout and stderr file handles are passed to the # subprocess directly for writing. No processing happens on the output of # the subprocess. proc = subprocess.Popen(command, stdout=sys.stdout, stderr=sys.stderr, bufsize=0, **kwargs) else: if not (parser_func or filter_obj): filter_obj = RunCommandFilter() # Start the initial process. proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=0, **kwargs) proc_handles = [proc] if pipes: pipe_number = 0 for pipe in pipes: pipe_number = pipe_number + 1 if pipe_number == len(pipes) and not (parser_func or filter_obj): # The last pipe process needs to output to sys.stdout or filter stdout = sys.stdout else: # Output to a pipe, since another pipe is on top of us. stdout = subprocess.PIPE pipe_proc = subprocess.Popen(pipe, stdin=proc_handles[0].stdout, stdout=stdout, stderr=subprocess.STDOUT) proc_handles.insert(0, pipe_proc) # Allow proc to receive a SIGPIPE if the piped process exits. for handle in proc_handles[1:]: handle.stdout.close() log_event = threading.Event() # Launch and start the reader thread. thread = threading.Thread(target=ProcessRead, args=(proc_handles[0].stdout, sys.stdout), kwargs={'parser_func': parser_func, 'filter_obj': filter_obj, 'log_event': log_event}) kill_lock = threading.Lock() def term_then_kill(handle, initial_timeout, numtimeouts, interval): def timed_check(): for _ in range(numtimeouts): if handle.poll() is not None: return True time.sleep(interval) handle.terminate() time.sleep(initial_timeout) timed_check() if handle.poll() is None: handle.kill() timed_check() return handle.poll() is not None def kill_proc(proc_handles, message=None): with kill_lock: if proc_handles: killed = term_then_kill(proc_handles[0], 0.1, 5, 1) if message: print >> sys.stderr, message if not killed: print >> sys.stderr, 'could not kill pid %d!' % proc_handles[0].pid else: print >> sys.stderr, 'program finished with exit code %d' % ( proc_handles[0].returncode) # Prevent other timeouts from double-killing. del proc_handles[:] def timeout_func(timeout, proc_handles, log_event, finished_event): while log_event.wait(timeout): log_event.clear() if finished_event.is_set(): return message = ('command timed out: %d seconds without output, attempting to ' 'kill' % timeout) kill_proc(proc_handles, message) def maxtimeout_func(timeout, proc_handles, finished_event): if not finished_event.wait(timeout): message = ('command timed out: %d seconds elapsed' % timeout) kill_proc(proc_handles, message) timeout_thread = None maxtimeout_thread = None finished_event = threading.Event() if timeout: timeout_thread = threading.Thread(target=timeout_func, args=(timeout, proc_handles, log_event, finished_event)) timeout_thread.daemon = True if max_time: maxtimeout_thread = threading.Thread(target=maxtimeout_func, args=(max_time, proc_handles, finished_event)) maxtimeout_thread.daemon = True thread.start() if timeout_thread: timeout_thread.start() if maxtimeout_thread: maxtimeout_thread.start() # Wait for the commands to terminate. for handle in proc_handles: handle.wait() # Wake up timeout threads. finished_event.set() log_event.set() # Wait for the reader thread to complete (implies EOF reached on stdout/ # stderr pipes). thread.join() # Check whether any of the sub commands has failed. for handle in proc_handles: if handle.returncode: return handle.returncode # Wait for the command to terminate. proc.wait() return proc.returncode def GetStatusOutput(command, **kwargs): """Runs the command list, returning its result and output.""" proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, **kwargs) output = proc.communicate()[0] result = proc.returncode return (result, output) def GetCommandOutput(command): """Runs the command list, returning its output. Run the command and returns its output (stdout and stderr) as a string. If the command exits with an error, raises ExternalError. """ (result, output) = GetStatusOutput(command) if result: raise ExternalError('%s: %s' % (subprocess.list2cmdline(command), output)) return output def GetGClientCommand(platform=None): """Returns the executable command name, depending on the platform. """ if not platform: platform = sys.platform if platform.startswith('win'): # Windows doesn't want to depend on bash. return 'gclient.bat' else: return 'gclient' # Linux scripts use ssh to to move files to the archive host. def SshMakeDirectory(host, dest_path): """Creates the entire dest_path on the remote ssh host. """ command = ['ssh', host, 'mkdir', '-p', dest_path] result = RunCommand(command) if result: raise ExternalError('Failed to ssh mkdir "%s" on "%s" (%s)' % (dest_path, host, result)) def SshMoveFile(host, src_path, dest_path): """Moves src_path (if it exists) to dest_path on the remote host. """ command = ['ssh', host, 'test', '-e', src_path] result = RunCommand(command) if result: # Nothing to do if src_path doesn't exist. return result command = ['ssh', host, 'mv', src_path, dest_path] result = RunCommand(command) if result: raise ExternalError('Failed to ssh mv "%s" -> "%s" on "%s" (%s)' % (src_path, dest_path, host, result)) def SshCopyFiles(srcs, host, dst): """Copies the srcs file(s) to dst on the remote ssh host. dst is expected to exist. """ command = ['scp', srcs, host + ':' + dst] result = RunCommand(command) if result: raise ExternalError('Failed to scp "%s" to "%s" (%s)' % (srcs, host + ':' + dst, result)) def SshExtractZip(host, zipname, dst): """extract the remote zip file to dst on the remote ssh host. """ command = ['ssh', host, 'unzip', '-o', '-d', dst, zipname] result = RunCommand(command) if result: raise ExternalError('Failed to ssh unzip -o -d "%s" "%s" on "%s" (%s)' % (dst, zipname, host, result)) # unzip will create directories with access 700, which is not often what we # need. Fix the permissions for the whole archive. command = ['ssh', host, 'chmod', '-R', '755', dst] result = RunCommand(command) if result: raise ExternalError('Failed to ssh chmod -R 755 "%s" on "%s" (%s)' % (dst, host, result)) def SshCopyTree(srctree, host, dst): """Recursively copies the srctree to dst on the remote ssh host. For consistency with shutil, dst is expected to not exist. """ command = ['ssh', host, '[ -d "%s" ]' % dst] result = RunCommand(command) if result: raise ExternalError('SshCopyTree destination directory "%s" already exists.' % host + ':' + dst) SshMakeDirectory(host, os.path.dirname(dst)) command = ['scp', '-r', '-p', srctree, host + ':' + dst] result = RunCommand(command) if result: raise ExternalError('Failed to scp "%s" to "%s" (%s)' % (srctree, host + ':' + dst, result)) def ListMasters(cue='master.cfg', include_public=True, include_internal=True): """Returns all the masters found.""" # Look for "internal" masters first. path_internal = os.path.join( BUILD_DIR, os.pardir, 'build_internal', 'masters/*/' + cue) path = os.path.join(BUILD_DIR, 'masters/*/' + cue) filenames = [] if include_public: filenames += glob.glob(path) if include_internal: filenames += glob.glob(path_internal) return [os.path.abspath(os.path.dirname(f)) for f in filenames] def MasterPath(mastername, include_public=True, include_internal=True): if not mastername.startswith('master.'): mastername = 'master.' + mastername for inc, base in ( (include_public, env.Build), (include_internal, env.BuildInternal)): if inc and base: path = os.path.join(base, 'masters', mastername) if os.path.isdir(path): return path raise LookupError('Path for master %s not found' % mastername) def ListMastersWithSlaves(include_public=True, include_internal=True): masters_path = ListMasters('builders.pyl', include_public, include_internal) masters_path.extend(ListMasters('slaves.cfg', include_public, include_internal)) return masters_path def GetSlavesFromMasterPath(path, fail_hard=False): builders_path = os.path.join(path, 'builders.pyl') if os.path.exists(builders_path): return GetSlavesFromBuildersFile(builders_path) return RunSlavesCfg(os.path.join(path, 'slaves.cfg'), fail_hard=fail_hard) def GetAllSlaves(fail_hard=False, include_public=True, include_internal=True): """Return all slave objects from masters.""" slaves = [] for master in ListMastersWithSlaves(include_public, include_internal): cur_slaves = GetSlavesFromMasterPath(master, fail_hard) for slave in cur_slaves: slave['mastername'] = os.path.basename(master) slaves.extend(cur_slaves) return slaves def GetSlavesForHost(): """Get slaves for a host, defaulting to current host.""" hostname = os.getenv('TESTING_SLAVENAME') if not hostname: hostname = socket.getfqdn().split('.', 1)[0].lower() return [s for s in GetAllSlaves() if s.get('hostname') == hostname] def GetActiveSubdir(): """Get current checkout's subdir, if checkout uses subdir layout.""" rootdir, subdir = os.path.split(os.path.dirname(BUILD_DIR)) if subdir != 'b' and os.path.basename(rootdir) == 'c': return subdir def GetActiveSlavename(): slavename = os.getenv('TESTING_SLAVENAME') if not slavename: slavename = socket.getfqdn().split('.', 1)[0].lower() subdir = GetActiveSubdir() if subdir: return '%s#%s' % (slavename, subdir) return slavename def EntryToSlaveName(entry): """Produces slave name from the slaves config dict.""" name = entry.get('slavename') or entry.get('hostname') if 'subdir' in entry: return '%s#%s' % (name, entry['subdir']) return name def GetActiveMaster(slavename=None, default=None): """Returns the name of the Active master serving the current host. Parse all of the active masters with slaves matching the current hostname and optional slavename. Returns |default| if no match found. """ slavename = slavename or GetActiveSlavename() for slave in GetAllSlaves(): if slavename == EntryToSlaveName(slave): return slave['master'] return default @contextmanager def MasterEnvironment(master_dir): """Context manager that enters an enviornment similar to a master's. This involves: - Modifying 'sys.path' to include paths available to the master. - Changing directory (via os.chdir()) to the master's base directory. These changes will be reverted after the context manager completes. Args: master_dir: (str) The master's base directory. """ master_dir = os.path.abspath(master_dir) # Setup a 'sys.path' that is adequate for loading 'slaves.cfg'. old_cwd = os.getcwd() with env.GetInfraPythonPath(master_dir=master_dir).Enter(): try: os.chdir(master_dir) yield finally: os.chdir(old_cwd) def ParsePythonCfg(cfg_filepath, fail_hard=False): """Retrieves data from a python config file.""" if not os.path.exists(cfg_filepath): return None # Execute 'slaves.cfg' in the master path environment. with MasterEnvironment(os.path.dirname(os.path.abspath(cfg_filepath))): try: local_vars = {} execfile(os.path.join(cfg_filepath), local_vars) del local_vars['__builtins__'] return local_vars except Exception as e: # pylint: disable=C0323 print >>sys.stderr, 'An error occurred while parsing %s: %s' % ( cfg_filepath, e) print >>sys.stderr, traceback.format_exc() # pylint: disable=C0323 if fail_hard: raise return {} def RunSlavesCfg(slaves_cfg, fail_hard=False): """Runs slaves.cfg in a consistent way.""" slave_config = ParsePythonCfg(slaves_cfg, fail_hard=fail_hard) or {} return slave_config.get('slaves', []) def convert_json(option, _, value, parser): """Provide an OptionParser callback to unmarshal a JSON string.""" setattr(parser.values, option.dest, json.loads(value)) def b64_gz_json_encode(obj): """Serialize a python object into base64.""" # The |separators| argument is to densify the command line. return base64.b64encode(zlib.compress( json.dumps(obj or {}, sort_keys=True, separators=(',', ':')), 9)) def convert_gz_json(option, _, value, parser): """Provide an OptionParser callback to unmarshal a b64 gz JSON string.""" setattr( parser.values, option.dest, json.loads(zlib.decompress(base64.b64decode(value)))) def SafeTranslate(inputstr): """Convert a free form string to one that can be used in a path. This is similar to the safeTranslate function in buildbot. """ badchars_map = string.maketrans('\t !#$%&\'()*+,./:;<=>?@[\\]^{|}~', '______________________________') if isinstance(inputstr, unicode): inputstr = inputstr.encode('utf8') return inputstr.translate(badchars_map) def GetPrimaryProject(options): """Returns: (str) the key of the primary project, or 'None' if none exists. """ # The preferred way is to reference the 'primary_project' parameter. result = options.build_properties.get('primary_project') if result: return result # TODO(dnj): The 'primary_repo' parameter is used by some scripts to indictate # the primary project name. This is not consistently used and will be # deprecated in favor of 'primary_project' once that is rolled out. result = options.build_properties.get('primary_repo') if not result: # The 'primary_repo' property currently contains a trailing underscore. # However, this isn't an obvious thing given its name, so we'll strip it # here and remove that expectation. return result.strip('_') return None def GetBuildSortKey(options, project=None): """Reads a variety of sources to determine the current build revision. NOTE: Currently, the return value does not qualify branch name. This can present a problem with git numbering scheme, where numbers are only unique in the context of their respective branches. When this happens, this function will return a branch name as part of the sort key and its callers will need to adapt their naming/querying schemes to accommodate this. Until then, we will return 'None' as the branch name. (e.g., refs/foo/bar@{#12345} => ("refs/foo/bar", 12345) Args: options: Command-line options structure project: (str/None) If not None, the project to get the build sort key for. Otherwise, the build-wide sort key will be used. Returns: (branch, value) The qualified sortkey value branch: (str/None) The name of the branch, or 'None' if there is no branch context. Currently this always returns 'None'. value: (int) The iteration value within the specified branch Raises: (NoIdentifiedRevision) if no revision could be identified from the supplied options. """ # Is there a commit position for this build key? try: return GetCommitPosition(options, project=project) except NoIdentifiedRevision: pass # Nope; derive the sort key from the 'got_[*_]revision' build properties. Note # that this could be a Git commit (post flag day). if project: revision_key = 'got_%s_revision' % (project,) else: revision_key = 'got_revision' revision = options.build_properties.get(revision_key) if revision and not IsGitCommit(revision): return None, int(revision) raise NoIdentifiedRevision("Unable to identify revision for revision key " "[%s]" % (revision_key,)) def GetGitCommit(options, project=None): """Returns the 'git' commit hash for the specified repository This function uses environmental options to identify the 'git' commit hash for the specified repository. Args: options: Command-line options structure project: (str/None) The project key to use. If None, use the topmost repository identification properties. Raises: (NoIdentifiedRevision) if no git commit could be identified from the supplied options. """ if project: git_commit_key = 'got_%s_revision_git' % (project,) else: git_commit_key = 'got_revision_git' commit = options.build_properties.get(git_commit_key) if commit: return commit # Is 'got_[_*]revision' itself is the Git commit? if project: commit_key = 'got_%s_revision' % (project,) else: commit_key = 'got_revision' commit = options.build_properties.get(commit_key) if commit and IsGitCommit(commit): return commit raise NoIdentifiedRevision("Unable to identify commit for commit key: %s" % ( (git_commit_key, commit_key),)) def GetSortableUploadPathForSortKey(branch, value, delimiter=None): """Returns: (str) the canonical sort key path constructed from a sort key. Returns a canonical sort key path for a sort key. The result will be one of the following forms: - (Without Branch or With Branch=='refs/heads/master'): (e.g., 12345) - (With non-Master Branch): - (e.g., "refs_my-branch-12345") When a 'branch' is supplied, it is converted to a path-suitable form. This conversion replaces undesirable characters ('/') with underscores. Note that when parsing the upload path, 'rsplit' should be used to isolate the commit position value, as the branch path may have instances of the delimiter in it. See 'GetBuildSortKey' for more information about sort keys. Args: branch: (str/None) The sort key branch, or 'None' if there is no associated branch. value: (int) The sort key value. delimiter: (str) The delimiter to insert in between and when constructing the branch-inclusive form. If omitted (default), a hyphen ('-') will be used. """ if branch and branch != 'refs/heads/master': delimiter = delimiter or '-' branch = branch.replace('/', '_') return '%s%s%s' % (branch, delimiter, value) return str(value) def ParseCommitPosition(value): """Returns: The (branch, value) parsed from a commit position string. Args: value: (str) The value to parse. Raises: ValueError: If a commit position could not be parsed from 'value'. """ match = COMMIT_POSITION_RE.match(value) if not match: raise ValueError("Failed to parse commit position from '%s'" % (value,)) return match.group(1), int(match.group(2)) def BuildCommitPosition(branch, value): """Returns: A constructed commit position. An example commit position for branch 'refs/heads/master' value '12345' is: refs/heads/master@{#12345} This value can be parsed via 'ParseCommitPosition'. Args: branch: (str) The name of the commit position branch value: (int): The commit position number. """ return '%s@{#%s}' % (branch, value) def GetCommitPosition(options, project=None): """Returns: (branch, value) The parsed commit position from build options. Returns the parsed commit position from the build options. This is identified by examining the 'got_revision_cp' (or 'got_REPO_revision_cp', if 'project' is specified) keys. Args: options: Command-line options structure project: (str/None) If not None, the project to get the build sort key for. Otherwise, the build-wide sort key will be used. Returns: (branch, value) The qualified commit position value Raises: NoIdentifiedRevision: if no revision could be identified from the supplied options. ValueError: If the supplied commit position failed to parse successfully. """ if project: key = 'got_%s_revision_cp' % (project,) else: key = 'got_revision_cp' cp = options.build_properties.get(key) if not cp: raise NoIdentifiedRevision("Unable to identify the commit position; the " "build property is missing: %s" % (key,)) return ParseCommitPosition(cp) def AddPropertiesOptions(option_parser): """Registers command line options for parsing build and factory properties. After parsing, the options object will have the 'build_properties' and 'factory_properties' attributes. The corresponding values will be python dictionaries containing the properties. If the options are not given on the command line, the dictionaries will be empty. Args: option_parser: An optparse.OptionParser to register command line options for build and factory properties. """ option_parser.add_option('--build-properties', action='callback', callback=convert_json, type='string', nargs=1, default={}, help='build properties in JSON format') option_parser.add_option('--factory-properties', action='callback', callback=convert_json, type='string', nargs=1, default={}, help='factory properties in JSON format') def AddThirdPartyLibToPath(lib, override=False): """Adds the specified dir in build/third_party to sys.path. Setting 'override' to true will place the directory in the beginning of sys.path, useful for overriding previously set packages. NOTE: We would like to deprecate this method, as it allows (encourages?) scripts to define their own one-off Python path sequences, creating a difficult-to-manage state where different scripts and libraries have different path expectations. Please don't use this method if possible; it preferred to augment 'common.env' instead. """ libpath = os.path.abspath(os.path.join(BUILD_DIR, 'third_party', lib)) if override: sys.path.insert(0, libpath) else: sys.path.append(libpath) def GetLKGR(): """Connect to chromium LKGR server and get LKGR revision. On success, returns the LKGR and 'ok'. On error, returns None and the text of the error message. """ try: conn = urllib.urlopen('https://chromium-status.appspot.com/lkgr') except IOError: return (None, 'Error connecting to LKGR server! Is your internet ' 'connection working properly?') try: rev = int('\n'.join(conn.readlines())) except IOError: return (None, 'Error connecting to LKGR server! Is your internet ' 'connection working properly?') except ValueError: return None, 'LKGR server returned malformed data! Aborting...' finally: conn.close() return rev, 'ok' def AbsoluteCanonicalPath(*path): """Return the most canonical path Python can provide.""" file_path = os.path.join(*path) return os.path.realpath(os.path.abspath(os.path.expanduser(file_path))) def IsolatedImportFromPath(path, extra_paths=None): dir_path, module_file = os.path.split(path) module_file = os.path.splitext(module_file)[0] saved = sys.path sys.path = [dir_path] + (extra_paths or []) try: return __import__(module_file) except ImportError: pass finally: sys.path = saved @contextmanager def MultiPool(processes): """Manages a multiprocessing.Pool making sure to close the pool when done. This will also call pool.terminate() when an exception is raised (and re-raised the exception to the calling procedure can handle it). """ try: pool = multiprocessing.Pool(processes=processes) yield pool pool.close() except: pool.terminate() raise finally: pool.join() def ReadJsonAsUtf8(filename=None, text=None): """Read a json file or string and output a dict. This function is different from json.load and json.loads in that it returns utf8-encoded string for keys and values instead of unicode. Args: filename: path of a file to parse text: json string to parse If both 'filename' and 'text' are provided, 'filename' is used. """ def _decode_list(data): rv = [] for item in data: if isinstance(item, unicode): item = item.encode('utf-8') elif isinstance(item, list): item = _decode_list(item) elif isinstance(item, dict): item = _decode_dict(item) rv.append(item) return rv def _decode_dict(data): rv = {} for key, value in data.iteritems(): if isinstance(key, unicode): key = key.encode('utf-8') if isinstance(value, unicode): value = value.encode('utf-8') elif isinstance(value, list): value = _decode_list(value) elif isinstance(value, dict): value = _decode_dict(value) rv[key] = value return rv if filename: with open(filename, 'rb') as f: return json.load(f, object_hook=_decode_dict) if text: return json.loads(text, object_hook=_decode_dict) def GetMasterDevParameters(filename='master_cfg_params.json'): """Look for master development parameter files in the master directory. Return the parsed content if the file exists, as a dictionary. Every string value in the dictionary is utf8-encoded str. If the file is not found, returns an empty dict. This is on purpose, to make the file optional. """ if os.path.isfile(filename): return ReadJsonAsUtf8(filename=filename) return {} def FileExclusions(): all_platforms = ['.landmines', 'obj', 'gen', '.ninja_deps', '.ninja_log'] # Skip files that the testers don't care about. Mostly directories. if IsWindows(): # Remove obj or lib dir entries return all_platforms + ['cfinstaller_archive', 'lib', 'installer_archive'] if IsMac(): return all_platforms + [ # We don't need the arm bits v8 builds. 'd8_arm', 'v8_shell_arm', # pdfsqueeze is a build helper, no need to copy it to testers. 'pdfsqueeze', # We copy the framework into the app bundle, we don't need the second # copy outside the app. # TODO(mark): Since r28431, the copy in the build directory is actually # used by tests. Putting two copies in the .zip isn't great, so maybe # we can find another workaround. # 'Chromium Framework.framework', # 'Google Chrome Framework.framework', # We copy the Helper into the app bundle, we don't need the second # copy outside the app. 'Chromium Helper.app', 'Google Chrome Helper.app', 'App Shim Socket', '.deps', 'obj.host', 'obj.target', 'lib' ] if IsLinux(): return all_platforms + [ # intermediate build directories (full of .o, .d, etc.). 'appcache', 'glue', 'lib.host', 'obj.host', 'obj.target', 'src', '.deps', # scons build cruft '.sconsign.dblite', # build helper, not needed on testers 'mksnapshot', ] return all_platforms def DatabaseSetup(buildmaster_config, require_dbconfig=False): """Read database credentials in the master directory.""" if os.path.isfile('.dbconfig'): values = {} execfile('.dbconfig', values) if 'password' not in values: raise Exception('could not get db password') buildmaster_config['db_url'] = 'postgresql://%s:%s@%s/%s' % ( values['username'], values['password'], values.get('hostname', 'localhost'), values['dbname']) else: assert not require_dbconfig def ReadBuildersFile(builders_path): with open(builders_path) as fp: contents = fp.read() return ParseBuildersFileContents(builders_path, contents) def ParseBuildersFileContents(path, contents): builders = ast.literal_eval(contents) # Set some additional derived fields that are derived from the # file's location in the filesystem. basedir = os.path.dirname(os.path.abspath(path)) master_dirname = os.path.basename(basedir) master_name_comps = master_dirname.split('.')[1:] buildbot_path = '.'.join(master_name_comps) master_classname = ''.join(c[0].upper() + c[1:] for c in master_name_comps) builders['master_dirname'] = master_dirname builders.setdefault('master_classname', master_classname) builders.setdefault('buildbot_url', 'https://build.chromium.org/p/%s/' % buildbot_path) builders.setdefault('buildbucket_bucket', None) builders.setdefault('service_account_file', None) # The _str fields are printable representations of Python values: # if builders['foo'] == "hello", then builders['foo_str'] == "'hello'". # This allows them to be read back in by Python scripts properly. builders['buildbucket_bucket_str'] = repr(builders['buildbucket_bucket']) builders['service_account_file_str'] = repr(builders['service_account_file']) return builders def GetSlavesFromBuildersFile(builders_path): """Read builders_path and return a list of slave dicts.""" builders = ReadBuildersFile(builders_path) return GetSlavesFromBuilders(builders) def GetSlavesFromBuilders(builders): """Returns a list of slave dicts derived from the builders dict.""" builders_in_pool = {} # builders.pyl contains a list of builders -> slave_pools # and a list of slave_pools -> slaves. # We require that each slave is in a single pool, but each slave # may have multiple builders, so we need to build up the list of # builders each slave pool supports. for builder_name, builder_vals in builders['builders'].items(): pool_names = builder_vals['slave_pools'] for pool_name in pool_names: if pool_name not in builders_in_pool: builders_in_pool[pool_name] = set() pool_data = builders['slave_pools'][pool_name] for slave in pool_data['slaves']: builders_in_pool[pool_name].add(builder_name) # Now we can generate the list of slaves using the above lookup table. slaves = [] for pool_name, pool_data in builders['slave_pools'].items(): slave_data = pool_data['slave_data'] builder_names = sorted(builders_in_pool[pool_name]) for slave in pool_data['slaves']: slaves.append({ 'hostname': slave, 'builder_name': builder_names, 'master': builders['master_classname'], 'os': slave_data['os'], 'version': slave_data['version'], 'bits': slave_data['bits'], }) return slaves def GetSlaveNamesForBuilder(builders, builder_name): """Returns a list of slave hostnames for the given builder name.""" slaves = [] pool_names = builders['builders'][builder_name]['slave_pools'] for pool_name in pool_names: slaves.extend(builders['slave_pools'][pool_name]['slaves']) return slaves def IsClangWinBuild(build_dir, target): """Check if a ninja build has been build with Clang on Windows. This checks the build.ninja file from the build directory to see which compiler has been used for a build. """ # TODO(sebmarchand): Make this works for gn builds, currently only the gyp one # is supported. if not IsWindows(): return False build_file = os.path.join(build_dir, target, 'build.ninja') if not os.path.isfile(build_file): return False # Matches e.g. "cl_x86 = path/to/clang-cl.exe" clang_cl_re = re.compile( r'^cl_x\d\d\s+\=\s+(?P[^ ]+)\s.*$', re.VERBOSE) for line in open(build_file): m = clang_cl_re.match(line) if m: return 'clang' in m.group('compiler_path') return False