# -*- coding: utf-8 -*- # Copyright 2013 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Utilities for toolchain build.""" __author__ = "asharif@google.com (Ahmad Sharif)" from contextlib import contextmanager import os import re import shutil import sys from cros_utils import command_executer from cros_utils import logger CHROMEOS_SCRIPTS_DIR = "/mnt/host/source/src/scripts" TOOLCHAIN_UTILS_PATH = ( "/mnt/host/source/src/third_party/toolchain-utils/" "cros_utils/toolchain_utils.sh" ) def GetChromeOSVersionFromLSBVersion(lsb_version): """Get Chromeos version from Lsb version.""" ce = command_executer.GetCommandExecuter() command = ( "git ls-remote " "https://chromium.googlesource.com/chromiumos/manifest.git " "refs/heads/release-R*" ) ret, out, _ = ce.RunCommandWOutput(command, print_to_console=False) assert ret == 0, "Command %s failed" % command lower = [] for line in out.splitlines(): mo = re.search(r"refs/heads/release-R(\d+)-(\d+)\.B", line) if mo: revision = int(mo.group(1)) build = int(mo.group(2)) lsb_build = int(lsb_version.split(".")[0]) if lsb_build > build: lower.append(revision) lower = sorted(lower) if lower: return "R%d-%s" % (lower[-1] + 1, lsb_version) else: return "Unknown" def ApplySubs(string, *substitutions): for pattern, replacement in substitutions: string = re.sub(pattern, replacement, string) return string def UnitToNumber(unit_num, base=1000): """Convert a number with unit to float.""" unit_dict = {"kilo": base, "mega": base ** 2, "giga": base ** 3} unit_num = unit_num.lower() mo = re.search(r"(\d*)(.+)?", unit_num) number = mo.group(1) unit = mo.group(2) if not unit: return float(number) for k, v in unit_dict.items(): if k.startswith(unit): return float(number) * v raise RuntimeError("Unit: %s not found in byte: %s!" % (unit, unit_num)) def GetFilenameFromString(string): return ApplySubs( string, (r"/", "__"), (r"\s", "_"), (r'[\\$="?^]', ""), ) def GetRoot(scr_name): """Break up pathname into (dir+name).""" abs_path = os.path.abspath(scr_name) return (os.path.dirname(abs_path), os.path.basename(abs_path)) def GetChromeOSKeyFile(chromeos_root): return os.path.join( chromeos_root, "src", "scripts", "mod_for_test_scripts", "ssh_keys", "testing_rsa", ) def GetChrootPath(chromeos_root): return os.path.join(chromeos_root, "chroot") def GetInsideChrootPath(chromeos_root, file_path): if not file_path.startswith(GetChrootPath(chromeos_root)): raise RuntimeError( "File: %s doesn't seem to be in the chroot: %s" % (file_path, chromeos_root) ) return file_path[len(GetChrootPath(chromeos_root)) :] def GetOutsideChrootPath(chromeos_root, file_path): return os.path.join(GetChrootPath(chromeos_root), file_path.lstrip("/")) def FormatQuotedCommand(command): return ApplySubs(command, ('"', r"\"")) def FormatCommands(commands): return ApplySubs( str(commands), ("&&", "&&\n"), (";", ";\n"), (r"\n+\s*", "\n") ) def GetImageDir(chromeos_root, board): return os.path.join(chromeos_root, "src", "build", "images", board) def LabelLatestImage(chromeos_root, board, label, vanilla_path=None): image_dir = GetImageDir(chromeos_root, board) latest_image_dir = os.path.join(image_dir, "latest") latest_image_dir = os.path.realpath(latest_image_dir) latest_image_dir = os.path.basename(latest_image_dir) retval = 0 with WorkingDirectory(image_dir): command = "ln -sf -T %s %s" % (latest_image_dir, label) ce = command_executer.GetCommandExecuter() retval = ce.RunCommand(command) if retval: return retval if vanilla_path: command = "ln -sf -T %s %s" % (vanilla_path, "vanilla") retval2 = ce.RunCommand(command) return retval2 return retval def DoesLabelExist(chromeos_root, board, label): image_label = os.path.join(GetImageDir(chromeos_root, board), label) return os.path.exists(image_label) def GetBuildPackagesCommand(board, usepkg=False, debug=False): if usepkg: usepkg_flag = "--usepkg" else: usepkg_flag = "--nousepkg" if debug: withdebug_flag = "--withdebug" else: withdebug_flag = "--nowithdebug" return ( "%s/build_packages %s --withdev --withtest --withautotest " "--skip_toolchain_update %s --board=%s " "--accept_licenses=@CHROMEOS" % (CHROMEOS_SCRIPTS_DIR, usepkg_flag, withdebug_flag, board) ) def GetBuildImageCommand(board, dev=False): dev_args = "" if dev: dev_args = "--noenable_rootfs_verification --disk_layout=2gb-rootfs" return "%s/build_image --board=%s %s test" % ( CHROMEOS_SCRIPTS_DIR, board, dev_args, ) def GetSetupBoardCommand(board, usepkg=None, force=None): """Get setup_board command.""" options = [] if usepkg: options.append("--usepkg") else: options.append("--nousepkg") if force: options.append("--force") options.append("--accept-licenses=@CHROMEOS") return "setup_board --board=%s %s" % (board, " ".join(options)) def CanonicalizePath(path): path = os.path.expanduser(path) path = os.path.realpath(path) return path def GetCtargetFromBoard(board, chromeos_root): """Get Ctarget from board.""" base_board = board.split("_")[0] command = "source %s; get_ctarget_from_board %s" % ( TOOLCHAIN_UTILS_PATH, base_board, ) ce = command_executer.GetCommandExecuter() ret, out, _ = ce.ChrootRunCommandWOutput(chromeos_root, command) if ret != 0: raise ValueError("Board %s is invalid!" % board) # Remove ANSI escape sequences. out = StripANSIEscapeSequences(out) return out.strip() def GetArchFromBoard(board, chromeos_root): """Get Arch from board.""" base_board = board.split("_")[0] command = "source %s; get_board_arch %s" % ( TOOLCHAIN_UTILS_PATH, base_board, ) ce = command_executer.GetCommandExecuter() ret, out, _ = ce.ChrootRunCommandWOutput(chromeos_root, command) if ret != 0: raise ValueError("Board %s is invalid!" % board) # Remove ANSI escape sequences. out = StripANSIEscapeSequences(out) return out.strip() def GetGccLibsDestForBoard(board, chromeos_root): """Get gcc libs destination from board.""" arch = GetArchFromBoard(board, chromeos_root) if arch == "x86": return "/build/%s/usr/lib/gcc/" % board if arch == "amd64": return "/build/%s/usr/lib64/gcc/" % board if arch == "arm": return "/build/%s/usr/lib/gcc/" % board if arch == "arm64": return "/build/%s/usr/lib/gcc/" % board raise ValueError("Arch %s is invalid!" % arch) def StripANSIEscapeSequences(string): string = re.sub(r"\x1b\[[0-9]*[a-zA-Z]", "", string) return string def GetChromeSrcDir(): return "var/cache/distfiles/target/chrome-src/src" def GetEnvStringFromDict(env_dict): return " ".join(['%s="%s"' % var for var in env_dict.items()]) def MergeEnvStringWithDict(env_string, env_dict, prepend=True): """Merge env string with dict.""" if not env_string.strip(): return GetEnvStringFromDict(env_dict) override_env_list = [] ce = command_executer.GetCommandExecuter() for k, v in env_dict.items(): v = v.strip("\"'") if prepend: new_env = '%s="%s $%s"' % (k, v, k) else: new_env = '%s="$%s %s"' % (k, k, v) command = "; ".join([env_string, new_env, "echo $%s" % k]) ret, out, _ = ce.RunCommandWOutput(command) override_env_list.append("%s=%r" % (k, out.strip())) ret = env_string + " " + " ".join(override_env_list) return ret.strip() def GetAllImages(chromeos_root, board): ce = command_executer.GetCommandExecuter() command = "find %s/src/build/images/%s -name chromiumos_test_image.bin" % ( chromeos_root, board, ) ret, out, _ = ce.RunCommandWOutput(command) assert ret == 0, "Could not run command: %s" % command return out.splitlines() def IsFloat(text): if text is None: return False try: float(text) return True except ValueError: return False def RemoveChromeBrowserObjectFiles(chromeos_root, board): """Remove any object files from all the posible locations.""" out_dir = os.path.join( GetChrootPath(chromeos_root), "var/cache/chromeos-chrome/chrome-src/src/out_%s" % board, ) if os.path.exists(out_dir): shutil.rmtree(out_dir) logger.GetLogger().LogCmd("rm -rf %s" % out_dir) out_dir = os.path.join( GetChrootPath(chromeos_root), "var/cache/chromeos-chrome/chrome-src-internal/src/out_%s" % board, ) if os.path.exists(out_dir): shutil.rmtree(out_dir) logger.GetLogger().LogCmd("rm -rf %s" % out_dir) @contextmanager def WorkingDirectory(new_dir): """Get the working directory.""" old_dir = os.getcwd() if old_dir != new_dir: msg = "cd %s" % new_dir logger.GetLogger().LogCmd(msg) os.chdir(new_dir) yield new_dir if old_dir != new_dir: msg = "cd %s" % old_dir logger.GetLogger().LogCmd(msg) os.chdir(old_dir) def HasGitStagedChanges(git_dir): """Return True if git repository has staged changes.""" command = f"cd {git_dir} && git diff --quiet --cached --exit-code HEAD" return command_executer.GetCommandExecuter().RunCommand( command, print_to_console=False ) def HasGitUnstagedChanges(git_dir): """Return True if git repository has un-staged changes.""" command = f"cd {git_dir} && git diff --quiet --exit-code HEAD" return command_executer.GetCommandExecuter().RunCommand( command, print_to_console=False ) def HasGitUntrackedChanges(git_dir): """Return True if git repository has un-tracked changes.""" command = ( f"cd {git_dir} && test -z " "$(git ls-files --exclude-standard --others)" ) return command_executer.GetCommandExecuter().RunCommand( command, print_to_console=False ) def GitGetCommitHash(git_dir, commit_symbolic_name): """Return githash for the symbolic git commit. For example, commit_symbolic_name could be "cros/gcc.gnu.org/branches/gcc/gcc-4_8-mobile, this function returns the git hash for this symbolic name. Args: git_dir: a git working tree. commit_symbolic_name: a symbolic name for a particular git commit. Returns: The git hash for the symbolic name or None if fails. """ command = ( f"cd {git_dir} && git log -n 1" f' --pretty="format:%H" {commit_symbolic_name}' ) rv, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( command, print_to_console=False ) if rv == 0: return out.strip() return None def IsGitTreeClean(git_dir): """Test if git tree has no local changes. Args: git_dir: git tree directory. Returns: True if git dir is clean. """ if HasGitStagedChanges(git_dir): logger.GetLogger().LogWarning("Git tree has staged changes.") return False if HasGitUnstagedChanges(git_dir): logger.GetLogger().LogWarning("Git tree has unstaged changes.") return False if HasGitUntrackedChanges(git_dir): logger.GetLogger().LogWarning("Git tree has un-tracked changes.") return False return True def GetGitChangesAsList(git_dir, path=None, staged=False): """Get changed files as a list. Args: git_dir: git tree directory. path: a relative path that is part of the tree directory, could be null. staged: whether to include staged files as well. Returns: A list containing all the changed files. """ command = f"cd {git_dir} && git diff --name-only" if staged: command += " --cached" if path: command += " -- " + path _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( command, print_to_console=False ) rv = [] for line in out.splitlines(): rv.append(line) return rv def IsChromeOsTree(chromeos_root): return os.path.isdir( os.path.join(chromeos_root, "src/third_party/chromiumos-overlay") ) and os.path.isdir(os.path.join(chromeos_root, "manifest")) def DeleteChromeOsTree(chromeos_root, dry_run=False): """Delete a ChromeOs tree *safely*. Args: chromeos_root: dir of the tree, could be a relative one (but be careful) dry_run: only prints out the command if True Returns: True if everything is ok. """ if not IsChromeOsTree(chromeos_root): logger.GetLogger().LogWarning( f'"{chromeos_root}" does not seem to be a' " valid chromeos tree, do nothing." ) return False cmd0 = f"cd {chromeos_root} && cros_sdk --delete" if dry_run: print(cmd0) else: if ( command_executer.GetCommandExecuter().RunCommand( cmd0, print_to_console=True ) != 0 ): return False cmd1 = ( f'export CHROMEOSDIRNAME="$(dirname $(cd {chromeos_root} && pwd))" && ' f'export CHROMEOSBASENAME="$(basename $(cd {chromeos_root} && pwd))" && ' "cd $CHROMEOSDIRNAME && sudo rm -fr $CHROMEOSBASENAME" ) if dry_run: print(cmd1) return True return ( command_executer.GetCommandExecuter().RunCommand( cmd1, print_to_console=True ) == 0 ) def BooleanPrompt( prompt="Do you want to continue?", default=True, true_value="yes", false_value="no", prolog=None, ): """Helper function for processing boolean choice prompts. Args: prompt: The question to present to the user. default: Boolean to return if the user just presses enter. true_value: The text to display that represents a True returned. false_value: The text to display that represents a False returned. prolog: The text to display before prompt. Returns: True or False. """ true_value, false_value = true_value.lower(), false_value.lower() true_text, false_text = true_value, false_value if true_value == false_value: raise ValueError( "true_value and false_value must differ: got %r" % true_value ) if default: true_text = true_text[0].upper() + true_text[1:] else: false_text = false_text[0].upper() + false_text[1:] prompt = "\n%s (%s/%s)? " % (prompt, true_text, false_text) if prolog: prompt = "\n%s\n%s" % (prolog, prompt) while True: try: # pylint: disable=input-builtin, bad-builtin response = input(prompt).lower() except EOFError: # If the user hits CTRL+D, or stdin is disabled, use the default. print() response = None except KeyboardInterrupt: # If the user hits CTRL+C, just exit the process. print() print("CTRL+C detected; exiting") sys.exit() if not response: return default if true_value.startswith(response): if not false_value.startswith(response): return True # common prefix between the two... elif false_value.startswith(response): return False # pylint: disable=unused-argument def rgb2short(r, g, b): """Converts RGB values to xterm-256 color.""" redcolor = [255, 124, 160, 196, 9] greencolor = [255, 118, 82, 46, 10] if g == 0: return redcolor[r // 52] if r == 0: return greencolor[g // 52] return 4