#!/usr/bin/python2 # # Copyright 2010 Google Inc. All Rights Reserved. """Tool script for auto dejagnu.""" __author__ = 'shenhan@google.com (Han Shen)' import getpass import optparse import os from os import path import re import shutil import stat import sys import tempfile import time import lock_machine import tc_enter_chroot from cros_utils import command_executer from cros_utils import constants from cros_utils import logger from cros_utils import misc def ProcessArguments(argv): """Processing/validating script arguments.""" parser = optparse.OptionParser(description=( 'Launches gcc dejagnu test in chroot for chromeos toolchain, compares ' 'the test result with a repository baseline and prints out the result.'), usage='run_dejagnu options') parser.add_option('-c', '--chromeos_root', dest='chromeos_root', help='Required. Specify chromeos root') parser.add_option('-m', '--mount', dest='mount', help=('Specify gcc source to mount instead of "auto". ' 'Under "auto" mode, which is the default - gcc is ' 'checked out and built automatically at default ' 'directories. Under "mount" mode ' '- the gcc_source is set to "$chromeos_' 'root/chroot/usr/local/toolchain_root/gcc", which is ' 'the mount point for this option value, the ' 'gcc-build-dir then is computed as ' '"${gcc_source_dir}-build-${ctarget}". In this mode, ' 'a complete gcc build must be performed in the ' 'computed gcc-build-dir beforehand.')) parser.add_option('-b', '--board', dest='board', help=('Required. Specify board.')) parser.add_option('-r', '--remote', dest='remote', help=('Required. Specify addresses/names of the board, ' 'seperate each address/name using comma(\',\').')) parser.add_option('-f', '--flags', dest='flags', help='Optional. Extra run test flags to pass to dejagnu.') parser.add_option('-k', '--keep', dest='keep_intermediate_files', action='store_true', default=False, help=('Optional. Default to false. Do not remove dejagnu ' 'intermediate files after test run.')) parser.add_option('--cleanup', dest='cleanup', default=None, help=('Optional. Values to this option could be ' '\'mount\' (unmount gcc source and ' 'directory directory, ' 'only valid when --mount is given), ' '\'chroot\' (delete chroot) and ' '\'chromeos\' (delete the whole chromeos tree).')) parser.add_option('-t', '--tools', dest='tools', default='gcc,g++', help=('Optional. Specify which tools to check, using ' '","(comma) as separator. A typical value would be ' '"g++" so that only g++ tests are performed. ' 'Defaults to "gcc,g++".')) options, args = parser.parse_args(argv) if not options.chromeos_root: raise SyntaxError('Missing argument for --chromeos_root.') if not options.remote: raise SyntaxError('Missing argument for --remote.') if not options.board: raise SyntaxError('Missing argument for --board.') if options.cleanup == 'mount' and not options.mount: raise SyntaxError('--cleanup=\'mount\' not valid unless --mount is given.') if options.cleanup and not ( options.cleanup == 'mount' or \ options.cleanup == 'chroot' or options.cleanup == 'chromeos'): raise ValueError('Invalid option value for --cleanup') if options.cleanup and options.keep_intermediate_files: raise SyntaxError('Only one of --keep and --cleanup could be given.') return options class DejagnuExecuter(object): """The class wrapper for dejagnu test executer.""" def __init__(self, base_dir, mount, chromeos_root, remote, board, flags, keep_intermediate_files, tools, cleanup): self._l = logger.GetLogger() self._chromeos_root = chromeos_root self._chromeos_chroot = path.join(chromeos_root, 'chroot') if mount: self._gcc_source_dir_to_mount = mount self._gcc_source_dir = path.join(constants.MOUNTED_TOOLCHAIN_ROOT, 'gcc') else: self._gcc_source_dir = None self._remote = remote self._board = board ## Compute target from board self._target = misc.GetCtargetFromBoard(board, chromeos_root) if not self._target: raise ValueError('Unsupported board "%s"' % board) self._executer = command_executer.GetCommandExecuter() self._flags = flags or '' self._base_dir = base_dir self._tmp_abs = None self._keep_intermediate_files = keep_intermediate_files self._tools = tools.split(',') self._cleanup = cleanup def SetupTestingDir(self): self._tmp_abs = tempfile.mkdtemp( prefix='dejagnu_', dir=path.join(self._chromeos_chroot, 'tmp')) self._tmp = self._tmp_abs[len(self._chromeos_chroot):] self._tmp_testing_rsa = path.join(self._tmp, 'testing_rsa') self._tmp_testing_rsa_abs = path.join(self._tmp_abs, 'testing_rsa') def MakeCheckString(self): return ' '.join(['check-{0}'.format(t) for t in self._tools if t]) def CleanupIntermediateFiles(self): if self._tmp_abs and path.isdir(self._tmp_abs): if self._keep_intermediate_files: self._l.LogOutput( 'Your intermediate dejagnu files are kept, you can re-run ' 'inside chroot the command:') self._l.LogOutput( ' DEJAGNU={0} make -C {1} {2} RUNTESTFLAGS="--target_board={3} {4}"' \ .format(path.join(self._tmp, 'site.exp'), self._gcc_build_dir, self.MakeCheckString(), self._board, self._flags)) else: self._l.LogOutput('[Cleanup] - Removing temp dir - {0}'.format( self._tmp_abs)) shutil.rmtree(self._tmp_abs) def Cleanup(self): if not self._cleanup: return # Optionally cleanup mounted diretory, chroot and chromeos tree. if self._cleanup == 'mount' or self._cleanup == 'chroot' or \ self._cleanup == 'chromeos': # No exceptions are allowed from this method. try: self._l.LogOutput('[Cleanup] - Unmounting directories ...') self.MountGccSourceAndBuildDir(unmount=True) except: print 'Warning: failed to unmount gcc source/build directory.' if self._cleanup == 'chroot' or self._cleanup == 'chromeos': self._l.LogOutput('[Cleanup]: Deleting chroot inside \'{0}\''.format( self._chromeos_root)) command = 'cd %s; cros_sdk --delete' % self._chromeos_root rv = self._executer.RunCommand(command) if rv: self._l.LogWarning('Warning - failed to delete chroot.') # Delete .cache - crosbug.com/34956 command = 'sudo rm -fr %s' % os.path.join(self._chromeos_root, '.cache') rv = self._executer.RunCommand(command) if rv: self._l.LogWarning('Warning - failed to delete \'.cache\'.') if self._cleanup == 'chromeos': self._l.LogOutput('[Cleanup]: Deleting chromeos tree \'{0}\' ...'.format( self._chromeos_root)) command = 'rm -fr {0}'.format(self._chromeos_root) rv = self._executer.RunCommand(command) if rv: self._l.LogWarning('Warning - failed to remove chromeos tree.') def PrepareTestingRsaKeys(self): if not path.isfile(self._tmp_testing_rsa_abs): shutil.copy( path.join(self._chromeos_root, 'src/scripts/mod_for_test_scripts/ssh_keys/testing_rsa'), self._tmp_testing_rsa_abs) os.chmod(self._tmp_testing_rsa_abs, stat.S_IRUSR) def PrepareTestFiles(self): """Prepare site.exp and board exp files.""" # Create the boards directory. os.mkdir('%s/boards' % self._tmp_abs) # Generate the chromeos.exp file. with open('%s/chromeos.exp.in' % self._base_dir, 'r') as template_file: content = template_file.read() substitutions = dict({ '__boardname__': self._board, '__board_hostname__': self._remote, '__tmp_testing_rsa__': self._tmp_testing_rsa, '__tmp_dir__': self._tmp }) for pat, sub in substitutions.items(): content = content.replace(pat, sub) board_file_name = '%s/boards/%s.exp' % (self._tmp_abs, self._board) with open(board_file_name, 'w') as board_file: board_file.write(content) # Generate the site file with open('%s/site.exp' % self._tmp_abs, 'w') as site_file: site_file.write('set target_list "%s"\n' % self._board) def PrepareGcc(self): if self._gcc_source_dir: self.PrepareGccFromCustomizedPath() else: self.PrepareGccDefault() self._l.LogOutput('Gcc source dir - {0}'.format(self._gcc_source_dir)) self._l.LogOutput('Gcc build dir - {0}'.format(self._gcc_top_build_dir)) def PrepareGccFromCustomizedPath(self): """Prepare gcc source, build directory from mounted source.""" # We have these source directories - # _gcc_source_dir # e.g. '/usr/local/toolchain_root/gcc' # _gcc_source_dir_abs # e.g. '/somewhere/chromeos.live/chroot/usr/local/toolchain_root/gcc' # _gcc_source_dir_to_mount # e.g. '/somewhere/gcc' self._gcc_source_dir_abs = path.join(self._chromeos_chroot, self._gcc_source_dir.lstrip('/')) if not path.isdir(self._gcc_source_dir_abs) and \ self._executer.RunCommand( 'sudo mkdir -p {0}'.format(self._gcc_source_dir_abs)): raise RuntimeError("Failed to create \'{0}\' inside chroot.".format( self._gcc_source_dir)) if not (path.isdir(self._gcc_source_dir_to_mount) and path.isdir(path.join(self._gcc_source_dir_to_mount, 'gcc'))): raise RuntimeError('{0} is not a valid gcc source tree.'.format( self._gcc_source_dir_to_mount)) # We have these build directories - # _gcc_top_build_dir # e.g. '/usr/local/toolchain_root/gcc-build-x86_64-cros-linux-gnu' # _gcc_top_build_dir_abs # e.g. '/somewhere/chromeos.live/chroo/tusr/local/toolchain_root/ # gcc-build-x86_64-cros-linux-gnu' # _gcc_build_dir # e.g. '/usr/local/toolchain_root/gcc-build-x86_64-cros-linux-gnu/gcc' # _gcc_build_dir_to_mount # e.g. '/somewhere/gcc-build-x86_64-cros-linux-gnu' self._gcc_top_build_dir = '{0}-build-{1}'.format( self._gcc_source_dir.rstrip('/'), self._target) self._gcc_build_dir = path.join(self._gcc_top_build_dir, 'gcc') self._gcc_build_dir_to_mount = '{0}-build-{1}'.format( self._gcc_source_dir_to_mount, self._target) self._gcc_top_build_dir_abs = path.join(self._chromeos_chroot, self._gcc_top_build_dir.lstrip('/')) if not path.isdir(self._gcc_top_build_dir_abs) and \ self._executer.RunCommand( 'sudo mkdir -p {0}'.format(self._gcc_top_build_dir_abs)): raise RuntimeError('Failed to create \'{0}\' inside chroot.'.format( self._gcc_top_build_dir)) if not (path.isdir(self._gcc_build_dir_to_mount) and path.join( self._gcc_build_dir_to_mount, 'gcc')): raise RuntimeError('{0} is not a valid gcc build tree.'.format( self._gcc_build_dir_to_mount)) # All check passed. Now mount gcc source and build directories. self.MountGccSourceAndBuildDir() def PrepareGccDefault(self): """Auto emerging gcc for building purpose only.""" ret = self._executer.ChrootRunCommandWOutput( self._chromeos_root, 'equery w cross-%s/gcc' % self._target)[1] ret = path.basename(ret.strip()) # ret is expected to be something like 'gcc-4.6.2-r11.ebuild' or # 'gcc-9999.ebuild' parse it. matcher = re.match('((.*)-r\d+).ebuild', ret) if matcher: gccrevision, gccversion = matcher.group(1, 2) elif ret == 'gcc-9999.ebuild': gccrevision = 'gcc-9999' gccversion = 'gcc-9999' else: raise RuntimeError('Failed to get gcc version.') gcc_portage_dir = '/var/tmp/portage/cross-%s/%s/work' % (self._target, gccrevision) self._gcc_source_dir = path.join(gcc_portage_dir, gccversion) self._gcc_top_build_dir = (gcc_portage_dir + '/%s-build-%s') % ( gccversion, self._target) self._gcc_build_dir = path.join(self._gcc_top_build_dir, 'gcc') gcc_build_dir_abs = path.join(self._chromeos_root, 'chroot', self._gcc_build_dir.lstrip('/')) if not path.isdir(gcc_build_dir_abs): ret = self._executer.ChrootRunCommand(self._chromeos_root, ( 'ebuild $(equery w cross-%s/gcc) clean prepare compile' % ( self._target))) if ret: raise RuntimeError('ebuild gcc failed.') def MakeCheck(self): self.MountGccSourceAndBuildDir() cmd = ('cd %s ; ' 'DEJAGNU=%s make %s RUNTESTFLAGS="--target_board=%s %s"' % (self._gcc_build_dir, path.join(self._tmp, 'site.exp'), self.MakeCheckString(), self._board, self._flags)) self._executer.ChrootRunCommand(self._chromeos_root, cmd) def ValidateFailures(self): validate_failures_py = path.join( self._gcc_source_dir, 'contrib/testsuite-management/validate_failures.py') cmd = 'cd {0} ; {1} --build_dir={0}'.format(self._gcc_top_build_dir, validate_failures_py) self.MountGccSourceAndBuildDir() ret = self._executer.ChrootRunCommandWOutput(self._chromeos_root, cmd) if ret[0] != 0: self._l.LogWarning('*** validate_failures.py exited with non-zero code,' 'please run it manually inside chroot - \n' ' ' + cmd) return ret # This method ensures necessary mount points before executing chroot comamnd. def MountGccSourceAndBuildDir(self, unmount=False): mount_points = [tc_enter_chroot.MountPoint(self._gcc_source_dir_to_mount, self._gcc_source_dir_abs, getpass.getuser(), 'ro'), tc_enter_chroot.MountPoint(self._gcc_build_dir_to_mount, self._gcc_top_build_dir_abs, getpass.getuser(), 'rw')] for mp in mount_points: if unmount: if mp.UnMount(): raise RuntimeError('Failed to unmount {0}'.format(mp.mount_dir)) else: self._l.LogOutput('{0} unmounted successfully.'.format(mp.mount_dir)) elif mp.DoMount(): raise RuntimeError( 'Failed to mount {0} onto {1}'.format(mp.external_dir, mp.mount_dir)) else: self._l.LogOutput('{0} mounted successfully.'.format(mp.mount_dir)) # The end of class DejagnuExecuter def TryAcquireMachine(remotes): available_machine = None for r in remotes.split(','): machine = lock_machine.Machine(r) if machine.TryLock(timeout=300, exclusive=True): available_machine = machine break else: logger.GetLogger().LogWarning( '*** Failed to lock machine \'{0}\'.'.format(r)) if not available_machine: raise RuntimeError("Failed to acquire one machine from \'{0}\'.".format( remotes)) return available_machine def Main(argv): opts = ProcessArguments(argv) available_machine = TryAcquireMachine(opts.remote) executer = DejagnuExecuter( misc.GetRoot(argv[0])[0], opts.mount, opts.chromeos_root, available_machine._name, opts.board, opts.flags, opts.keep_intermediate_files, opts.tools, opts.cleanup) # Return value is a 3- or 4-element tuple # element#1 - exit code # element#2 - stdout # element#3 - stderr # element#4 - exception infor # Some other scripts need these detailed information. ret = (1, '', '') try: executer.SetupTestingDir() executer.PrepareTestingRsaKeys() executer.PrepareTestFiles() executer.PrepareGcc() executer.MakeCheck() ret = executer.ValidateFailures() except Exception as e: # At least log the exception on console. print e # The #4 element encodes the runtime exception. ret = (1, '', '', 'Exception happened during execution: \n' + str(e)) finally: available_machine.Unlock(exclusive=True) executer.CleanupIntermediateFiles() executer.Cleanup() return ret if __name__ == '__main__': retval = Main(sys.argv)[0] sys.exit(retval)