From 4b133961b76c2cb8bc58f0ea2cded9e3438ffb6f Mon Sep 17 00:00:00 2001 From: Manoj Gupta Date: Wed, 2 Nov 2016 10:03:17 -0700 Subject: Download autotest files with crosperf and add autotest path to test_that when running non-telemetry tests BUG: chromium:647429 TEST:crosperf unit tests, sample experiment file that runs WebGl_Aquarium Change-Id: I067e350fee36596ce269b954773c39a3868632aa Reviewed-on: https://chrome-internal-review.googlesource.com/302296 Commit-Ready: Manoj Gupta Tested-by: Manoj Gupta Reviewed-by: Caroline Tice --- crosperf/benchmark_run_unittest.py | 2 + crosperf/download_images.py | 106 ++++++++++++++++++++++++++- crosperf/download_images_unittest.py | 34 ++++++++- crosperf/experiment_factory.py | 20 +++-- crosperf/experiment_factory_unittest.py | 13 +++- crosperf/experiment_file.py | 16 +++- crosperf/label.py | 4 + crosperf/machine_manager_unittest.py | 7 +- crosperf/mock_instance.py | 2 + crosperf/results_cache_unittest.py | 18 ++--- crosperf/settings.py | 9 ++- crosperf/settings_factory.py | 7 ++ crosperf/settings_factory_unittest.py | 5 +- crosperf/settings_unittest.py | 125 ++++++++++++++++++-------------- crosperf/suite_runner.py | 10 ++- crosperf/suite_runner_unittest.py | 7 +- 16 files changed, 290 insertions(+), 95 deletions(-) diff --git a/crosperf/benchmark_run_unittest.py b/crosperf/benchmark_run_unittest.py index b37594dc..9af66a33 100755 --- a/crosperf/benchmark_run_unittest.py +++ b/crosperf/benchmark_run_unittest.py @@ -49,6 +49,7 @@ class BenchmarkRunTest(unittest.TestCase): self.test_label = MockLabel( 'test1', 'image1', + 'autotest_dir', '/tmp/test_benchmark_run', 'x86-alex', 'chromeos2-row1-rack4-host9.cros', @@ -70,6 +71,7 @@ class BenchmarkRunTest(unittest.TestCase): my_label = MockLabel( 'test1', 'image1', + 'autotest_dir', '/tmp/test_benchmark_run', 'x86-alex', 'chromeos2-row1-rack4-host9.cros', diff --git a/crosperf/download_images.py b/crosperf/download_images.py index c07d82d0..cabeeb5a 100644 --- a/crosperf/download_images.py +++ b/crosperf/download_images.py @@ -17,6 +17,10 @@ class MissingImage(Exception): """Raised when the requested image does not exist in gs://""" +class MissingFile(Exception): + """Raised when the requested file does not exist in gs://""" + + class ImageDownloader(object): """Download images from Cloud Storage.""" @@ -85,7 +89,102 @@ class ImageDownloader(object): if retval != 0: raise MissingImage('Cannot uncompress image: %s.' % build_id) - def Run(self, chromeos_root, xbuddy_label): + def DownloadSingleAutotestFile(self, chromeos_root, build_id, + package_file_name): + # Verify if package files exist + status = 0 + gs_package_name = ('gs://chromeos-image-archive/%s/%s' % + (build_id, package_file_name)) + if not test_flag.GetTestMode(): + cmd = 'gsutil ls %s' % gs_package_name + status = self._ce.ChrootRunCommand(chromeos_root, cmd) + if status != 0: + raise MissingFile('Cannot find autotest package file: %s.' % + package_file_name) + + if self.log_level == 'average': + self._logger.LogOutput('Preparing to download %s package to local ' + 'directory.' % package_file_name) + + # Make sure the directory for downloading the package exists. + download_path = os.path.join(chromeos_root, 'chroot/tmp', build_id) + package_path = os.path.join(download_path, package_file_name) + if not os.path.exists(download_path): + os.makedirs(download_path) + + # Check to see if the package file has already been downloaded. If not, + # download it. + if not os.path.exists(package_path): + command = 'gsutil cp %s /tmp/%s' % (gs_package_name, build_id) + + if self.log_level != 'verbose': + self._logger.LogOutput('CMD: %s' % command) + status = self._ce.ChrootRunCommand(chromeos_root, command) + if status != 0 or not os.path.exists(package_path): + raise MissingFile('Cannot download package: %s .' % package_path) + + def UncompressSingleAutotestFile(self, chromeos_root, build_id, + package_file_name, uncompress_cmd): + # Uncompress file + command = ('cd /tmp/%s ; %s %s' % + (build_id, uncompress_cmd, package_file_name)) + + if self.log_level != 'verbose': + self._logger.LogOutput('CMD: %s' % command) + print('(Uncompressing autotest file %s .)' % package_file_name) + retval = self._ce.ChrootRunCommand(chromeos_root, command) + if retval != 0: + raise MissingFile('Cannot uncompress file: %s.' % package_file_name) + # Remove uncompressed downloaded file + command = ('cd /tmp/%s ; rm -f %s' % (build_id, package_file_name)) + if self.log_level != 'verbose': + self._logger.LogOutput('CMD: %s' % command) + print('(Removing processed autotest file %s .)' % package_file_name) + # try removing file, its ok to have an error, print if encountered + retval = self._ce.ChrootRunCommand(chromeos_root, command) + if retval != 0: + print('(Warning: Could not remove file %s .)' % package_file_name) + + def DownloadAutotestFiles(self, chromeos_root, build_id): + # Download autest package files (3 files) + autotest_packages_name = ('autotest_packages.tar') + autotest_server_package_name = ('autotest_server_package.tar.bz2') + autotest_control_files_name = ('control_files.tar') + + # Autotest directory relative path wrt chroot + autotest_rel_path = os.path.join('/tmp', build_id, 'autotest_files') + # Absolute Path to download files + autotest_path = os.path.join(chromeos_root, 'chroot/tmp', build_id, + 'autotest_files') + + if not os.path.exists(autotest_path): + self.DownloadSingleAutotestFile(chromeos_root, build_id, + autotest_packages_name) + self.DownloadSingleAutotestFile(chromeos_root, build_id, + autotest_server_package_name) + self.DownloadSingleAutotestFile(chromeos_root, build_id, + autotest_control_files_name) + + self.UncompressSingleAutotestFile(chromeos_root, build_id, + autotest_packages_name, 'tar -xvf ') + self.UncompressSingleAutotestFile(chromeos_root, build_id, + autotest_server_package_name, + 'tar -jxvf ') + self.UncompressSingleAutotestFile(chromeos_root, build_id, + autotest_control_files_name, + 'tar -xvf ') + # Rename created autotest directory to autotest_files + command = ('cd /tmp/%s ; mv autotest autotest_files' % build_id) + if self.log_level != 'verbose': + self._logger.LogOutput('CMD: %s' % command) + print('(Moving downloaded autotest files to autotest_files)') + retval = self._ce.ChrootRunCommand(chromeos_root, command) + if retval != 0: + raise MissingFile('Could not create directory autotest_files') + + return autotest_rel_path + + def Run(self, chromeos_root, xbuddy_label, autotest_path): build_id = self.GetBuildID(chromeos_root, xbuddy_label) image_name = ('gs://chromeos-image-archive/%s/chromiumos_test_image.tar.xz' % build_id) @@ -105,4 +204,7 @@ class ImageDownloader(object): if self.log_level != 'quiet': self._logger.LogOutput('Using image from %s.' % image_path) - return image_path + if autotest_path == '': + autotest_path = self.DownloadAutotestFiles(chromeos_root, build_id) + + return image_path, autotest_path diff --git a/crosperf/download_images_unittest.py b/crosperf/download_images_unittest.py index 06ea5bb7..fbf2ec79 100755 --- a/crosperf/download_images_unittest.py +++ b/crosperf/download_images_unittest.py @@ -137,11 +137,14 @@ class ImageDownloaderTestcast(unittest.TestCase): # Set test arguments test_chroot = '/usr/local/home/chromeos' test_build_id = 'remote/lumpy/latest-dev' + test_empty_autotest_path = '' + test_autotest_path = '/tmp/autotest' # Set values to test/check. self.called_download_image = False self.called_uncompress_image = False self.called_get_build_id = False + self.called_download_autotest_files = False # Define fake stub functions for Run to call def FakeGetBuildID(unused_root, unused_xbuddy_label): @@ -166,6 +169,12 @@ class ImageDownloaderTestcast(unittest.TestCase): self.called_uncompress_image = True return 0 + def FakeDownloadAutotestFiles(root, build_id): + if root or build_id: + pass + self.called_download_autotest_files = True + return 'autotest' + # Initialize downloader downloader = download_images.ImageDownloader(logger_to_use=MOCK_LOGGER) @@ -173,26 +182,45 @@ class ImageDownloaderTestcast(unittest.TestCase): downloader.GetBuildID = FakeGetBuildID downloader.UncompressImage = FakeUncompressImage downloader.DownloadImage = GoodDownloadImage + downloader.DownloadAutotestFiles = FakeDownloadAutotestFiles # Call Run. - downloader.Run(test_chroot, test_build_id) + image_path, autotest_path = downloader.Run(test_chroot, test_build_id, + test_empty_autotest_path) # Make sure it called both _DownloadImage and _UncompressImage self.assertTrue(self.called_download_image) self.assertTrue(self.called_uncompress_image) + # Make sure it called DownloadAutotestFiles + self.assertTrue(self.called_download_autotest_files) + # Make sure it returned a autotest path returned from this call + self.assertTrue(autotest_path == 'autotest') + + # Call Run with a non-empty autotest path + self.called_download_autotest_files = False + + image_path, autotest_path = downloader.Run(test_chroot, test_build_id, + test_autotest_path) + + # Verify that downloadAutotestFiles was not called + self.assertFalse(self.called_download_autotest_files) + # Make sure it returned the specified autotest path returned from this call + self.assertTrue(autotest_path == test_autotest_path) # Reset values; Now use fake stub that simulates DownloadImage failing. self.called_download_image = False self.called_uncompress_image = False + self.called_download_autotest_files = False downloader.DownloadImage = BadDownloadImage # Call Run again. self.assertRaises(download_images.MissingImage, downloader.Run, test_chroot, - test_build_id) + test_autotest_path, test_build_id) - # Verify that UncompressImage was not called, since _DownloadImage "failed" + # Verify that UncompressImage and downloadAutotestFiles was not called, since _DownloadImage "failed" self.assertTrue(self.called_download_image) self.assertFalse(self.called_uncompress_image) + self.assertFalse(self.called_download_autotest_files) if __name__ == '__main__': diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py index a6807285..0fdaea03 100644 --- a/crosperf/experiment_factory.py +++ b/crosperf/experiment_factory.py @@ -208,6 +208,7 @@ class ExperimentFactory(object): for label_settings in all_label_settings: label_name = label_settings.name image = label_settings.GetField('chromeos_image') + autotest_path = label_settings.GetField('autotest_path') chromeos_root = label_settings.GetField('chromeos_root') my_remote = label_settings.GetField('remote') compiler = label_settings.GetField('compiler') @@ -221,8 +222,11 @@ class ExperimentFactory(object): build = label_settings.GetField('build') if len(build) == 0: raise RuntimeError("Can not have empty 'build' field!") - image = label_settings.GetXbuddyPath(build, board, chromeos_root, - log_level) + image, autotest_path = label_settings.GetXbuddyPath(build, + autotest_path, + board, + chromeos_root, + log_level) cache_dir = label_settings.GetField('cache_dir') chrome_src = label_settings.GetField('chrome_src') @@ -240,13 +244,13 @@ class ExperimentFactory(object): image_args = label_settings.GetField('image_args') if test_flag.GetTestMode(): # pylint: disable=too-many-function-args - label = MockLabel(label_name, image, chromeos_root, board, my_remote, - image_args, cache_dir, cache_only, log_level, - compiler, chrome_src) + label = MockLabel(label_name, image, autotest_path, chromeos_root, + board, my_remote, image_args, cache_dir, cache_only, + log_level, compiler, chrome_src) else: - label = Label(label_name, image, chromeos_root, board, my_remote, - image_args, cache_dir, cache_only, log_level, compiler, - chrome_src) + label = Label(label_name, image, autotest_path, chromeos_root, board, + my_remote, image_args, cache_dir, cache_only, log_level, + compiler, chrome_src) labels.append(label) if not labels: diff --git a/crosperf/experiment_factory_unittest.py b/crosperf/experiment_factory_unittest.py index b4bbbfa4..02bfd0a1 100755 --- a/crosperf/experiment_factory_unittest.py +++ b/crosperf/experiment_factory_unittest.py @@ -109,10 +109,13 @@ class ExperimentFactoryTest(unittest.TestCase): return [] return ['fake_chromeos_machine1.cros', 'fake_chromeos_machine2.cros'] - def FakeGetXbuddyPath(build, board, chroot, log_level): + def FakeGetXbuddyPath(build, autotest_dir, board, chroot, log_level): + autotest_path = autotest_dir + if not autotest_path: + autotest_path = 'fake_autotest_path' if not build or not board or not chroot or not log_level: - return '' - return 'fake_image_path' + return '', autotest_path + return 'fake_image_path', autotest_path ef = ExperimentFactory() ef.AppendBenchmarkSet = FakeAppendBenchmarkSet @@ -139,6 +142,7 @@ class ExperimentFactoryTest(unittest.TestCase): 'chromeos_image', 'chromeos/src/build/images/lumpy/latest/chromiumos_test_image.bin') label_settings.SetField('chrome_src', '/usr/local/google/home/chrome-top') + label_settings.SetField('autotest_path', '/tmp/autotest') mock_experiment_file.global_settings = global_settings mock_experiment_file.all_settings.append(label_settings) @@ -164,6 +168,7 @@ class ExperimentFactoryTest(unittest.TestCase): self.assertEqual(exp.labels[0].chromeos_image, 'chromeos/src/build/images/lumpy/latest/' 'chromiumos_test_image.bin') + self.assertEqual(exp.labels[0].autotest_path, '/tmp/autotest') self.assertEqual(exp.labels[0].board, 'lumpy') # Second test: Remotes listed in labels. @@ -200,12 +205,14 @@ class ExperimentFactoryTest(unittest.TestCase): label_settings_2 = settings_factory.LabelSettings('official_image_label') label_settings_2.SetField('chromeos_root', 'chromeos') label_settings_2.SetField('build', 'official-dev') + label_settings_2.SetField('autotest_path', '') label_settings_2.GetXbuddyPath = FakeGetXbuddyPath mock_experiment_file.all_settings.append(label_settings_2) exp = ef.GetExperiment(mock_experiment_file, '', '') self.assertEqual(len(exp.labels), 2) self.assertEqual(exp.labels[1].chromeos_image, 'fake_image_path') + self.assertEqual(exp.labels[1].autotest_path, 'fake_autotest_path') self.assertEqual( exp.remote, ['fake_chromeos_machine1.cros', 'fake_chromeos_machine2.cros']) diff --git a/crosperf/experiment_file.py b/crosperf/experiment_file.py index fe22bec8..016e9d86 100644 --- a/crosperf/experiment_file.py +++ b/crosperf/experiment_file.py @@ -156,9 +156,19 @@ class ExperimentFile(object): if chromeos_root_field: chromeos_root = chromeos_root_field.GetString() value = field.GetString() - xbuddy_path = settings.GetXbuddyPath(value, board, chromeos_root, - 'quiet') - res += '\t#actual_image: %s\n' % xbuddy_path + autotest_field = settings.fields['autotest_path'] + autotest_path = '' + if autotest_field.assigned: + autotest_path = autotest_field.GetString() + image_path, autotest_path = settings.GetXbuddyPath(value, + autotest_path, + board, + chromeos_root, + 'quiet') + res += '\t#actual_image: %s\n' % image_path + if not autotest_field.assigned: + res += '\t#actual_autotest_path: %s\n' % autotest_path + res += '}\n\n' return res diff --git a/crosperf/label.py b/crosperf/label.py index 3326efa7..d993c15c 100644 --- a/crosperf/label.py +++ b/crosperf/label.py @@ -19,6 +19,7 @@ class Label(object): def __init__(self, name, chromeos_image, + autotest_path, chromeos_root, board, remote, @@ -38,6 +39,7 @@ class Label(object): self.name = name self.chromeos_image = chromeos_image + self.autotest_path = autotest_path self.board = board self.remote = remote self.image_args = image_args @@ -117,6 +119,7 @@ class MockLabel(object): def __init__(self, name, chromeos_image, + autotest_path, chromeos_root, board, remote, @@ -128,6 +131,7 @@ class MockLabel(object): chrome_src=None): self.name = name self.chromeos_image = chromeos_image + self.autotest_path = autotest_path self.board = board self.remote = remote self.cache_dir = cache_dir diff --git a/crosperf/machine_manager_unittest.py b/crosperf/machine_manager_unittest.py index b632336b..8652f171 100755 --- a/crosperf/machine_manager_unittest.py +++ b/crosperf/machine_manager_unittest.py @@ -48,12 +48,13 @@ class MyMachineManager(machine_manager.MachineManager): CHROMEOS_ROOT = '/tmp/chromeos-root' MACHINE_NAMES = ['lumpy1', 'lumpy2', 'lumpy3', 'daisy1', 'daisy2'] -LABEL_LUMPY = label.MockLabel('lumpy', 'lumpy_chromeos_image', CHROMEOS_ROOT, - 'lumpy', +LABEL_LUMPY = label.MockLabel('lumpy', 'lumpy_chromeos_image', 'autotest_dir', + CHROMEOS_ROOT, 'lumpy', ['lumpy1', 'lumpy2', 'lumpy3', 'lumpy4'], '', '', False, 'average,' 'gcc', None) -LABEL_MIX = label.MockLabel('mix', 'chromeos_image', CHROMEOS_ROOT, 'mix', +LABEL_MIX = label.MockLabel('mix', 'chromeos_image', 'autotest_dir', + CHROMEOS_ROOT, 'mix', ['daisy1', 'daisy2', 'lumpy3', 'lumpy4'], '', '', False, 'average', 'gcc', None) diff --git a/crosperf/mock_instance.py b/crosperf/mock_instance.py index 6092a981..758108fa 100644 --- a/crosperf/mock_instance.py +++ b/crosperf/mock_instance.py @@ -12,6 +12,7 @@ perf_args = 'record -a -e cycles' label1 = MockLabel( 'test1', 'image1', + 'autotest_dir', '/tmp/test_benchmark_run', 'x86-alex', 'chromeos-alex1', @@ -24,6 +25,7 @@ label1 = MockLabel( label2 = MockLabel( 'test2', 'image2', + 'autotest_dir', '/tmp/test_benchmark_run_2', 'x86-alex', 'chromeos-alex2', diff --git a/crosperf/results_cache_unittest.py b/crosperf/results_cache_unittest.py index 0d0dc990..e478667b 100755 --- a/crosperf/results_cache_unittest.py +++ b/crosperf/results_cache_unittest.py @@ -192,9 +192,9 @@ class ResultTest(unittest.TestCase): self.callGatherPerfResults = False self.mock_logger = mock.Mock(spec=logger.Logger) self.mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter) - self.mock_label = MockLabel('mock_label', 'chromeos_image', '/tmp', 'lumpy', - 'remote', 'image_args', 'cache_dir', 'average', - 'gcc', None) + self.mock_label = MockLabel('mock_label', 'chromeos_image', 'autotest_dir', + '/tmp', 'lumpy', 'remote', 'image_args', + 'cache_dir', 'average', 'gcc', None) def testCreateFromRun(self): result = MockResult.CreateFromRun(logger.GetLogger(), 'average', @@ -896,9 +896,9 @@ class TelemetryResultTest(unittest.TestCase): self.result = None self.mock_logger = mock.Mock(spec=logger.Logger) self.mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter) - self.mock_label = MockLabel('mock_label', 'chromeos_image', '/tmp', 'lumpy', - 'remote', 'image_args', 'cache_dir', 'average', - 'gcc', None) + self.mock_label = MockLabel('mock_label', 'chromeos_image', 'autotest_dir', + '/tmp', 'lumpy', 'remote', 'image_args', + 'cache_dir', 'average', 'gcc', None) self.mock_machine = machine_manager.MockCrosMachine('falco.cros', '/tmp/chromeos', 'average') @@ -940,9 +940,9 @@ class ResultsCacheTest(unittest.TestCase): super(ResultsCacheTest, self).__init__(*args, **kwargs) self.fakeCacheReturnResult = None self.mock_logger = mock.Mock(spec=logger.Logger) - self.mock_label = MockLabel('mock_label', 'chromeos_image', '/tmp', 'lumpy', - 'remote', 'image_args', 'cache_dir', 'average', - 'gcc', None) + self.mock_label = MockLabel('mock_label', 'chromeos_image', 'autotest_dir', + '/tmp', 'lumpy', 'remote', 'image_args', + 'cache_dir', 'average', 'gcc', None) def setUp(self): self.results_cache = ResultsCache() diff --git a/crosperf/settings.py b/crosperf/settings.py index 3182bf6c..8d5a25fd 100644 --- a/crosperf/settings.py +++ b/crosperf/settings.py @@ -66,7 +66,8 @@ class Settings(object): if not self.fields[name].assigned and self.fields[name].required: raise SyntaxError('Field %s is invalid.' % name) - def GetXbuddyPath(self, path_str, board, chromeos_root, log_level): + def GetXbuddyPath(self, path_str, autotest_path, board, chromeos_root, + log_level): prefix = 'remote' l = logger.GetLogger() if (path_str.find('trybot') < 0 and path_str.find('toolchain') < 0 and @@ -75,6 +76,6 @@ class Settings(object): else: xbuddy_path = '%s/%s' % (prefix, path_str) image_downloader = ImageDownloader(l, log_level) - image_path = image_downloader.Run( - misc.CanonicalizePath(chromeos_root), xbuddy_path) - return image_path + image_and_autotest_path = image_downloader.Run( + misc.CanonicalizePath(chromeos_root), xbuddy_path, autotest_path) + return image_and_autotest_path diff --git a/crosperf/settings_factory.py b/crosperf/settings_factory.py index 3bda6139..e42d82a9 100644 --- a/crosperf/settings_factory.py +++ b/crosperf/settings_factory.py @@ -63,6 +63,13 @@ class LabelSettings(Settings): description='The path to the image to run tests ' 'on, for local/custom-built images. See the ' "'build' option for official or trybot images.")) + self.AddField( + TextField( + 'autotest_path', + required=False, + description='Autotest directory path relative to chroot which ' + 'has autotest files for the image to run tests requiring autotest files' + )) self.AddField( TextField( 'chromeos_root', diff --git a/crosperf/settings_factory_unittest.py b/crosperf/settings_factory_unittest.py index c2b8980a..127bfd3c 100755 --- a/crosperf/settings_factory_unittest.py +++ b/crosperf/settings_factory_unittest.py @@ -29,8 +29,9 @@ class LabelSettingsTest(unittest.TestCase): def test_init(self): res = settings_factory.LabelSettings('l_settings') self.assertIsNotNone(res) - self.assertEqual(len(res.fields), 8) + self.assertEqual(len(res.fields), 9) self.assertEqual(res.GetField('chromeos_image'), '') + self.assertEqual(res.GetField('autotest_path'), '') self.assertEqual(res.GetField('chromeos_root'), '') self.assertEqual(res.GetField('remote'), None) self.assertEqual(res.GetField('image_args'), '') @@ -79,7 +80,7 @@ class SettingsFactoryTest(unittest.TestCase): l_settings = settings_factory.SettingsFactory().GetSettings('label', 'label') self.assertIsInstance(l_settings, settings_factory.LabelSettings) - self.assertEqual(len(l_settings.fields), 8) + self.assertEqual(len(l_settings.fields), 9) b_settings = settings_factory.SettingsFactory().GetSettings('benchmark', 'benchmark') diff --git a/crosperf/settings_unittest.py b/crosperf/settings_unittest.py index 9ea7a2f1..f1062f0d 100755 --- a/crosperf/settings_unittest.py +++ b/crosperf/settings_unittest.py @@ -39,20 +39,23 @@ class TestSettings(unittest.TestCase): def test_add_field(self): self.assertEqual(self.settings.fields, {}) - self.settings.AddField(IntegerField('iterations', - default=1, - required=False, - description='Number of iterations to ' - 'run the test.')) + self.settings.AddField( + IntegerField( + 'iterations', + default=1, + required=False, + description='Number of iterations to ' + 'run the test.')) self.assertEqual(len(self.settings.fields), 1) # Adding the same field twice raises an exception. - self.assertRaises(Exception, - self.settings.AddField, - (IntegerField('iterations', - default=1, - required=False, - description='Number of iterations to run ' - 'the test.'))) + self.assertRaises( + Exception, + self.settings.AddField, (IntegerField( + 'iterations', + default=1, + required=False, + description='Number of iterations to run ' + 'the test.'))) res = self.settings.fields['iterations'] self.assertIsInstance(res, IntegerField) self.assertEqual(res.Get(), 1) @@ -60,11 +63,12 @@ class TestSettings(unittest.TestCase): def test_set_field(self): self.assertEqual(self.settings.fields, {}) self.settings.AddField( - IntegerField('iterations', - default=1, - required=False, - description='Number of iterations to run the ' - 'test.')) + IntegerField( + 'iterations', + default=1, + required=False, + description='Number of iterations to run the ' + 'test.')) res = self.settings.fields['iterations'] self.assertEqual(res.Get(), 1) @@ -77,11 +81,12 @@ class TestSettings(unittest.TestCase): 'lumpy1.cros') self.settings.AddField( - ListField('remote', - default=[], - description="A comma-separated list of ip's of " - 'chromeos devices to run ' - 'experiments on.')) + ListField( + 'remote', + default=[], + description="A comma-separated list of ip's of " + 'chromeos devices to run ' + 'experiments on.')) self.assertEqual(type(self.settings.fields), dict) self.assertEqual(len(self.settings.fields), 2) res = self.settings.fields['remote'] @@ -96,10 +101,12 @@ class TestSettings(unittest.TestCase): self.assertRaises(Exception, self.settings.GetField, 'iterations') # Getting a required field that hasn't been assigned raises an exception. - self.settings.AddField(IntegerField('iterations', - required=True, - description='Number of iterations to ' - 'run the test.')) + self.settings.AddField( + IntegerField( + 'iterations', + required=True, + description='Number of iterations to ' + 'run the test.')) self.assertIsNotNone(self.settings.fields['iterations']) self.assertRaises(Exception, self.settings.GetField, 'iterations') @@ -125,11 +132,13 @@ class TestSettings(unittest.TestCase): self.assertEqual(label_settings.GetField('chromeos_root'), '/tmp/chromeos') def test_override(self): - self.settings.AddField(ListField('email', - default=[], - description='Space-seperated' - 'list of email addresses to send ' - 'email to.')) + self.settings.AddField( + ListField( + 'email', + default=[], + description='Space-seperated' + 'list of email addresses to send ' + 'email to.')) global_settings = settings_factory.SettingsFactory().GetSettings('global', 'global') @@ -146,21 +155,27 @@ class TestSettings(unittest.TestCase): def test_validate(self): - self.settings.AddField(IntegerField('iterations', - required=True, - description='Number of iterations ' - 'to run the test.')) - self.settings.AddField(ListField('remote', - default=[], - required=True, - description='A comma-separated list ' - "of ip's of chromeos " - 'devices to run experiments on.')) - self.settings.AddField(ListField('email', - default=[], - description='Space-seperated' - 'list of email addresses to ' - 'send email to.')) + self.settings.AddField( + IntegerField( + 'iterations', + required=True, + description='Number of iterations ' + 'to run the test.')) + self.settings.AddField( + ListField( + 'remote', + default=[], + required=True, + description='A comma-separated list ' + "of ip's of chromeos " + 'devices to run experiments on.')) + self.settings.AddField( + ListField( + 'email', + default=[], + description='Space-seperated' + 'list of email addresses to ' + 'send email to.')) # 'required' fields have not been assigned; should raise an exception. self.assertRaises(Exception, self.settings.Validate) @@ -183,24 +198,28 @@ class TestSettings(unittest.TestCase): trybot_str = 'trybot-lumpy-paladin/R34-5417.0.0-b1506' official_str = 'lumpy-release/R34-5417.0.0' xbuddy_str = 'latest-dev' + autotest_path = '' - self.settings.GetXbuddyPath(trybot_str, board, chromeos_root, log_level) + self.settings.GetXbuddyPath(trybot_str, autotest_path, board, chromeos_root, + log_level) self.assertEqual(mock_run.call_count, 1) self.assertEqual(mock_run.call_args_list[0][0], ('/tmp/chromeos', - 'remote/trybot-lumpy-paladin/R34-5417.0.0-b1506')) + 'remote/trybot-lumpy-paladin/R34-5417.0.0-b1506', '')) mock_run.reset_mock() - self.settings.GetXbuddyPath(official_str, board, chromeos_root, log_level) + self.settings.GetXbuddyPath(official_str, autotest_path, board, + chromeos_root, log_level) self.assertEqual(mock_run.call_count, 1) self.assertEqual(mock_run.call_args_list[0][0], - ('/tmp/chromeos', 'remote/lumpy-release/R34-5417.0.0')) + ('/tmp/chromeos', 'remote/lumpy-release/R34-5417.0.0', '')) mock_run.reset_mock() - self.settings.GetXbuddyPath(xbuddy_str, board, chromeos_root, log_level) + self.settings.GetXbuddyPath(xbuddy_str, autotest_path, board, chromeos_root, + log_level) self.assertEqual(mock_run.call_count, 1) - self.assertEqual(mock_run.call_args_list[0][0], ('/tmp/chromeos', - 'remote/lumpy/latest-dev')) + self.assertEqual(mock_run.call_args_list[0][0], + ('/tmp/chromeos', 'remote/lumpy/latest-dev', '')) if mock_logger: return diff --git a/crosperf/suite_runner.py b/crosperf/suite_runner.py index 9824a085..81c8bd5c 100644 --- a/crosperf/suite_runner.py +++ b/crosperf/suite_runner.py @@ -172,9 +172,13 @@ class SuiteRunner(object): # Rebooting between iterations has proven to help with this. self.RebootMachine(machine, label.chromeos_root) + autotest_dir = '~/trunk/src/third_party/autotest/files' + if label.autotest_path != '': + autotest_dir = label.autotest_path command = ( - ('%s --autotest_dir ~/trunk/src/third_party/autotest/files --fast ' - '%s %s %s') % (TEST_THAT_PATH, options, machine, benchmark.test_name)) + ('%s --autotest_dir %s --fast ' + '%s %s %s') % + (TEST_THAT_PATH, autotest_dir, options, machine, benchmark.test_name)) if self.log_level != 'verbose': self._logger.LogOutput('Running test.') self._logger.LogOutput('CMD: %s' % command) @@ -207,6 +211,8 @@ class SuiteRunner(object): # For telemetry runs, we can use the autotest copy from the source # location. No need to have one under /build/. autotest_dir_arg = '--autotest_dir ~/trunk/src/third_party/autotest/files' + if label.autotest_path != '': + autotest_dir_arg = '--autotest_dir %s' % label.autotest_path profiler_args = GetProfilerArgs(profiler_args) fast_arg = '' diff --git a/crosperf/suite_runner_unittest.py b/crosperf/suite_runner_unittest.py index 0b184934..3c36f3c2 100755 --- a/crosperf/suite_runner_unittest.py +++ b/crosperf/suite_runner_unittest.py @@ -28,9 +28,10 @@ class SuiteRunnerTest(unittest.TestCase): mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter) mock_cmd_term = mock.Mock(spec=command_executer.CommandTerminator) mock_logger = mock.Mock(spec=logger.Logger) - mock_label = label.MockLabel('lumpy', 'lumpy_chromeos_image', '/tmp/chromeos', - 'lumpy', ['lumpy1.cros', 'lumpy.cros2'], '', '', - False, 'average', 'gcc', '') + mock_label = label.MockLabel('lumpy', 'lumpy_chromeos_image', '', + '/tmp/chromeos', 'lumpy', + ['lumpy1.cros', 'lumpy.cros2'], '', '', False, + 'average', 'gcc', '') telemetry_crosperf_bench = Benchmark( 'b1_test', # name 'octane', # test_name -- cgit v1.2.3