diff options
Diffstat (limited to 'cli/cros')
-rw-r--r-- | cli/cros/cros_build.py | 40 | ||||
-rw-r--r-- | cli/cros/cros_chrome_sdk.py | 6 | ||||
-rw-r--r-- | cli/cros/cros_payload.py | 78 | ||||
-rw-r--r-- | cli/cros/cros_payload_unittest.py | 165 | ||||
-rw-r--r-- | cli/cros/lint.py | 23 | ||||
-rw-r--r-- | cli/cros/lint_unittest.py | 25 |
6 files changed, 247 insertions, 90 deletions
diff --git a/cli/cros/cros_build.py b/cli/cros/cros_build.py index b868c8f13..0709288f5 100644 --- a/cli/cros/cros_build.py +++ b/cli/cros/cros_build.py @@ -7,15 +7,12 @@ from __future__ import print_function from chromite.cli import command -from chromite.lib import blueprint_lib -from chromite.lib import brick_lib from chromite.lib import chroot_util from chromite.lib import commandline from chromite.lib import cros_build_lib from chromite.lib import cros_logging as logging from chromite.lib import operation from chromite.lib import parallel -from chromite.lib import toolchain from chromite.lib import workon_helper @@ -51,40 +48,23 @@ To just build a single package: self.host = False self.board = None self.brick = None - self.blueprint = None if self.options.host: self.host = True elif self.options.board: self.board = self.options.board - elif self.options.blueprint: - self.blueprint = blueprint_lib.Blueprint(self.options.blueprint) - - if not self.build_pkgs: - self.build_pkgs = self.blueprint.GetPackages() - elif self.options.brick or self.curr_brick_locator: - self.brick = brick_lib.Brick(self.options.brick - or self.curr_brick_locator) - self.board = self.brick.FriendlyName() - if not self.build_pkgs: - self.build_pkgs = self.brick.MainPackages() else: # If nothing is explicitly set, use the default board. self.board = cros_build_lib.GetDefaultBoard() # Set sysroot and friendly name. The latter is None if building for host. - self.sysroot = cros_build_lib.GetSysroot(self.blueprint.FriendlyName() - if self.blueprint else self.board) + self.sysroot = cros_build_lib.GetSysroot(self.board) @classmethod def AddParser(cls, parser): super(cls, BuildCommand).AddParser(parser) target = parser.add_mutually_exclusive_group() target.add_argument('--board', help='The board to build packages for.') - target.add_argument('--brick', type='brick_path', - help='The brick to build packages for.') - target.add_argument('--blueprint', type='blueprint_path', - help='The blueprint to build packages for.') target.add_argument('--host', help='Build packages for the chroot itself.', default=False, action='store_true') parser.add_argument('--no-binary', help="Don't use binary packages.", @@ -134,7 +114,7 @@ To just build a single package: Only print the output if this step fails or if we're in debug mode. """ - if self.options.deps and not self.host and not self.blueprint: + if self.options.deps and not self.host: cmd = chroot_util.GetEmergeCommand(sysroot=self.sysroot) cmd += ['-pe', '--backtrack=0'] + self.build_pkgs try: @@ -160,7 +140,7 @@ To just build a single package: self.options.Freeze() if not self.host: - if not (self.board or self.brick or self.blueprint): + if not (self.board or self.brick): cros_build_lib.Die('You did not specify a board/brick to build for. ' 'You need to be in a brick directory or set ' '--board/--brick/--host') @@ -169,11 +149,7 @@ To just build a single package: cros_build_lib.Die('--brick should not be used with board names. Use ' '--board=%s instead.' % self.brick.config['name']) - if self.blueprint: - chroot_args = ['--toolchains', - ','.join(toolchain.GetToolchainsForBrick( - self.blueprint.GetBSP()).iterkeys())] - elif self.board: + if self.board: chroot_args = ['--board', self.board] else: chroot_args = None @@ -184,13 +160,7 @@ To just build a single package: cros_build_lib.Die('No packages found, nothing to build.') # Set up the sysroots if not building for host. - if self.blueprint: - if self.chroot_update: - chroot_util.UpdateChroot( - update_host_packages=self.options.host_packages_update, - brick=brick_lib.Brick(self.blueprint.GetBSP())) - chroot_util.InitializeSysroots(self.blueprint) - elif self.brick or self.board: + if self.brick or self.board: chroot_util.SetupBoard( brick=self.brick, board=self.board, update_chroot=self.chroot_update, diff --git a/cli/cros/cros_chrome_sdk.py b/cli/cros/cros_chrome_sdk.py index 73ac533a3..116a94196 100644 --- a/cli/cros/cros_chrome_sdk.py +++ b/cli/cros/cros_chrome_sdk.py @@ -23,6 +23,7 @@ from chromite.lib import gs from chromite.lib import osutils from chromite.lib import path_util from chromite.lib import stats +from chromite.cbuildbot import archive_lib from chromite.cbuildbot import config_lib from chromite.cbuildbot import constants @@ -91,7 +92,7 @@ class SDKFetcher(object): force usage of the external configuration if both external and internal are available. """ - site_config = config_lib.LoadConfigFromFile() + site_config = config_lib.GetConfig() self.cache_base = os.path.join(cache_dir, COMMAND_NAME) if clear_cache: @@ -104,8 +105,7 @@ class SDKFetcher(object): self.board = board self.config = site_config.FindCanonicalConfigForBoard( board, allow_internal=not use_external_config) - self.gs_base = '%s/%s' % (constants.DEFAULT_ARCHIVE_BUCKET, - self.config['name']) + self.gs_base = archive_lib.GetBaseUploadURI(self.config) self.clear_cache = clear_cache self.chrome_src = chrome_src self.sdk_path = sdk_path diff --git a/cli/cros/cros_payload.py b/cli/cros/cros_payload.py index ebac811ab..9581b597d 100644 --- a/cli/cros/cros_payload.py +++ b/cli/cros/cros_payload.py @@ -8,7 +8,6 @@ from __future__ import print_function import itertools import os -import string import sys import textwrap @@ -18,6 +17,8 @@ from chromite.cli import command # Needed for the dev.host.lib import below. sys.path.insert(0, os.path.join(constants.SOURCE_ROOT, 'src', 'platform')) +MAJOR_PAYLOAD_VERSION_CHROMEOS = 1 +MAJOR_PAYLOAD_VERSION_BRILLO = 2 def DisplayValue(key, value): """Print out a key, value pair with values left-aligned.""" @@ -35,7 +36,7 @@ def DisplayHexData(data, indent=0): ' '.join('%.2x' % ord(c) for c in chunk) + ' ' * (16 - len(chunk)) + ' | ' + - ''.join(c if c in string.printable else '.' for c in chunk)) + ''.join(c if 32 <= ord(c) and ord(c) < 127 else '.' for c in chunk)) @command.CommandDecorator('payload') @@ -85,30 +86,52 @@ Example: def _DisplayManifest(self): """Show information from the payload manifest.""" manifest = self.payload.manifest - DisplayValue('Number of operations', len(manifest.install_operations)) - DisplayValue('Number of kernel ops', - len(manifest.kernel_install_operations)) + if self.payload.header.version == MAJOR_PAYLOAD_VERSION_BRILLO: + DisplayValue('Number of partitions', len(manifest.partitions)) + for partition in manifest.partitions: + DisplayValue(' Number of "%s" ops' % partition.partition_name, + len(partition.operations)) + else: + DisplayValue('Number of operations', len(manifest.install_operations)) + DisplayValue('Number of kernel ops', + len(manifest.kernel_install_operations)) DisplayValue('Block size', manifest.block_size) DisplayValue('Minor version', manifest.minor_version) def _DisplaySignatures(self): """Show information about the signatures from the manifest.""" - manifest = self.payload.manifest - if not manifest.HasField('signatures_offset'): - print('No signatures stored in the payload') - return - - signature_msg = 'offset=%d' % manifest.signatures_offset - if manifest.signatures_size: - signature_msg += ' (%d bytes)' % manifest.signatures_size - DisplayValue('Signature blob', signature_msg) - signatures_blob = self.payload.ReadDataBlob(manifest.signatures_offset, - manifest.signatures_size) + header = self.payload.header + if header.metadata_signature_len: + offset = header.size + header.manifest_len + DisplayValue('Metadata signatures blob', + 'file_offset=%d (%d bytes)' % + (offset, header.metadata_signature_len)) + signatures_blob = self.payload.ReadDataBlob( + -header.metadata_signature_len, + header.metadata_signature_len) + self._DisplaySignaturesBlob('Metadata', signatures_blob) + else: + print('No metadata signatures stored in the payload') + manifest = self.payload.manifest + if manifest.HasField('signatures_offset'): + signature_msg = 'blob_offset=%d' % manifest.signatures_offset + if manifest.signatures_size: + signature_msg += ' (%d bytes)' % manifest.signatures_size + DisplayValue('Payload signatures blob', signature_msg) + signatures_blob = self.payload.ReadDataBlob(manifest.signatures_offset, + manifest.signatures_size) + self._DisplaySignaturesBlob('Payload', signatures_blob) + else: + print('No payload signatures stored in the payload') + + @staticmethod + def _DisplaySignaturesBlob(signature_name, signatures_blob): from dev.host.lib.update_payload import update_metadata_pb2 signatures = update_metadata_pb2.Signatures() signatures.ParseFromString(signatures_blob) - print('Payload signatures: (%d entries)' % len(signatures.signatures)) + print('%s signatures: (%d entries)' % + (signature_name, len(signatures.signatures))) for signature in signatures.signatures: print(' version=%s, hex_data: (%d bytes)' % (signature.version if signature.HasField('version') else None, @@ -162,8 +185,12 @@ Example: read_blocks = 0 written_blocks = 0 num_write_seeks = 0 - for operations in (manifest.install_operations, - manifest.kernel_install_operations): + if self.payload.header.version == MAJOR_PAYLOAD_VERSION_BRILLO: + partitions_operations = [part.operations for part in manifest.partitions] + else: + partitions_operations = [manifest.install_operations, + manifest.kernel_install_operations] + for operations in partitions_operations: last_ext = None for curr_op in operations: read_blocks += sum([ext.num_blocks for ext in curr_op.src_extents]) @@ -207,7 +234,12 @@ Example: self._DisplayStats(self.payload.manifest) if self.options.list_ops: print() - self._DisplayOps('Install operations', - self.payload.manifest.install_operations) - self._DisplayOps('Kernel install operations', - self.payload.manifest.kernel_install_operations) + if self.payload.header.version == MAJOR_PAYLOAD_VERSION_BRILLO: + for partition in self.payload.manifest.partitions: + self._DisplayOps('%s install operations' % partition.partition_name, + partition.operations) + else: + self._DisplayOps('Install operations', + self.payload.manifest.install_operations) + self._DisplayOps('Kernel install operations', + self.payload.manifest.kernel_install_operations) diff --git a/cli/cros/cros_payload_unittest.py b/cli/cros/cros_payload_unittest.py index d99f0e9a1..36f4749bd 100644 --- a/cli/cros/cros_payload_unittest.py +++ b/cli/cros/cros_payload_unittest.py @@ -50,10 +50,17 @@ class FakeOp(object): def HasField(self, field): return hasattr(self, field) +class FakePartition(object): + """Fake PartitionUpdate field for testing.""" + + def __init__(self, partition_name, operations): + self.partition_name = partition_name + self.operations = operations + class FakeManifest(object): """Fake manifest for testing.""" - def __init__(self): + def __init__(self, major_version): FakeExtent = collections.namedtuple('FakeExtent', ['start_block', 'num_blocks']) self.install_operations = [FakeOp([], @@ -67,6 +74,11 @@ class FakeManifest(object): [FakeExtent(x, x) for x in xrange(20)], update_payload.common.OpType.SOURCE_COPY, src_length=4096)] + if major_version == cros_payload.MAJOR_PAYLOAD_VERSION_BRILLO: + self.partitions = [FakePartition('rootfs', self.install_operations), + FakePartition('kernel', + self.kernel_install_operations)] + self.install_operations = self.kernel_install_operations = [] self.block_size = 4096 self.minor_version = 4 FakePartInfo = collections.namedtuple('FakePartInfo', ['size']) @@ -81,19 +93,32 @@ class FakeManifest(object): """Fake HasField method based on the python members.""" return hasattr(self, field_name) and getattr(self, field_name) is not None +class FakeHeader(object): + """Fake payload header for testing.""" + + def __init__(self, version, manifest_len, metadata_signature_len): + self.version = version + self.manifest_len = manifest_len + self.metadata_signature_len = metadata_signature_len + + @property + def size(self): + return (20 if self.version == cros_payload.MAJOR_PAYLOAD_VERSION_CHROMEOS + else 24) + + class FakePayload(object): """Fake payload for testing.""" - def __init__(self): - FakeHeader = collections.namedtuple('FakeHeader', - ['version', 'manifest_len']) - self._header = FakeHeader('111', 222) + def __init__(self, major_version): + self._header = FakeHeader(major_version, 222, 0) self.header = None - self._manifest = FakeManifest() + self._manifest = FakeManifest(major_version) self.manifest = None self._blobs = {} - self._signatures = update_metadata_pb2.Signatures() + self._payload_signatures = update_metadata_pb2.Signatures() + self._metadata_signatures = update_metadata_pb2.Signatures() def Init(self): """Fake Init that sets header and manifest. @@ -114,15 +139,28 @@ class FakePayload(object): 'actual: %d)' % (len(blob), length)) return blob - def AddSignature(self, **kwargs): - new_signature = self._signatures.signatures.add() + @staticmethod + def _AddSignatureToProto(proto, **kwargs): + """Add a new Signature element to the passed proto.""" + new_signature = proto.signatures.add() for key, val in kwargs.iteritems(): setattr(new_signature, key, val) - blob = self._signatures.SerializeToString() + + def AddPayloadSignature(self, **kwargs): + self._AddSignatureToProto(self._payload_signatures, **kwargs) + blob = self._payload_signatures.SerializeToString() self._manifest.signatures_offset = 1234 self._manifest.signatures_size = len(blob) self._blobs[self._manifest.signatures_offset] = blob + def AddMetadataSignature(self, **kwargs): + self._AddSignatureToProto(self._metadata_signatures, **kwargs) + if self._header.metadata_signature_len: + del self._blobs[-self._header.metadata_signature_len] + blob = self._metadata_signatures.SerializeToString() + self._header.metadata_signature_len = len(blob) + self._blobs[-len(blob)] = blob + class PayloadCommandTest(cros_test_lib.MockOutputTestCase): """Test class for our PayloadCommand class.""" @@ -137,13 +175,14 @@ class PayloadCommandTest(cros_test_lib.MockOutputTestCase): def testRun(self): """Verify that Run parses and displays the payload like we expect.""" payload_cmd = cros_payload.PayloadCommand(FakeOption(action='show')) - self.PatchObject(update_payload, 'Payload', return_value=FakePayload()) + self.PatchObject(update_payload, 'Payload', return_value=FakePayload( + cros_payload.MAJOR_PAYLOAD_VERSION_CHROMEOS)) with self.OutputCapturer() as output: payload_cmd.Run() stdout = output.GetStdout() - expected_out = """Payload version: 111 + expected_out = """Payload version: 1 Manifest length: 222 Number of operations: 1 Number of kernel ops: 1 @@ -152,17 +191,18 @@ Minor version: 4 """ self.assertEquals(stdout, expected_out) - def testListOps(self): + def testListOpsOnVersion1(self): """Verify that the --list_ops option gives the correct output.""" payload_cmd = cros_payload.PayloadCommand(FakeOption(list_ops=True, action='show')) - self.PatchObject(update_payload, 'Payload', return_value=FakePayload()) + self.PatchObject(update_payload, 'Payload', return_value=FakePayload( + cros_payload.MAJOR_PAYLOAD_VERSION_CHROMEOS)) with self.OutputCapturer() as output: payload_cmd.Run() stdout = output.GetStdout() - expected_out = """Payload version: 111 + expected_out = """Payload version: 1 Manifest length: 222 Number of operations: 1 Number of kernel ops: 1 @@ -185,17 +225,53 @@ Kernel install operations: """ self.assertEquals(stdout, expected_out) - def testStats(self): + def testListOpsOnVersion2(self): + """Verify that the --list_ops option gives the correct output.""" + payload_cmd = cros_payload.PayloadCommand(FakeOption(list_ops=True, + action='show')) + self.PatchObject(update_payload, 'Payload', return_value=FakePayload( + cros_payload.MAJOR_PAYLOAD_VERSION_BRILLO)) + + with self.OutputCapturer() as output: + payload_cmd.Run() + + stdout = output.GetStdout() + expected_out = """Payload version: 2 +Manifest length: 222 +Number of partitions: 2 + Number of "rootfs" ops: 1 + Number of "kernel" ops: 1 +Block size: 4096 +Minor version: 4 + +rootfs install operations: + 0: REPLACE_BZ + Data offset: 1 + Data length: 1 + Destination: 2 extents (3 blocks) + (1,1) (2,2) +kernel install operations: + 0: SOURCE_COPY + Source: 1 extent (1 block) + (1,1) + Destination: 20 extents (190 blocks) + (0,0) (1,1) (2,2) (3,3) (4,4) (5,5) (6,6) (7,7) (8,8) (9,9) (10,10) + (11,11) (12,12) (13,13) (14,14) (15,15) (16,16) (17,17) (18,18) (19,19) +""" + self.assertEquals(stdout, expected_out) + + def testStatsOnVersion1(self): """Verify that the --stats option works correctly.""" payload_cmd = cros_payload.PayloadCommand(FakeOption(stats=True, action='show')) - self.PatchObject(update_payload, 'Payload', return_value=FakePayload()) + self.PatchObject(update_payload, 'Payload', return_value=FakePayload( + cros_payload.MAJOR_PAYLOAD_VERSION_CHROMEOS)) with self.OutputCapturer() as output: payload_cmd.Run() stdout = output.GetStdout() - expected_out = """Payload version: 111 + expected_out = """Payload version: 1 Manifest length: 222 Number of operations: 1 Number of kernel ops: 1 @@ -207,23 +283,49 @@ Seeks when writing: 18 """ self.assertEquals(stdout, expected_out) + def testStatsOnVersion2(self): + """Verify that the --stats option works correctly on version 2.""" + payload_cmd = cros_payload.PayloadCommand(FakeOption(stats=True, + action='show')) + self.PatchObject(update_payload, 'Payload', return_value=FakePayload( + cros_payload.MAJOR_PAYLOAD_VERSION_BRILLO)) + + with self.OutputCapturer() as output: + payload_cmd.Run() + + stdout = output.GetStdout() + expected_out = """Payload version: 2 +Manifest length: 222 +Number of partitions: 2 + Number of "rootfs" ops: 1 + Number of "kernel" ops: 1 +Block size: 4096 +Minor version: 4 +Blocks read: 11 +Blocks written: 193 +Seeks when writing: 18 +""" + self.assertEquals(stdout, expected_out) + def testEmptySignatures(self): """Verify that the --signatures option works with unsigned payloads.""" payload_cmd = cros_payload.PayloadCommand( FakeOption(action='show', signatures=True)) - self.PatchObject(update_payload, 'Payload', return_value=FakePayload()) + self.PatchObject(update_payload, 'Payload', return_value=FakePayload( + cros_payload.MAJOR_PAYLOAD_VERSION_CHROMEOS)) with self.OutputCapturer() as output: payload_cmd.Run() stdout = output.GetStdout() - expected_out = """Payload version: 111 + expected_out = """Payload version: 1 Manifest length: 222 Number of operations: 1 Number of kernel ops: 1 Block size: 4096 Minor version: 4 -No signatures stored in the payload +No metadata signatures stored in the payload +No payload signatures stored in the payload """ self.assertEquals(stdout, expected_out) @@ -232,22 +334,29 @@ No signatures stored in the payload """Verify that the --signatures option shows the present signatures.""" payload_cmd = cros_payload.PayloadCommand( FakeOption(action='show', signatures=True)) - payload = FakePayload() - payload.AddSignature(version=1, data='12345678abcdefgh\x00\x01\x02\x03') - payload.AddSignature(data='I am a signature so access is yes.') + payload = FakePayload(cros_payload.MAJOR_PAYLOAD_VERSION_BRILLO) + payload.AddPayloadSignature(version=1, + data='12345678abcdefgh\x00\x01\x02\x03') + payload.AddPayloadSignature(data='I am a signature so access is yes.') + payload.AddMetadataSignature(data='\x00\x0a\x0c') self.PatchObject(update_payload, 'Payload', return_value=payload) with self.OutputCapturer() as output: payload_cmd.Run() stdout = output.GetStdout() - expected_out = """Payload version: 111 + expected_out = """Payload version: 2 Manifest length: 222 -Number of operations: 1 -Number of kernel ops: 1 +Number of partitions: 2 + Number of "rootfs" ops: 1 + Number of "kernel" ops: 1 Block size: 4096 Minor version: 4 -Signature blob: offset=1234 (64 bytes) +Metadata signatures blob: file_offset=246 (7 bytes) +Metadata signatures: (1 entries) + version=None, hex_data: (3 bytes) + 00 0a 0c | ... +Payload signatures blob: blob_offset=1234 (64 bytes) Payload signatures: (2 entries) version=1, hex_data: (20 bytes) 31 32 33 34 35 36 37 38 61 62 63 64 65 66 67 68 | 12345678abcdefgh diff --git a/cli/cros/lint.py b/cli/cros/lint.py index f54c8a6b9..36d07e8e1 100644 --- a/cli/cros/lint.py +++ b/cli/cros/lint.py @@ -385,6 +385,8 @@ class SourceChecker(BaseChecker): class _MessageR9200(object): pass class _MessageR9201(object): pass class _MessageR9202(object): pass + class _MessageR9203(object): pass + class _MessageR9210(object): pass # pylint: enable=class-missing-docstring,multiple-statements name = 'source_checker' @@ -397,6 +399,10 @@ class SourceChecker(BaseChecker): ('missing-shebang'), _MessageR9201), 'R9202': ('Shebang is set, but file is not executable', ('spurious-shebang'), _MessageR9202), + 'R9203': ('Unittest not named xxx_unittest.py', + ('unittest-misnamed'), _MessageR9203), + 'R9210': ('Trailing new lines found at end of file', + ('excess-trailing-newlines'), _MessageR9210), } options = () @@ -405,6 +411,8 @@ class SourceChecker(BaseChecker): stream = node.file_stream stream.seek(0) self._check_shebang(node, stream) + self._check_module_name(node) + self._check_trailing_lines(node, stream) def _check_shebang(self, _node, stream): """Verify the shebang is version specific""" @@ -424,6 +432,21 @@ class SourceChecker(BaseChecker): if parts[0] not in ('#!/usr/bin/python2', '#!/usr/bin/python3'): self.add_message('R9200') + def _check_module_name(self, node): + """Make sure the module name is sane""" + # Catch various typos. + name = node.name.rsplit('.', 2)[-1] + if name.rsplit('_', 2)[-1] in ('unittests',): + self.add_message('R9203') + + def _check_trailing_lines(self, _node, stream): + """Reject trailing lines""" + st = os.fstat(stream.fileno()) + if st.st_size > 1: + stream.seek(st.st_size - 2) + if not stream.read().strip('\n'): + self.add_message('R9210') + class ChromiteLoggingChecker(BaseChecker): """Make sure we enforce rules on importing logging.""" diff --git a/cli/cros/lint_unittest.py b/cli/cros/lint_unittest.py index b78c6553e..20b86c78f 100644 --- a/cli/cros/lint_unittest.py +++ b/cli/cros/lint_unittest.py @@ -23,7 +23,7 @@ class TestNode(object): Arg = collections.namedtuple('Arg', ('name',)) def __init__(self, doc='', fromlineno=0, path='foo.py', args=(), vararg='', - kwarg='', names=None, lineno=0): + kwarg='', names=None, lineno=0, name='module'): if names is None: names = [('name', None)] self.doc = doc @@ -34,6 +34,7 @@ class TestNode(object): self.args = self.Args(args=[self.Arg(name=x) for x in args], vararg=vararg, kwarg=kwarg) self.names = names + self.name = name def argnames(self): return self.args @@ -396,3 +397,25 @@ class SourceCheckerTest(CheckerTestCase): ) with open('/bin/sh') as f: self._testShebang(shebangs, 0, f.fileno()) + + def testGoodUnittestName(self): + """Verify _check_module_name accepts good unittest names""" + module_names = ( + 'lint_unittest', + ) + for name in module_names: + node = TestNode(name=name) + self.results = [] + self.checker._check_module_name(node) + self.assertEqual(len(self.results), 0) + + def testBadUnittestName(self): + """Verify _check_module_name accepts good unittest names""" + module_names = ( + 'lint_unittests', + ) + for name in module_names: + node = TestNode(name=name) + self.results = [] + self.checker._check_module_name(node) + self.assertEqual(len(self.results), 1) |