diff options
Diffstat (limited to 'generator/google/protobuf/text_format.py')
| -rw-r--r-- | generator/google/protobuf/text_format.py | 1359 |
1 files changed, 1055 insertions, 304 deletions
diff --git a/generator/google/protobuf/text_format.py b/generator/google/protobuf/text_format.py index 24dd07f..06b79d7 100644 --- a/generator/google/protobuf/text_format.py +++ b/generator/google/protobuf/text_format.py @@ -1,6 +1,6 @@ # Protocol Buffers - Google's data interchange format # Copyright 2008 Google Inc. All rights reserved. -# http://code.google.com/p/protobuf/ +# https://developers.google.com/protocol-buffers/ # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -28,20 +28,35 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"""Contains routines for printing protocol messages in text format.""" +"""Contains routines for printing protocol messages in text format. + +Simple usage example: + + # Create a proto object and serialize it to a text proto string. + message = my_proto_pb2.MyMessage(foo='bar') + text_proto = text_format.MessageToString(message) + + # Parse a text proto string. + message = text_format.Parse(text_proto, my_proto_pb2.MyMessage()) +""" __author__ = 'kenton@google.com (Kenton Varda)' -import cStringIO +import io import re -from collections import deque +import six + +if six.PY3: + long = int # pylint: disable=redefined-builtin,invalid-name + +# pylint: disable=g-import-not-at-top from google.protobuf.internal import type_checkers from google.protobuf import descriptor +from google.protobuf import text_encoding -__all__ = [ 'MessageToString', 'PrintMessage', 'PrintField', - 'PrintFieldValue', 'Merge' ] - +__all__ = ['MessageToString', 'PrintMessage', 'PrintField', 'PrintFieldValue', + 'Merge'] _INTEGER_CHECKERS = (type_checkers.Uint32ValueChecker(), type_checkers.Int32ValueChecker(), @@ -49,15 +64,99 @@ _INTEGER_CHECKERS = (type_checkers.Uint32ValueChecker(), type_checkers.Int64ValueChecker()) _FLOAT_INFINITY = re.compile('-?inf(?:inity)?f?', re.IGNORECASE) _FLOAT_NAN = re.compile('nanf?', re.IGNORECASE) +_FLOAT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_FLOAT, + descriptor.FieldDescriptor.CPPTYPE_DOUBLE]) +_QUOTES = frozenset(("'", '"')) +_ANY_FULL_TYPE_NAME = 'google.protobuf.Any' + + +class Error(Exception): + """Top-level module error for text_format.""" + + +class ParseError(Error): + """Thrown in case of text parsing or tokenizing error.""" + + def __init__(self, message=None, line=None, column=None): + if message is not None and line is not None: + loc = str(line) + if column is not None: + loc += ':{0}'.format(column) + message = '{0} : {1}'.format(loc, message) + if message is not None: + super(ParseError, self).__init__(message) + else: + super(ParseError, self).__init__() + self._line = line + self._column = column + + def GetLine(self): + return self._line + + def GetColumn(self): + return self._column + + +class TextWriter(object): + + def __init__(self, as_utf8): + if six.PY2: + self._writer = io.BytesIO() + else: + self._writer = io.StringIO() + def write(self, val): + if six.PY2: + if isinstance(val, six.text_type): + val = val.encode('utf-8') + return self._writer.write(val) -class ParseError(Exception): - """Thrown in case of ASCII parsing error.""" + def close(self): + return self._writer.close() + def getvalue(self): + return self._writer.getvalue() -def MessageToString(message, as_utf8=False, as_one_line=False): - out = cStringIO.StringIO() - PrintMessage(message, out, as_utf8=as_utf8, as_one_line=as_one_line) + +def MessageToString(message, + as_utf8=False, + as_one_line=False, + pointy_brackets=False, + use_index_order=False, + float_format=None, + use_field_number=False, + descriptor_pool=None, + indent=0): + """Convert protobuf message to text format. + + Floating point values can be formatted compactly with 15 digits of + precision (which is the most that IEEE 754 "double" can guarantee) + using float_format='.15g'. To ensure that converting to text and back to a + proto will result in an identical value, float_format='.17g' should be used. + + Args: + message: The protocol buffers message. + as_utf8: Produce text output in UTF8 format. + as_one_line: Don't introduce newlines between fields. + pointy_brackets: If True, use angle brackets instead of curly braces for + nesting. + use_index_order: If True, print fields of a proto message using the order + defined in source code instead of the field number. By default, use the + field number order. + float_format: If set, use this to specify floating point number formatting + (per the "Format Specification Mini-Language"); otherwise, str() is used. + use_field_number: If True, print field numbers instead of names. + descriptor_pool: A DescriptorPool used to resolve Any types. + indent: The indent level, in terms of spaces, for pretty print. + + Returns: + A string of the text formatted protocol buffer message. + """ + out = TextWriter(as_utf8) + printer = _Printer(out, indent, as_utf8, as_one_line, pointy_brackets, + use_index_order, float_format, use_field_number, + descriptor_pool) + printer.PrintMessage(message) result = out.getvalue() out.close() if as_one_line: @@ -65,152 +164,553 @@ def MessageToString(message, as_utf8=False, as_one_line=False): return result -def PrintMessage(message, out, indent=0, as_utf8=False, as_one_line=False): - for field, value in message.ListFields(): - if field.label == descriptor.FieldDescriptor.LABEL_REPEATED: - for element in value: - PrintField(field, element, out, indent, as_utf8, as_one_line) - else: - PrintField(field, value, out, indent, as_utf8, as_one_line) +def _IsMapEntry(field): + return (field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and + field.message_type.has_options and + field.message_type.GetOptions().map_entry) + + +def PrintMessage(message, + out, + indent=0, + as_utf8=False, + as_one_line=False, + pointy_brackets=False, + use_index_order=False, + float_format=None, + use_field_number=False, + descriptor_pool=None): + printer = _Printer(out, indent, as_utf8, as_one_line, pointy_brackets, + use_index_order, float_format, use_field_number, + descriptor_pool) + printer.PrintMessage(message) + + +def PrintField(field, + value, + out, + indent=0, + as_utf8=False, + as_one_line=False, + pointy_brackets=False, + use_index_order=False, + float_format=None): + """Print a single field name/value pair.""" + printer = _Printer(out, indent, as_utf8, as_one_line, pointy_brackets, + use_index_order, float_format) + printer.PrintField(field, value) + + +def PrintFieldValue(field, + value, + out, + indent=0, + as_utf8=False, + as_one_line=False, + pointy_brackets=False, + use_index_order=False, + float_format=None): + """Print a single field value (not including name).""" + printer = _Printer(out, indent, as_utf8, as_one_line, pointy_brackets, + use_index_order, float_format) + printer.PrintFieldValue(field, value) + + +def _BuildMessageFromTypeName(type_name, descriptor_pool): + """Returns a protobuf message instance. + Args: + type_name: Fully-qualified protobuf message type name string. + descriptor_pool: DescriptorPool instance. -def PrintField(field, value, out, indent=0, as_utf8=False, as_one_line=False): - """Print a single field name/value pair. For repeated fields, the value - should be a single element.""" + Returns: + A Message instance of type matching type_name, or None if the a Descriptor + wasn't found matching type_name. + """ + # pylint: disable=g-import-not-at-top + from google.protobuf import message_factory + factory = message_factory.MessageFactory(descriptor_pool) + try: + message_descriptor = descriptor_pool.FindMessageTypeByName(type_name) + except KeyError: + return None + message_type = factory.GetPrototype(message_descriptor) + return message_type() + + +class _Printer(object): + """Text format printer for protocol message.""" + + def __init__(self, + out, + indent=0, + as_utf8=False, + as_one_line=False, + pointy_brackets=False, + use_index_order=False, + float_format=None, + use_field_number=False, + descriptor_pool=None): + """Initialize the Printer. + + Floating point values can be formatted compactly with 15 digits of + precision (which is the most that IEEE 754 "double" can guarantee) + using float_format='.15g'. To ensure that converting to text and back to a + proto will result in an identical value, float_format='.17g' should be used. - out.write(' ' * indent); - if field.is_extension: - out.write('[') - if (field.containing_type.GetOptions().message_set_wire_format and - field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and - field.message_type == field.extension_scope and - field.label == descriptor.FieldDescriptor.LABEL_OPTIONAL): - out.write(field.message_type.full_name) + Args: + out: To record the text format result. + indent: The indent level for pretty print. + as_utf8: Produce text output in UTF8 format. + as_one_line: Don't introduce newlines between fields. + pointy_brackets: If True, use angle brackets instead of curly braces for + nesting. + use_index_order: If True, print fields of a proto message using the order + defined in source code instead of the field number. By default, use the + field number order. + float_format: If set, use this to specify floating point number formatting + (per the "Format Specification Mini-Language"); otherwise, str() is + used. + use_field_number: If True, print field numbers instead of names. + descriptor_pool: A DescriptorPool used to resolve Any types. + """ + self.out = out + self.indent = indent + self.as_utf8 = as_utf8 + self.as_one_line = as_one_line + self.pointy_brackets = pointy_brackets + self.use_index_order = use_index_order + self.float_format = float_format + self.use_field_number = use_field_number + self.descriptor_pool = descriptor_pool + + def _TryPrintAsAnyMessage(self, message): + """Serializes if message is a google.protobuf.Any field.""" + packed_message = _BuildMessageFromTypeName(message.TypeName(), + self.descriptor_pool) + if packed_message: + packed_message.MergeFromString(message.value) + self.out.write('%s[%s]' % (self.indent * ' ', message.type_url)) + self._PrintMessageFieldValue(packed_message) + self.out.write(' ' if self.as_one_line else '\n') + return True else: - out.write(field.full_name) - out.write(']') - elif field.type == descriptor.FieldDescriptor.TYPE_GROUP: - # For groups, use the capitalized name. - out.write(field.message_type.name) - else: - out.write(field.name) + return False - if field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_MESSAGE: - # The colon is optional in this case, but our cross-language golden files - # don't include it. - out.write(': ') - - PrintFieldValue(field, value, out, indent, as_utf8, as_one_line) - if as_one_line: - out.write(' ') - else: - out.write('\n') + def PrintMessage(self, message): + """Convert protobuf message to text format. + Args: + message: The protocol buffers message. + """ + if (message.DESCRIPTOR.full_name == _ANY_FULL_TYPE_NAME and + self.descriptor_pool and self._TryPrintAsAnyMessage(message)): + return + fields = message.ListFields() + if self.use_index_order: + fields.sort(key=lambda x: x[0].index) + for field, value in fields: + if _IsMapEntry(field): + for key in sorted(value): + # This is slow for maps with submessage entires because it copies the + # entire tree. Unfortunately this would take significant refactoring + # of this file to work around. + # + # TODO(haberman): refactor and optimize if this becomes an issue. + entry_submsg = field.message_type._concrete_class(key=key, + value=value[key]) + self.PrintField(field, entry_submsg) + elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED: + for element in value: + self.PrintField(field, element) + else: + self.PrintField(field, value) + + def PrintField(self, field, value): + """Print a single field name/value pair.""" + out = self.out + out.write(' ' * self.indent) + if self.use_field_number: + out.write(str(field.number)) + else: + if field.is_extension: + out.write('[') + if (field.containing_type.GetOptions().message_set_wire_format and + field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and + field.label == descriptor.FieldDescriptor.LABEL_OPTIONAL): + out.write(field.message_type.full_name) + else: + out.write(field.full_name) + out.write(']') + elif field.type == descriptor.FieldDescriptor.TYPE_GROUP: + # For groups, use the capitalized name. + out.write(field.message_type.name) + else: + out.write(field.name) -def PrintFieldValue(field, value, out, indent=0, - as_utf8=False, as_one_line=False): - """Print a single field value (not including name). For repeated fields, - the value should be a single element.""" + if field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_MESSAGE: + # The colon is optional in this case, but our cross-language golden files + # don't include it. + out.write(': ') - if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: - if as_one_line: - out.write(' { ') - PrintMessage(value, out, indent, as_utf8, as_one_line) - out.write('}') + self.PrintFieldValue(field, value) + if self.as_one_line: + out.write(' ') else: - out.write(' {\n') - PrintMessage(value, out, indent + 2, as_utf8, as_one_line) - out.write(' ' * indent + '}') - elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM: - enum_value = field.enum_type.values_by_number.get(value, None) - if enum_value is not None: - out.write(enum_value.name) + out.write('\n') + + def _PrintMessageFieldValue(self, value): + if self.pointy_brackets: + openb = '<' + closeb = '>' else: - out.write(str(value)) - elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING: - out.write('\"') - if type(value) is unicode: - out.write(_CEscape(value.encode('utf-8'), as_utf8)) + openb = '{' + closeb = '}' + + if self.as_one_line: + self.out.write(' %s ' % openb) + self.PrintMessage(value) + self.out.write(closeb) else: - out.write(_CEscape(value, as_utf8)) - out.write('\"') - elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL: - if value: - out.write("true") + self.out.write(' %s\n' % openb) + self.indent += 2 + self.PrintMessage(value) + self.indent -= 2 + self.out.write(' ' * self.indent + closeb) + + def PrintFieldValue(self, field, value): + """Print a single field value (not including name). + + For repeated fields, the value should be a single element. + + Args: + field: The descriptor of the field to be printed. + value: The value of the field. + """ + out = self.out + if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: + self._PrintMessageFieldValue(value) + elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM: + enum_value = field.enum_type.values_by_number.get(value, None) + if enum_value is not None: + out.write(enum_value.name) + else: + out.write(str(value)) + elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING: + out.write('\"') + if isinstance(value, six.text_type): + out_value = value.encode('utf-8') + else: + out_value = value + if field.type == descriptor.FieldDescriptor.TYPE_BYTES: + # We need to escape non-UTF8 chars in TYPE_BYTES field. + out_as_utf8 = False + else: + out_as_utf8 = self.as_utf8 + out.write(text_encoding.CEscape(out_value, out_as_utf8)) + out.write('\"') + elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL: + if value: + out.write('true') + else: + out.write('false') + elif field.cpp_type in _FLOAT_TYPES and self.float_format is not None: + out.write('{1:{0}}'.format(self.float_format, value)) else: - out.write("false") - else: - out.write(str(value)) + out.write(str(value)) -def Merge(text, message): - """Merges an ASCII representation of a protocol message into a message. +def Parse(text, + message, + allow_unknown_extension=False, + allow_field_number=False): + """Parses a text representation of a protocol message into a message. Args: - text: Message ASCII representation. + text: Message text representation. message: A protocol buffer message to merge into. + allow_unknown_extension: if True, skip over missing extensions and keep + parsing + allow_field_number: if True, both field number and field name are allowed. + + Returns: + The same message passed as argument. Raises: - ParseError: On ASCII parsing problems. + ParseError: On text parsing problems. """ - tokenizer = _Tokenizer(text) - while not tokenizer.AtEnd(): - _MergeField(tokenizer, message) + if not isinstance(text, str): + text = text.decode('utf-8') + return ParseLines( + text.split('\n'), message, allow_unknown_extension, allow_field_number) -def _MergeField(tokenizer, message): - """Merges a single protocol message field into a message. +def Merge(text, + message, + allow_unknown_extension=False, + allow_field_number=False, + descriptor_pool=None): + """Parses a text representation of a protocol message into a message. + + Like Parse(), but allows repeated values for a non-repeated field, and uses + the last one. Args: - tokenizer: A tokenizer to parse the field name and values. - message: A protocol message to record the data. + text: Message text representation. + message: A protocol buffer message to merge into. + allow_unknown_extension: if True, skip over missing extensions and keep + parsing + allow_field_number: if True, both field number and field name are allowed. + descriptor_pool: A DescriptorPool used to resolve Any types. + + Returns: + The same message passed as argument. Raises: - ParseError: In case of ASCII parsing problems. + ParseError: On text parsing problems. """ - message_descriptor = message.DESCRIPTOR - if tokenizer.TryConsume('['): + return MergeLines( + text.split('\n'), + message, + allow_unknown_extension, + allow_field_number, + descriptor_pool=descriptor_pool) + + +def ParseLines(lines, + message, + allow_unknown_extension=False, + allow_field_number=False): + """Parses a text representation of a protocol message into a message. + + Args: + lines: An iterable of lines of a message's text representation. + message: A protocol buffer message to merge into. + allow_unknown_extension: if True, skip over missing extensions and keep + parsing + allow_field_number: if True, both field number and field name are allowed. + descriptor_pool: A DescriptorPool used to resolve Any types. + + Returns: + The same message passed as argument. + + Raises: + ParseError: On text parsing problems. + """ + parser = _Parser(allow_unknown_extension, allow_field_number) + return parser.ParseLines(lines, message) + + +def MergeLines(lines, + message, + allow_unknown_extension=False, + allow_field_number=False, + descriptor_pool=None): + """Parses a text representation of a protocol message into a message. + + Args: + lines: An iterable of lines of a message's text representation. + message: A protocol buffer message to merge into. + allow_unknown_extension: if True, skip over missing extensions and keep + parsing + allow_field_number: if True, both field number and field name are allowed. + + Returns: + The same message passed as argument. + + Raises: + ParseError: On text parsing problems. + """ + parser = _Parser(allow_unknown_extension, + allow_field_number, + descriptor_pool=descriptor_pool) + return parser.MergeLines(lines, message) + + +class _Parser(object): + """Text format parser for protocol message.""" + + def __init__(self, + allow_unknown_extension=False, + allow_field_number=False, + descriptor_pool=None): + self.allow_unknown_extension = allow_unknown_extension + self.allow_field_number = allow_field_number + self.descriptor_pool = descriptor_pool + + def ParseFromString(self, text, message): + """Parses a text representation of a protocol message into a message.""" + if not isinstance(text, str): + text = text.decode('utf-8') + return self.ParseLines(text.split('\n'), message) + + def ParseLines(self, lines, message): + """Parses a text representation of a protocol message into a message.""" + self._allow_multiple_scalars = False + self._ParseOrMerge(lines, message) + return message + + def MergeFromString(self, text, message): + """Merges a text representation of a protocol message into a message.""" + return self._MergeLines(text.split('\n'), message) + + def MergeLines(self, lines, message): + """Merges a text representation of a protocol message into a message.""" + self._allow_multiple_scalars = True + self._ParseOrMerge(lines, message) + return message + + def _ParseOrMerge(self, lines, message): + """Converts a text representation of a protocol message into a message. + + Args: + lines: Lines of a message's text representation. + message: A protocol buffer message to merge into. + + Raises: + ParseError: On text parsing problems. + """ + tokenizer = Tokenizer(lines) + while not tokenizer.AtEnd(): + self._MergeField(tokenizer, message) + + def _MergeField(self, tokenizer, message): + """Merges a single protocol message field into a message. + + Args: + tokenizer: A tokenizer to parse the field name and values. + message: A protocol message to record the data. + + Raises: + ParseError: In case of text parsing problems. + """ + message_descriptor = message.DESCRIPTOR + if (hasattr(message_descriptor, 'syntax') and + message_descriptor.syntax == 'proto3'): + # Proto3 doesn't represent presence so we can't test if multiple + # scalars have occurred. We have to allow them. + self._allow_multiple_scalars = True + if tokenizer.TryConsume('['): + name = [tokenizer.ConsumeIdentifier()] + while tokenizer.TryConsume('.'): + name.append(tokenizer.ConsumeIdentifier()) + name = '.'.join(name) + + if not message_descriptor.is_extendable: + raise tokenizer.ParseErrorPreviousToken( + 'Message type "%s" does not have extensions.' % + message_descriptor.full_name) + # pylint: disable=protected-access + field = message.Extensions._FindExtensionByName(name) + # pylint: enable=protected-access + if not field: + if self.allow_unknown_extension: + field = None + else: + raise tokenizer.ParseErrorPreviousToken( + 'Extension "%s" not registered.' % name) + elif message_descriptor != field.containing_type: + raise tokenizer.ParseErrorPreviousToken( + 'Extension "%s" does not extend message type "%s".' % + (name, message_descriptor.full_name)) + + tokenizer.Consume(']') + + else: + name = tokenizer.ConsumeIdentifierOrNumber() + if self.allow_field_number and name.isdigit(): + number = ParseInteger(name, True, True) + field = message_descriptor.fields_by_number.get(number, None) + if not field and message_descriptor.is_extendable: + field = message.Extensions._FindExtensionByNumber(number) + else: + field = message_descriptor.fields_by_name.get(name, None) + + # Group names are expected to be capitalized as they appear in the + # .proto file, which actually matches their type names, not their field + # names. + if not field: + field = message_descriptor.fields_by_name.get(name.lower(), None) + if field and field.type != descriptor.FieldDescriptor.TYPE_GROUP: + field = None + + if (field and field.type == descriptor.FieldDescriptor.TYPE_GROUP and + field.message_type.name != name): + field = None + + if not field: + raise tokenizer.ParseErrorPreviousToken( + 'Message type "%s" has no field named "%s".' % + (message_descriptor.full_name, name)) + + if field: + if not self._allow_multiple_scalars and field.containing_oneof: + # Check if there's a different field set in this oneof. + # Note that we ignore the case if the same field was set before, and we + # apply _allow_multiple_scalars to non-scalar fields as well. + which_oneof = message.WhichOneof(field.containing_oneof.name) + if which_oneof is not None and which_oneof != field.name: + raise tokenizer.ParseErrorPreviousToken( + 'Field "%s" is specified along with field "%s", another member ' + 'of oneof "%s" for message type "%s".' % + (field.name, which_oneof, field.containing_oneof.name, + message_descriptor.full_name)) + + if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: + tokenizer.TryConsume(':') + merger = self._MergeMessageField + else: + tokenizer.Consume(':') + merger = self._MergeScalarField + + if (field.label == descriptor.FieldDescriptor.LABEL_REPEATED and + tokenizer.TryConsume('[')): + # Short repeated format, e.g. "foo: [1, 2, 3]" + while True: + merger(tokenizer, message, field) + if tokenizer.TryConsume(']'): + break + tokenizer.Consume(',') + + else: + merger(tokenizer, message, field) + + else: # Proto field is unknown. + assert self.allow_unknown_extension + _SkipFieldContents(tokenizer) + + # For historical reasons, fields may optionally be separated by commas or + # semicolons. + if not tokenizer.TryConsume(','): + tokenizer.TryConsume(';') + + def _ConsumeAnyTypeUrl(self, tokenizer): + """Consumes a google.protobuf.Any type URL and returns the type name.""" + # Consume "type.googleapis.com/". + tokenizer.ConsumeIdentifier() + tokenizer.Consume('.') + tokenizer.ConsumeIdentifier() + tokenizer.Consume('.') + tokenizer.ConsumeIdentifier() + tokenizer.Consume('/') + # Consume the fully-qualified type name. name = [tokenizer.ConsumeIdentifier()] while tokenizer.TryConsume('.'): name.append(tokenizer.ConsumeIdentifier()) - name = '.'.join(name) - - if not message_descriptor.is_extendable: - raise tokenizer.ParseErrorPreviousToken( - 'Message type "%s" does not have extensions.' % - message_descriptor.full_name) - field = message.Extensions._FindExtensionByName(name) - if not field: - raise tokenizer.ParseErrorPreviousToken( - 'Extension "%s" not registered.' % name) - elif message_descriptor != field.containing_type: - raise tokenizer.ParseErrorPreviousToken( - 'Extension "%s" does not extend message type "%s".' % ( - name, message_descriptor.full_name)) - tokenizer.Consume(']') - else: - name = tokenizer.ConsumeIdentifier() - field = message_descriptor.fields_by_name.get(name, None) + return '.'.join(name) - # Group names are expected to be capitalized as they appear in the - # .proto file, which actually matches their type names, not their field - # names. - if not field: - field = message_descriptor.fields_by_name.get(name.lower(), None) - if field and field.type != descriptor.FieldDescriptor.TYPE_GROUP: - field = None + def _MergeMessageField(self, tokenizer, message, field): + """Merges a single scalar field into a message. - if (field and field.type == descriptor.FieldDescriptor.TYPE_GROUP and - field.message_type.name != name): - field = None - - if not field: - raise tokenizer.ParseErrorPreviousToken( - 'Message type "%s" has no field named "%s".' % ( - message_descriptor.full_name, name)) + Args: + tokenizer: A tokenizer to parse the field value. + message: The message of which field is a member. + field: The descriptor of the field to be merged. - if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: - tokenizer.TryConsume(':') + Raises: + ParseError: In case of text parsing problems. + """ + is_map_entry = _IsMapEntry(field) if tokenizer.TryConsume('<'): end_token = '>' @@ -218,9 +718,39 @@ def _MergeField(tokenizer, message): tokenizer.Consume('{') end_token = '}' - if field.label == descriptor.FieldDescriptor.LABEL_REPEATED: + if (field.message_type.full_name == _ANY_FULL_TYPE_NAME and + tokenizer.TryConsume('[')): + packed_type_name = self._ConsumeAnyTypeUrl(tokenizer) + tokenizer.Consume(']') + tokenizer.TryConsume(':') + if tokenizer.TryConsume('<'): + expanded_any_end_token = '>' + else: + tokenizer.Consume('{') + expanded_any_end_token = '}' + if not self.descriptor_pool: + raise ParseError('Descriptor pool required to parse expanded Any field') + expanded_any_sub_message = _BuildMessageFromTypeName(packed_type_name, + self.descriptor_pool) + if not expanded_any_sub_message: + raise ParseError('Type %s not found in descriptor pool' % + packed_type_name) + while not tokenizer.TryConsume(expanded_any_end_token): + if tokenizer.AtEnd(): + raise tokenizer.ParseErrorPreviousToken('Expected "%s".' % + (expanded_any_end_token,)) + self._MergeField(tokenizer, expanded_any_sub_message) + if field.label == descriptor.FieldDescriptor.LABEL_REPEATED: + any_message = getattr(message, field.name).add() + else: + any_message = getattr(message, field.name) + any_message.Pack(expanded_any_sub_message) + elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED: if field.is_extension: sub_message = message.Extensions[field].add() + elif is_map_entry: + # pylint: disable=protected-access + sub_message = field.message_type._concrete_class() else: sub_message = getattr(message, field.name).add() else: @@ -232,69 +762,167 @@ def _MergeField(tokenizer, message): while not tokenizer.TryConsume(end_token): if tokenizer.AtEnd(): - raise tokenizer.ParseErrorPreviousToken('Expected "%s".' % (end_token)) - _MergeField(tokenizer, sub_message) - else: - _MergeScalarField(tokenizer, message, field) + raise tokenizer.ParseErrorPreviousToken('Expected "%s".' % (end_token,)) + self._MergeField(tokenizer, sub_message) + + if is_map_entry: + value_cpptype = field.message_type.fields_by_name['value'].cpp_type + if value_cpptype == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: + value = getattr(message, field.name)[sub_message.key] + value.MergeFrom(sub_message.value) + else: + getattr(message, field.name)[sub_message.key] = sub_message.value + def _MergeScalarField(self, tokenizer, message, field): + """Merges a single scalar field into a message. -def _MergeScalarField(tokenizer, message, field): - """Merges a single protocol message scalar field into a message. + Args: + tokenizer: A tokenizer to parse the field value. + message: A protocol message to record the data. + field: The descriptor of the field to be merged. + + Raises: + ParseError: In case of text parsing problems. + RuntimeError: On runtime errors. + """ + _ = self.allow_unknown_extension + value = None + + if field.type in (descriptor.FieldDescriptor.TYPE_INT32, + descriptor.FieldDescriptor.TYPE_SINT32, + descriptor.FieldDescriptor.TYPE_SFIXED32): + value = _ConsumeInt32(tokenizer) + elif field.type in (descriptor.FieldDescriptor.TYPE_INT64, + descriptor.FieldDescriptor.TYPE_SINT64, + descriptor.FieldDescriptor.TYPE_SFIXED64): + value = _ConsumeInt64(tokenizer) + elif field.type in (descriptor.FieldDescriptor.TYPE_UINT32, + descriptor.FieldDescriptor.TYPE_FIXED32): + value = _ConsumeUint32(tokenizer) + elif field.type in (descriptor.FieldDescriptor.TYPE_UINT64, + descriptor.FieldDescriptor.TYPE_FIXED64): + value = _ConsumeUint64(tokenizer) + elif field.type in (descriptor.FieldDescriptor.TYPE_FLOAT, + descriptor.FieldDescriptor.TYPE_DOUBLE): + value = tokenizer.ConsumeFloat() + elif field.type == descriptor.FieldDescriptor.TYPE_BOOL: + value = tokenizer.ConsumeBool() + elif field.type == descriptor.FieldDescriptor.TYPE_STRING: + value = tokenizer.ConsumeString() + elif field.type == descriptor.FieldDescriptor.TYPE_BYTES: + value = tokenizer.ConsumeByteString() + elif field.type == descriptor.FieldDescriptor.TYPE_ENUM: + value = tokenizer.ConsumeEnum(field) + else: + raise RuntimeError('Unknown field type %d' % field.type) + + if field.label == descriptor.FieldDescriptor.LABEL_REPEATED: + if field.is_extension: + message.Extensions[field].append(value) + else: + getattr(message, field.name).append(value) + else: + if field.is_extension: + if not self._allow_multiple_scalars and message.HasExtension(field): + raise tokenizer.ParseErrorPreviousToken( + 'Message type "%s" should not have multiple "%s" extensions.' % + (message.DESCRIPTOR.full_name, field.full_name)) + else: + message.Extensions[field] = value + else: + if not self._allow_multiple_scalars and message.HasField(field.name): + raise tokenizer.ParseErrorPreviousToken( + 'Message type "%s" should not have multiple "%s" fields.' % + (message.DESCRIPTOR.full_name, field.name)) + else: + setattr(message, field.name, value) + + +def _SkipFieldContents(tokenizer): + """Skips over contents (value or message) of a field. Args: - tokenizer: A tokenizer to parse the field value. - message: A protocol message to record the data. - field: The descriptor of the field to be merged. + tokenizer: A tokenizer to parse the field name and values. + """ + # Try to guess the type of this field. + # If this field is not a message, there should be a ":" between the + # field name and the field value and also the field value should not + # start with "{" or "<" which indicates the beginning of a message body. + # If there is no ":" or there is a "{" or "<" after ":", this field has + # to be a message or the input is ill-formed. + if tokenizer.TryConsume(':') and not tokenizer.LookingAt( + '{') and not tokenizer.LookingAt('<'): + _SkipFieldValue(tokenizer) + else: + _SkipFieldMessage(tokenizer) - Raises: - ParseError: In case of ASCII parsing problems. - RuntimeError: On runtime errors. + +def _SkipField(tokenizer): + """Skips over a complete field (name and value/message). + + Args: + tokenizer: A tokenizer to parse the field name and values. """ - tokenizer.Consume(':') - value = None - - if field.type in (descriptor.FieldDescriptor.TYPE_INT32, - descriptor.FieldDescriptor.TYPE_SINT32, - descriptor.FieldDescriptor.TYPE_SFIXED32): - value = tokenizer.ConsumeInt32() - elif field.type in (descriptor.FieldDescriptor.TYPE_INT64, - descriptor.FieldDescriptor.TYPE_SINT64, - descriptor.FieldDescriptor.TYPE_SFIXED64): - value = tokenizer.ConsumeInt64() - elif field.type in (descriptor.FieldDescriptor.TYPE_UINT32, - descriptor.FieldDescriptor.TYPE_FIXED32): - value = tokenizer.ConsumeUint32() - elif field.type in (descriptor.FieldDescriptor.TYPE_UINT64, - descriptor.FieldDescriptor.TYPE_FIXED64): - value = tokenizer.ConsumeUint64() - elif field.type in (descriptor.FieldDescriptor.TYPE_FLOAT, - descriptor.FieldDescriptor.TYPE_DOUBLE): - value = tokenizer.ConsumeFloat() - elif field.type == descriptor.FieldDescriptor.TYPE_BOOL: - value = tokenizer.ConsumeBool() - elif field.type == descriptor.FieldDescriptor.TYPE_STRING: - value = tokenizer.ConsumeString() - elif field.type == descriptor.FieldDescriptor.TYPE_BYTES: - value = tokenizer.ConsumeByteString() - elif field.type == descriptor.FieldDescriptor.TYPE_ENUM: - value = tokenizer.ConsumeEnum(field) + if tokenizer.TryConsume('['): + # Consume extension name. + tokenizer.ConsumeIdentifier() + while tokenizer.TryConsume('.'): + tokenizer.ConsumeIdentifier() + tokenizer.Consume(']') else: - raise RuntimeError('Unknown field type %d' % field.type) + tokenizer.ConsumeIdentifier() - if field.label == descriptor.FieldDescriptor.LABEL_REPEATED: - if field.is_extension: - message.Extensions[field].append(value) - else: - getattr(message, field.name).append(value) + _SkipFieldContents(tokenizer) + + # For historical reasons, fields may optionally be separated by commas or + # semicolons. + if not tokenizer.TryConsume(','): + tokenizer.TryConsume(';') + + +def _SkipFieldMessage(tokenizer): + """Skips over a field message. + + Args: + tokenizer: A tokenizer to parse the field name and values. + """ + + if tokenizer.TryConsume('<'): + delimiter = '>' else: - if field.is_extension: - message.Extensions[field] = value - else: - setattr(message, field.name, value) + tokenizer.Consume('{') + delimiter = '}' + while not tokenizer.LookingAt('>') and not tokenizer.LookingAt('}'): + _SkipField(tokenizer) + + tokenizer.Consume(delimiter) + + +def _SkipFieldValue(tokenizer): + """Skips over a field value. + + Args: + tokenizer: A tokenizer to parse the field name and values. + + Raises: + ParseError: In case an invalid field value is found. + """ + # String/bytes tokens can come in multiple adjacent string literals. + # If we can consume one, consume as many as we can. + if tokenizer.TryConsumeByteString(): + while tokenizer.TryConsumeByteString(): + pass + return -class _Tokenizer(object): - """Protocol buffer ASCII representation tokenizer. + if (not tokenizer.TryConsumeIdentifier() and + not _TryConsumeInt64(tokenizer) and not _TryConsumeUint64(tokenizer) and + not tokenizer.TryConsumeFloat()): + raise ParseError('Invalid field value: ' + tokenizer.token) + + +class Tokenizer(object): + """Protocol buffer text representation tokenizer. This class handles the lower level string parsing by splitting it into meaningful tokens. @@ -302,50 +930,63 @@ class _Tokenizer(object): It was directly ported from the Java protocol buffer API. """ - _WHITESPACE = re.compile('(\\s|(#.*$))+', re.MULTILINE) - _TOKEN = re.compile( - '[a-zA-Z_][0-9a-zA-Z_+-]*|' # an identifier - '[0-9+-][0-9a-zA-Z_.+-]*|' # a number - '\"([^\"\n\\\\]|\\\\.)*(\"|\\\\?$)|' # a double-quoted string - '\'([^\'\n\\\\]|\\\\.)*(\'|\\\\?$)') # a single-quoted string - _IDENTIFIER = re.compile('\w+') + _WHITESPACE = re.compile(r'\s+') + _COMMENT = re.compile(r'(\s*#.*$)', re.MULTILINE) + _WHITESPACE_OR_COMMENT = re.compile(r'(\s|(#.*$))+', re.MULTILINE) + _TOKEN = re.compile('|'.join([ + r'[a-zA-Z_][0-9a-zA-Z_+-]*', # an identifier + r'([0-9+-]|(\.[0-9]))[0-9a-zA-Z_.+-]*', # a number + ] + [ # quoted str for each quote mark + r'{qt}([^{qt}\n\\]|\\.)*({qt}|\\?$)'.format(qt=mark) for mark in _QUOTES + ])) - def __init__(self, text_message): - self._text_message = text_message + _IDENTIFIER = re.compile(r'[^\d\W]\w*') + _IDENTIFIER_OR_NUMBER = re.compile(r'\w+') + def __init__(self, lines, skip_comments=True): self._position = 0 self._line = -1 self._column = 0 self._token_start = None self.token = '' - self._lines = deque(text_message.split('\n')) + self._lines = iter(lines) self._current_line = '' self._previous_line = 0 self._previous_column = 0 + self._more_lines = True + self._skip_comments = skip_comments + self._whitespace_pattern = (skip_comments and self._WHITESPACE_OR_COMMENT + or self._WHITESPACE) self._SkipWhitespace() self.NextToken() + def LookingAt(self, token): + return self.token == token + def AtEnd(self): """Checks the end of the text was reached. Returns: True iff the end was reached. """ - return self.token == '' + return not self.token def _PopLine(self): while len(self._current_line) <= self._column: - if not self._lines: + try: + self._current_line = next(self._lines) + except StopIteration: self._current_line = '' + self._more_lines = False return - self._line += 1 - self._column = 0 - self._current_line = self._lines.popleft() + else: + self._line += 1 + self._column = 0 def _SkipWhitespace(self): while True: self._PopLine() - match = self._WHITESPACE.match(self._current_line, self._column) + match = self._whitespace_pattern.match(self._current_line, self._column) if not match: break length = len(match.group(0)) @@ -375,7 +1016,21 @@ class _Tokenizer(object): ParseError: If the text couldn't be consumed. """ if not self.TryConsume(token): - raise self._ParseError('Expected "%s".' % token) + raise self.ParseError('Expected "%s".' % token) + + def ConsumeComment(self): + result = self.token + if not self._COMMENT.match(result): + raise self.ParseError('Expected comment.') + self.NextToken() + return result + + def TryConsumeIdentifier(self): + try: + self.ConsumeIdentifier() + return True + except ParseError: + return False def ConsumeIdentifier(self): """Consumes protocol message field identifier. @@ -388,74 +1043,65 @@ class _Tokenizer(object): """ result = self.token if not self._IDENTIFIER.match(result): - raise self._ParseError('Expected identifier.') + raise self.ParseError('Expected identifier.') self.NextToken() return result - def ConsumeInt32(self): - """Consumes a signed 32bit integer number. - - Returns: - The integer parsed. - - Raises: - ParseError: If a signed 32bit integer couldn't be consumed. - """ + def TryConsumeIdentifierOrNumber(self): try: - result = ParseInteger(self.token, is_signed=True, is_long=False) - except ValueError, e: - raise self._ParseError(str(e)) - self.NextToken() - return result + self.ConsumeIdentifierOrNumber() + return True + except ParseError: + return False - def ConsumeUint32(self): - """Consumes an unsigned 32bit integer number. + def ConsumeIdentifierOrNumber(self): + """Consumes protocol message field identifier. Returns: - The integer parsed. + Identifier string. Raises: - ParseError: If an unsigned 32bit integer couldn't be consumed. + ParseError: If an identifier couldn't be consumed. """ - try: - result = ParseInteger(self.token, is_signed=False, is_long=False) - except ValueError, e: - raise self._ParseError(str(e)) + result = self.token + if not self._IDENTIFIER_OR_NUMBER.match(result): + raise self.ParseError('Expected identifier or number.') self.NextToken() return result - def ConsumeInt64(self): - """Consumes a signed 64bit integer number. - - Returns: - The integer parsed. - - Raises: - ParseError: If a signed 64bit integer couldn't be consumed. - """ + def TryConsumeInteger(self): try: - result = ParseInteger(self.token, is_signed=True, is_long=True) - except ValueError, e: - raise self._ParseError(str(e)) - self.NextToken() - return result + # Note: is_long only affects value type, not whether an error is raised. + self.ConsumeInteger() + return True + except ParseError: + return False - def ConsumeUint64(self): - """Consumes an unsigned 64bit integer number. + def ConsumeInteger(self, is_long=False): + """Consumes an integer number. + Args: + is_long: True if the value should be returned as a long integer. Returns: The integer parsed. Raises: - ParseError: If an unsigned 64bit integer couldn't be consumed. + ParseError: If an integer couldn't be consumed. """ try: - result = ParseInteger(self.token, is_signed=False, is_long=True) - except ValueError, e: - raise self._ParseError(str(e)) + result = _ParseAbstractInteger(self.token, is_long=is_long) + except ValueError as e: + raise self.ParseError(str(e)) self.NextToken() return result + def TryConsumeFloat(self): + try: + self.ConsumeFloat() + return True + except ParseError: + return False + def ConsumeFloat(self): """Consumes an floating point number. @@ -467,8 +1113,8 @@ class _Tokenizer(object): """ try: result = ParseFloat(self.token) - except ValueError, e: - raise self._ParseError(str(e)) + except ValueError as e: + raise self.ParseError(str(e)) self.NextToken() return result @@ -483,11 +1129,18 @@ class _Tokenizer(object): """ try: result = ParseBool(self.token) - except ValueError, e: - raise self._ParseError(str(e)) + except ValueError as e: + raise self.ParseError(str(e)) self.NextToken() return result + def TryConsumeByteString(self): + try: + self.ConsumeByteString() + return True + except ParseError: + return False + def ConsumeString(self): """Consumes a string value. @@ -497,10 +1150,10 @@ class _Tokenizer(object): Raises: ParseError: If a string value couldn't be consumed. """ - bytes = self.ConsumeByteString() + the_bytes = self.ConsumeByteString() try: - return unicode(bytes, 'utf-8') - except UnicodeDecodeError, e: + return six.text_type(the_bytes, 'utf-8') + except UnicodeDecodeError as e: raise self._StringParseError(e) def ConsumeByteString(self): @@ -512,10 +1165,10 @@ class _Tokenizer(object): Raises: ParseError: If a byte array value couldn't be consumed. """ - list = [self._ConsumeSingleByteString()] - while len(self.token) > 0 and self.token[0] in ('\'', '"'): - list.append(self._ConsumeSingleByteString()) - return "".join(list) + the_list = [self._ConsumeSingleByteString()] + while self.token and self.token[0] in _QUOTES: + the_list.append(self._ConsumeSingleByteString()) + return b''.join(the_list) def _ConsumeSingleByteString(self): """Consume one token of a string literal. @@ -523,26 +1176,31 @@ class _Tokenizer(object): String literals (whether bytes or text) can come in multiple adjacent tokens which are automatically concatenated, like in C or Python. This method only consumes one token. + + Returns: + The token parsed. + Raises: + ParseError: When the wrong format data is found. """ text = self.token - if len(text) < 1 or text[0] not in ('\'', '"'): - raise self._ParseError('Expected string.') + if len(text) < 1 or text[0] not in _QUOTES: + raise self.ParseError('Expected string but found: %r' % (text,)) if len(text) < 2 or text[-1] != text[0]: - raise self._ParseError('String missing ending quote.') + raise self.ParseError('String missing ending quote: %r' % (text,)) try: - result = _CUnescape(text[1:-1]) - except ValueError, e: - raise self._ParseError(str(e)) + result = text_encoding.CUnescape(text[1:-1]) + except ValueError as e: + raise self.ParseError(str(e)) self.NextToken() return result def ConsumeEnum(self, field): try: result = ParseEnum(field, self.token) - except ValueError, e: - raise self._ParseError(str(e)) + except ValueError as e: + raise self.ParseError(str(e)) self.NextToken() return result @@ -555,16 +1213,15 @@ class _Tokenizer(object): Returns: A ParseError instance. """ - return ParseError('%d:%d : %s' % ( - self._previous_line + 1, self._previous_column + 1, message)) + return ParseError(message, self._previous_line + 1, + self._previous_column + 1) - def _ParseError(self, message): + def ParseError(self, message): """Creates and *returns* a ParseError for the current token.""" - return ParseError('%d:%d : %s' % ( - self._line + 1, self._column + 1, message)) + return ParseError(message, self._line + 1, self._column + 1) def _StringParseError(self, e): - return self._ParseError('Couldn\'t parse string: ' + str(e)) + return self.ParseError('Couldn\'t parse string: ' + str(e)) def NextToken(self): """Reads the next meaningful token.""" @@ -574,55 +1231,128 @@ class _Tokenizer(object): self._column += len(self.token) self._SkipWhitespace() - if not self._lines and len(self._current_line) <= self._column: + if not self._more_lines: self.token = '' return match = self._TOKEN.match(self._current_line, self._column) + if not match and not self._skip_comments: + match = self._COMMENT.match(self._current_line, self._column) if match: token = match.group(0) self.token = token else: self.token = self._current_line[self._column] +# Aliased so it can still be accessed by current visibility violators. +# TODO(dbarnett): Migrate violators to textformat_tokenizer. +_Tokenizer = Tokenizer # pylint: disable=invalid-name + + +def _ConsumeInt32(tokenizer): + """Consumes a signed 32bit integer number from tokenizer. + + Args: + tokenizer: A tokenizer used to parse the number. + + Returns: + The integer parsed. + + Raises: + ParseError: If a signed 32bit integer couldn't be consumed. + """ + return _ConsumeInteger(tokenizer, is_signed=True, is_long=False) + + +def _ConsumeUint32(tokenizer): + """Consumes an unsigned 32bit integer number from tokenizer. + + Args: + tokenizer: A tokenizer used to parse the number. + + Returns: + The integer parsed. + + Raises: + ParseError: If an unsigned 32bit integer couldn't be consumed. + """ + return _ConsumeInteger(tokenizer, is_signed=False, is_long=False) + + +def _TryConsumeInt64(tokenizer): + try: + _ConsumeInt64(tokenizer) + return True + except ParseError: + return False + + +def _ConsumeInt64(tokenizer): + """Consumes a signed 32bit integer number from tokenizer. + + Args: + tokenizer: A tokenizer used to parse the number. + + Returns: + The integer parsed. + + Raises: + ParseError: If a signed 32bit integer couldn't be consumed. + """ + return _ConsumeInteger(tokenizer, is_signed=True, is_long=True) + + +def _TryConsumeUint64(tokenizer): + try: + _ConsumeUint64(tokenizer) + return True + except ParseError: + return False + + +def _ConsumeUint64(tokenizer): + """Consumes an unsigned 64bit integer number from tokenizer. + + Args: + tokenizer: A tokenizer used to parse the number. + + Returns: + The integer parsed. -# text.encode('string_escape') does not seem to satisfy our needs as it -# encodes unprintable characters using two-digit hex escapes whereas our -# C++ unescaping function allows hex escapes to be any length. So, -# "\0011".encode('string_escape') ends up being "\\x011", which will be -# decoded in C++ as a single-character string with char code 0x11. -def _CEscape(text, as_utf8): - def escape(c): - o = ord(c) - if o == 10: return r"\n" # optional escape - if o == 13: return r"\r" # optional escape - if o == 9: return r"\t" # optional escape - if o == 39: return r"\'" # optional escape + Raises: + ParseError: If an unsigned 64bit integer couldn't be consumed. + """ + return _ConsumeInteger(tokenizer, is_signed=False, is_long=True) - if o == 34: return r'\"' # necessary escape - if o == 92: return r"\\" # necessary escape - # necessary escapes - if not as_utf8 and (o >= 127 or o < 32): return "\\%03o" % o - return c - return "".join([escape(c) for c in text]) +def _TryConsumeInteger(tokenizer, is_signed=False, is_long=False): + try: + _ConsumeInteger(tokenizer, is_signed=is_signed, is_long=is_long) + return True + except ParseError: + return False -_CUNESCAPE_HEX = re.compile(r'(\\+)x([0-9a-fA-F])(?![0-9a-fA-F])') +def _ConsumeInteger(tokenizer, is_signed=False, is_long=False): + """Consumes an integer number from tokenizer. + Args: + tokenizer: A tokenizer used to parse the number. + is_signed: True if a signed integer must be parsed. + is_long: True if a long integer must be parsed. -def _CUnescape(text): - def ReplaceHex(m): - # Only replace the match if the number of leading back slashes is odd. i.e. - # the slash itself is not escaped. - if len(m.group(1)) & 1: - return m.group(1) + 'x0' + m.group(2) - return m.group(0) + Returns: + The integer parsed. - # This is required because the 'string_escape' encoding doesn't - # allow single-digit hex escapes (like '\xf'). - result = _CUNESCAPE_HEX.sub(ReplaceHex, text) - return result.decode('string_escape') + Raises: + ParseError: If an integer with given characteristics couldn't be consumed. + """ + try: + result = ParseInteger(tokenizer.token, is_signed=is_signed, is_long=is_long) + except ValueError as e: + raise tokenizer.ParseError(str(e)) + tokenizer.NextToken() + return result def ParseInteger(text, is_signed=False, is_long=False): @@ -640,10 +1370,7 @@ def ParseInteger(text, is_signed=False, is_long=False): ValueError: Thrown Iff the text is not a valid integer. """ # Do the actual parsing. Exception handling is propagated to caller. - try: - result = int(text, 0) - except ValueError: - raise ValueError('Couldn\'t parse integer: %s' % text) + result = _ParseAbstractInteger(text, is_long=is_long) # Check if the integer is sane. Exceptions handled by callers. checker = _INTEGER_CHECKERS[2 * int(is_long) + int(is_signed)] @@ -651,6 +1378,32 @@ def ParseInteger(text, is_signed=False, is_long=False): return result +def _ParseAbstractInteger(text, is_long=False): + """Parses an integer without checking size/signedness. + + Args: + text: The text to parse. + is_long: True if the value should be returned as a long integer. + + Returns: + The integer value. + + Raises: + ValueError: Thrown Iff the text is not a valid integer. + """ + # Do the actual parsing. Exception handling is propagated to caller. + try: + # We force 32-bit values to int and 64-bit values to long to make + # alternate implementations where the distinction is more significant + # (e.g. the C++ implementation) simpler. + if is_long: + return long(text, 0) + else: + return int(text, 0) + except ValueError: + raise ValueError('Couldn\'t parse integer: %s' % text) + + def ParseFloat(text): """Parse a floating point number. @@ -726,14 +1479,12 @@ def ParseEnum(field, value): # Identifier. enum_value = enum_descriptor.values_by_name.get(value, None) if enum_value is None: - raise ValueError( - 'Enum type "%s" has no value named %s.' % ( - enum_descriptor.full_name, value)) + raise ValueError('Enum type "%s" has no value named %s.' % + (enum_descriptor.full_name, value)) else: # Numeric value. enum_value = enum_descriptor.values_by_number.get(number, None) if enum_value is None: - raise ValueError( - 'Enum type "%s" has no value with number %d.' % ( - enum_descriptor.full_name, number)) + raise ValueError('Enum type "%s" has no value with number %d.' % + (enum_descriptor.full_name, number)) return enum_value.number |